From 0559eaff5ac4afae14dcbb6396da2a0c3f325cd8 Mon Sep 17 00:00:00 2001 From: Nishith Kumar M Shah Date: Wed, 5 Jun 2024 11:42:13 -0700 Subject: [PATCH 001/462] Revert "Pass LangOpts from CompilerInstance to DependencyScanningWorker (#93753)" (#94488) This reverts commit 9862080b1cbf685c0d462b29596e3f7206d24aa2. --- .../clang/Lex/DependencyDirectivesScanner.h | 3 +- .../DependencyScanningFilesystem.h | 3 +- clang/lib/Frontend/FrontendActions.cpp | 4 +- clang/lib/Lex/DependencyDirectivesScanner.cpp | 22 ++--- .../DependencyScanningFilesystem.cpp | 4 +- .../DependencyScanningWorker.cpp | 5 +- .../Lex/DependencyDirectivesScannerTest.cpp | 82 +++---------------- .../Lex/PPDependencyDirectivesTest.cpp | 3 +- 8 files changed, 31 insertions(+), 95 deletions(-) diff --git a/clang/include/clang/Lex/DependencyDirectivesScanner.h b/clang/include/clang/Lex/DependencyDirectivesScanner.h index 2f8354dec939..0e115906fbfe 100644 --- a/clang/include/clang/Lex/DependencyDirectivesScanner.h +++ b/clang/include/clang/Lex/DependencyDirectivesScanner.h @@ -17,7 +17,6 @@ #ifndef LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSCANNER_H #define LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSCANNER_H -#include "clang/Basic/LangOptions.h" #include "clang/Basic/SourceLocation.h" #include "llvm/ADT/ArrayRef.h" @@ -118,7 +117,7 @@ struct Directive { bool scanSourceForDependencyDirectives( StringRef Input, SmallVectorImpl &Tokens, SmallVectorImpl &Directives, - const LangOptions &LangOpts, DiagnosticsEngine *Diags = nullptr, + DiagnosticsEngine *Diags = nullptr, SourceLocation InputSourceLoc = SourceLocation()); /// Print the previously scanned dependency directives as minimized source text. diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h index 9dc20065a09a..f7b4510d7f7b 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h @@ -363,8 +363,7 @@ public: /// /// Returns true if the directive tokens are populated for this file entry, /// false if not (i.e. this entry is not a file or its scan fails). - bool ensureDirectiveTokensArePopulated(EntryRef Entry, - const LangOptions &LangOpts); + bool ensureDirectiveTokensArePopulated(EntryRef Entry); /// Check whether \p Path exists. By default checks cached result of \c /// status(), and falls back on FS if unable to do so. diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index 1812b85860f6..4f064321997a 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -1169,8 +1169,8 @@ void PrintDependencyDirectivesSourceMinimizerAction::ExecuteAction() { llvm::SmallVector Tokens; llvm::SmallVector Directives; if (scanSourceForDependencyDirectives( - FromFile.getBuffer(), Tokens, Directives, CI.getLangOpts(), - &CI.getDiagnostics(), SM.getLocForStartOfFile(SM.getMainFileID()))) { + FromFile.getBuffer(), Tokens, Directives, &CI.getDiagnostics(), + SM.getLocForStartOfFile(SM.getMainFileID()))) { assert(CI.getDiagnostics().hasErrorOccurred() && "no errors reported for failure"); diff --git a/clang/lib/Lex/DependencyDirectivesScanner.cpp b/clang/lib/Lex/DependencyDirectivesScanner.cpp index fda54d314eef..0971daa1f366 100644 --- a/clang/lib/Lex/DependencyDirectivesScanner.cpp +++ b/clang/lib/Lex/DependencyDirectivesScanner.cpp @@ -62,17 +62,14 @@ struct DirectiveWithTokens { struct Scanner { Scanner(StringRef Input, SmallVectorImpl &Tokens, - DiagnosticsEngine *Diags, SourceLocation InputSourceLoc, - const LangOptions &LangOpts) + DiagnosticsEngine *Diags, SourceLocation InputSourceLoc) : Input(Input), Tokens(Tokens), Diags(Diags), - InputSourceLoc(InputSourceLoc), - LangOpts(getLangOptsForDepScanning(LangOpts)), - TheLexer(InputSourceLoc, this->LangOpts, Input.begin(), Input.begin(), + InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()), + TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(), Input.end()) {} - static LangOptions - getLangOptsForDepScanning(const LangOptions &invocationLangOpts) { - LangOptions LangOpts(invocationLangOpts); + static LangOptions getLangOptsForDepScanning() { + LangOptions LangOpts; // Set the lexer to use 'tok::at' for '@', instead of 'tok::unknown'. LangOpts.ObjC = true; LangOpts.LineComment = true; @@ -703,7 +700,7 @@ bool Scanner::lex_Pragma(const char *&First, const char *const End) { SmallVector DiscardTokens; const char *Begin = Buffer.c_str(); Scanner PragmaScanner{StringRef(Begin, Buffer.size()), DiscardTokens, Diags, - InputSourceLoc, LangOptions()}; + InputSourceLoc}; PragmaScanner.TheLexer.setParsingPreprocessorDirective(true); if (PragmaScanner.lexPragma(Begin, Buffer.end())) @@ -953,10 +950,9 @@ bool Scanner::scan(SmallVectorImpl &Directives) { bool clang::scanSourceForDependencyDirectives( StringRef Input, SmallVectorImpl &Tokens, - SmallVectorImpl &Directives, const LangOptions &LangOpts, - DiagnosticsEngine *Diags, SourceLocation InputSourceLoc) { - return Scanner(Input, Tokens, Diags, InputSourceLoc, LangOpts) - .scan(Directives); + SmallVectorImpl &Directives, DiagnosticsEngine *Diags, + SourceLocation InputSourceLoc) { + return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives); } void clang::printDependencyDirectivesAsSource( diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp index 66a2f6e0acb6..0cab17a34244 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -42,7 +42,7 @@ DependencyScanningWorkerFilesystem::readFile(StringRef Filename) { } bool DependencyScanningWorkerFilesystem::ensureDirectiveTokensArePopulated( - EntryRef Ref, const LangOptions &LangOpts) { + EntryRef Ref) { auto &Entry = Ref.Entry; if (Entry.isError() || Entry.isDirectory()) @@ -66,7 +66,7 @@ bool DependencyScanningWorkerFilesystem::ensureDirectiveTokensArePopulated( // dependencies. if (scanSourceForDependencyDirectives(Contents->Original->getBuffer(), Contents->DepDirectiveTokens, - Directives, LangOpts)) { + Directives)) { Contents->DepDirectiveTokens.clear(); // FIXME: Propagate the diagnostic if desired by the client. Contents->DepDirectives.store(new std::optional()); diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 07e1960dd905..0f82f22d8b9a 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -364,12 +364,11 @@ public: // Use the dependency scanning optimized file system if requested to do so. if (DepFS) ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile = - [LocalDepFS = DepFS, - &LangOpts = ScanInstance.getLangOpts()](FileEntryRef File) + [LocalDepFS = DepFS](FileEntryRef File) -> std::optional> { if (llvm::ErrorOr Entry = LocalDepFS->getOrCreateFileSystemEntry(File.getName())) - if (LocalDepFS->ensureDirectiveTokensArePopulated(*Entry, LangOpts)) + if (LocalDepFS->ensureDirectiveTokensArePopulated(*Entry)) return Entry->getDirectiveTokens(); return std::nullopt; }; diff --git a/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp index 044c3d65ec6f..59fef9ecbb9c 100644 --- a/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp +++ b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "clang/Lex/DependencyDirectivesScanner.h" -#include "clang/Basic/TokenKinds.h" #include "llvm/ADT/SmallString.h" #include "gtest/gtest.h" @@ -18,11 +17,11 @@ using namespace clang::dependency_directives_scan; static bool minimizeSourceToDependencyDirectives( StringRef Input, SmallVectorImpl &Out, SmallVectorImpl &Tokens, - SmallVectorImpl &Directives, const LangOptions &LangOpts) { + SmallVectorImpl &Directives) { Out.clear(); Tokens.clear(); Directives.clear(); - if (scanSourceForDependencyDirectives(Input, Tokens, Directives, LangOpts)) + if (scanSourceForDependencyDirectives(Input, Tokens, Directives)) return true; raw_svector_ostream OS(Out); @@ -39,9 +38,7 @@ static bool minimizeSourceToDependencyDirectives(StringRef Input, SmallVectorImpl &Out) { SmallVector Tokens; SmallVector Directives; - LangOptions LangOpts; - return minimizeSourceToDependencyDirectives(Input, Out, Tokens, Directives, - LangOpts); + return minimizeSourceToDependencyDirectives(Input, Out, Tokens, Directives); } namespace { @@ -50,17 +47,16 @@ TEST(MinimizeSourceToDependencyDirectivesTest, Empty) { SmallVector Out; SmallVector Tokens; SmallVector Directives; - LangOptions LangOpts; - ASSERT_FALSE(minimizeSourceToDependencyDirectives("", Out, Tokens, Directives, - LangOpts)); + ASSERT_FALSE( + minimizeSourceToDependencyDirectives("", Out, Tokens, Directives)); EXPECT_TRUE(Out.empty()); EXPECT_TRUE(Tokens.empty()); ASSERT_EQ(1u, Directives.size()); ASSERT_EQ(pp_eof, Directives.back().Kind); ASSERT_FALSE(minimizeSourceToDependencyDirectives("abc def\nxyz", Out, Tokens, - Directives, LangOpts)); + Directives)); EXPECT_STREQ("\n", Out.data()); EXPECT_TRUE(Tokens.empty()); ASSERT_EQ(2u, Directives.size()); @@ -72,7 +68,6 @@ TEST(MinimizeSourceToDependencyDirectivesTest, AllTokens) { SmallVector Out; SmallVector Tokens; SmallVector Directives; - LangOptions LangOpts; ASSERT_FALSE( minimizeSourceToDependencyDirectives("#define A\n" @@ -97,7 +92,7 @@ TEST(MinimizeSourceToDependencyDirectivesTest, AllTokens) { "export module m;\n" "import m;\n" "#pragma clang system_header\n", - Out, Tokens, Directives, LangOpts)); + Out, Tokens, Directives)); EXPECT_EQ(pp_define, Directives[0].Kind); EXPECT_EQ(pp_undef, Directives[1].Kind); EXPECT_EQ(pp_endif, Directives[2].Kind); @@ -150,10 +145,9 @@ TEST(MinimizeSourceToDependencyDirectivesTest, Define) { SmallVector Out; SmallVector Tokens; SmallVector Directives; - LangOptions LangOpts; - ASSERT_FALSE(minimizeSourceToDependencyDirectives( - "#define MACRO", Out, Tokens, Directives, LangOpts)); + ASSERT_FALSE(minimizeSourceToDependencyDirectives("#define MACRO", Out, + Tokens, Directives)); EXPECT_STREQ("#define MACRO\n", Out.data()); ASSERT_EQ(4u, Tokens.size()); ASSERT_EQ(2u, Directives.size()); @@ -844,7 +838,6 @@ TEST(MinimizeSourceToDependencyDirectivesTest, PragmaOnce) { SmallVector Out; SmallVector Tokens; SmallVector Directives; - LangOptions LangOpts; StringRef Source = R"(// comment #pragma once @@ -852,8 +845,8 @@ TEST(MinimizeSourceToDependencyDirectivesTest, PragmaOnce) { #include _Pragma("once") )"; - ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Tokens, - Directives, LangOpts)); + ASSERT_FALSE( + minimizeSourceToDependencyDirectives(Source, Out, Tokens, Directives)); EXPECT_STREQ("#pragma once\n#include \n_Pragma(\"once\")\n", Out.data()); ASSERT_EQ(Directives.size(), 4u); @@ -933,7 +926,6 @@ TEST(MinimizeSourceToDependencyDirectivesTest, CxxModules) { SmallVector Out; SmallVector Tokens; SmallVector Directives; - LangOptions LangOpts; StringRef Source = R"( module; @@ -962,8 +954,8 @@ ort \ import f(->a = 3); } )"; - ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Tokens, - Directives, LangOpts)); + ASSERT_FALSE( + minimizeSourceToDependencyDirectives(Source, Out, Tokens, Directives)); EXPECT_STREQ("#include \"textual-header.h\"\nexport module m;" "exp\\\nort import:l[[rename]];" "import<<=3;import a b d e d e f e;" @@ -1020,52 +1012,4 @@ TEST(MinimizeSourceToDependencyDirectivesTest, TokensBeforeEOF) { EXPECT_STREQ("#ifndef A\n#define A\n#endif\n\n", Out.data()); } -TEST(MinimizeSourceToDependencyDirectivesTest, CPlusPlus14PPNumber) { - SmallVector Out; - SmallVector Tokens; - SmallVector Directives; - LangOptions LangOpts; - - StringRef Source = R"( -#if 123'124 -#endif -)"; - - LangOpts.CPlusPlus14 = true; - ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Tokens, - Directives, LangOpts)); - EXPECT_STREQ("#if 123'124\n#endif\n", Out.data()); - ASSERT_EQ(Directives.size(), 3u); - EXPECT_EQ(Directives[0].Kind, dependency_directives_scan::pp_if); - EXPECT_EQ(Directives[1].Kind, dependency_directives_scan::pp_endif); - EXPECT_EQ(Directives[2].Kind, dependency_directives_scan::pp_eof); - ASSERT_EQ(Tokens.size(), 7u); - - ASSERT_TRUE(Tokens[0].is(tok::hash)); - ASSERT_TRUE(Tokens[1].is(tok::raw_identifier)); // "if" - ASSERT_TRUE(Tokens[2].is(tok::numeric_constant)); // 123'124 - ASSERT_TRUE(Tokens[3].is(tok::eod)); - ASSERT_TRUE(Tokens[4].is(tok::hash)); - ASSERT_TRUE(Tokens[5].is(tok::raw_identifier)); // #endif - ASSERT_TRUE(Tokens[6].is(tok::eod)); - - LangOpts.CPlusPlus14 = false; - ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Tokens, - Directives, LangOpts)); - EXPECT_STREQ("#if 123'124\n#endif\n", Out.data()); - ASSERT_EQ(Directives.size(), 3u); - EXPECT_EQ(Directives[0].Kind, dependency_directives_scan::pp_if); - EXPECT_EQ(Directives[1].Kind, dependency_directives_scan::pp_endif); - EXPECT_EQ(Directives[2].Kind, dependency_directives_scan::pp_eof); - ASSERT_EQ(Tokens.size(), 8u); - ASSERT_TRUE(Tokens[0].is(tok::hash)); - ASSERT_TRUE(Tokens[1].is(tok::raw_identifier)); // "if" - ASSERT_TRUE(Tokens[2].is(tok::numeric_constant)); // 123 - ASSERT_TRUE(Tokens[3].is(tok::unknown)); // '124 - ASSERT_TRUE(Tokens[4].is(tok::eod)); - ASSERT_TRUE(Tokens[5].is(tok::hash)); - ASSERT_TRUE(Tokens[6].is(tok::raw_identifier)); // #endif - ASSERT_TRUE(Tokens[7].is(tok::eod)); -} - } // end anonymous namespace diff --git a/clang/unittests/Lex/PPDependencyDirectivesTest.cpp b/clang/unittests/Lex/PPDependencyDirectivesTest.cpp index 410f378f1e89..6ff87f720a55 100644 --- a/clang/unittests/Lex/PPDependencyDirectivesTest.cpp +++ b/clang/unittests/Lex/PPDependencyDirectivesTest.cpp @@ -104,7 +104,6 @@ TEST_F(PPDependencyDirectivesTest, MacroGuard) { SmallVector Directives; }; SmallVector> DepDirectivesObjects; - LangOptions LangOpts; auto getDependencyDirectives = [&](FileEntryRef File) -> std::optional> { @@ -112,7 +111,7 @@ TEST_F(PPDependencyDirectivesTest, MacroGuard) { StringRef Input = (*FileMgr.getBufferForFile(File))->getBuffer(); bool Err = scanSourceForDependencyDirectives( Input, DepDirectivesObjects.back()->Tokens, - DepDirectivesObjects.back()->Directives, LangOpts); + DepDirectivesObjects.back()->Directives); EXPECT_FALSE(Err); return llvm::ArrayRef(DepDirectivesObjects.back()->Directives); }; -- GitLab From 4f40dfc2fdb6109702a4fcddff15d352eb216403 Mon Sep 17 00:00:00 2001 From: Joshua Cranmer Date: Wed, 5 Jun 2024 11:51:57 -0700 Subject: [PATCH 002/462] [IR] Remove the possibility of ConstantExpr having fast-math flags. (#94507) This possibility was added in https://reviews.llvm.org/D34303 to resolve some assertion failures with cases where FP math operations got constant-folded in constant expressions. However, at no point did the IR representation allow for expressing fast-math flags on constant expressions. With the change of https://github.com/llvm/llvm-project/pull/93038, there are no longer any constant expressions capable of being FP math operators, and thus FPMathOperator can go back to being Instruction-only. --- llvm/include/llvm/IR/Operator.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/llvm/include/llvm/IR/Operator.h b/llvm/include/llvm/IR/Operator.h index fda26891acfa..f63f54ef9410 100644 --- a/llvm/include/llvm/IR/Operator.h +++ b/llvm/include/llvm/IR/Operator.h @@ -330,8 +330,6 @@ public: unsigned Opcode; if (auto *I = dyn_cast(V)) Opcode = I->getOpcode(); - else if (auto *CE = dyn_cast(V)) - Opcode = CE->getOpcode(); else return false; -- GitLab From c8d63516ac9e15831975392dd9f6b9ae6a31b21c Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 5 Jun 2024 11:43:33 -0700 Subject: [PATCH 003/462] [SCEV] Add coverage for howFarToZero w/ non-constant strides Specifically, cases which require context sensative reasoning which the current code doesn't do. --- .../trip-count-unknown-stride.ll | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll index eb55e6abc95e..ecf13320a5e8 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll @@ -267,7 +267,228 @@ for.end: ; preds = %for.body, %entry ret void } +define void @ne_nsw_pos_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress { +; +; CHECK-LABEL: 'ne_nsw_pos_step' +; CHECK-NEXT: Determining loop execution counts for: @ne_nsw_pos_step +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. +; +entry: + %pos_step = icmp sgt i32 %s, 0 + call void @llvm.assume(i1 %pos_step) + %cmp4 = icmp sgt i32 %n, 0 + br i1 %cmp4, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05 + %0 = load i32, ptr %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, ptr %arrayidx, align 4 + %add = add nsw i32 %i.05, %s + %cmp = icmp ne i32 %add, %n + br i1 %cmp, label %for.body, label %for.end + +for.end: ; preds = %for.body, %entry + ret void +} + +define void @ne_nsw_neg_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress { +; +; CHECK-LABEL: 'ne_nsw_neg_step' +; CHECK-NEXT: Determining loop execution counts for: @ne_nsw_neg_step +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. +; +entry: + %neg_step = icmp slt i32 %s, 0 + call void @llvm.assume(i1 %neg_step) + %cmp4 = icmp sgt i32 %n, 0 + br i1 %cmp4, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05 + %0 = load i32, ptr %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, ptr %arrayidx, align 4 + %add = add nsw i32 %i.05, %s + %cmp = icmp ne i32 %add, %n + br i1 %cmp, label %for.body, label %for.end + +for.end: ; preds = %for.body, %entry + ret void +} + +define void @ne_nsw_nonneg_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress { +; +; CHECK-LABEL: 'ne_nsw_nonneg_step' +; CHECK-NEXT: Determining loop execution counts for: @ne_nsw_nonneg_step +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. +; +entry: + %nonneg_step = icmp sge i32 %s, 0 + call void @llvm.assume(i1 %nonneg_step) + %cmp4 = icmp sgt i32 %n, 0 + br i1 %cmp4, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05 + %0 = load i32, ptr %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, ptr %arrayidx, align 4 + %add = add nsw i32 %i.05, %s + %cmp = icmp ne i32 %add, %n + br i1 %cmp, label %for.body, label %for.end + +for.end: ; preds = %for.body, %entry + ret void +} + +define void @ne_nsw_unknown_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress { +; +; CHECK-LABEL: 'ne_nsw_unknown_step' +; CHECK-NEXT: Determining loop execution counts for: @ne_nsw_unknown_step +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. +; +entry: + %cmp4 = icmp sgt i32 %n, 0 + br i1 %cmp4, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05 + %0 = load i32, ptr %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, ptr %arrayidx, align 4 + %add = add nsw i32 %i.05, %s + %cmp = icmp ne i32 %add, %n + br i1 %cmp, label %for.body, label %for.end + +for.end: ; preds = %for.body, %entry + ret void +} + +define void @ne_nuw_pos_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress { +; +; CHECK-LABEL: 'ne_nuw_pos_step' +; CHECK-NEXT: Determining loop execution counts for: @ne_nuw_pos_step +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. +; +entry: + %pos_step = icmp sgt i32 %s, 0 + call void @llvm.assume(i1 %pos_step) + %cmp4 = icmp sgt i32 %n, 0 + br i1 %cmp4, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05 + %0 = load i32, ptr %arrayidx, align 4 + %inc = add nuw i32 %0, 1 + store i32 %inc, ptr %arrayidx, align 4 + %add = add nuw i32 %i.05, %s + %cmp = icmp ne i32 %add, %n + br i1 %cmp, label %for.body, label %for.end + +for.end: ; preds = %for.body, %entry + ret void +} + +define void @ne_nuw_neg_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress { +; +; CHECK-LABEL: 'ne_nuw_neg_step' +; CHECK-NEXT: Determining loop execution counts for: @ne_nuw_neg_step +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. +; +entry: + %neg_step = icmp slt i32 %s, 0 + call void @llvm.assume(i1 %neg_step) + %cmp4 = icmp sgt i32 %n, 0 + br i1 %cmp4, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05 + %0 = load i32, ptr %arrayidx, align 4 + %inc = add nuw i32 %0, 1 + store i32 %inc, ptr %arrayidx, align 4 + %add = add nuw i32 %i.05, %s + %cmp = icmp ne i32 %add, %n + br i1 %cmp, label %for.body, label %for.end + +for.end: ; preds = %for.body, %entry + ret void +} + +define void @ne_nuw_nonneg_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress { +; +; CHECK-LABEL: 'ne_nuw_nonneg_step' +; CHECK-NEXT: Determining loop execution counts for: @ne_nuw_nonneg_step +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. +; +entry: + %nonneg_step = icmp sge i32 %s, 0 + call void @llvm.assume(i1 %nonneg_step) + %cmp4 = icmp sgt i32 %n, 0 + br i1 %cmp4, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05 + %0 = load i32, ptr %arrayidx, align 4 + %inc = add nuw i32 %0, 1 + store i32 %inc, ptr %arrayidx, align 4 + %add = add nuw i32 %i.05, %s + %cmp = icmp ne i32 %add, %n + br i1 %cmp, label %for.body, label %for.end + +for.end: ; preds = %for.body, %entry + ret void +} + +define void @ne_nuw_unknown_step(ptr nocapture %A, i32 %n, i32 %s) mustprogress { +; +; CHECK-LABEL: 'ne_nuw_unknown_step' +; CHECK-NEXT: Determining loop execution counts for: @ne_nuw_unknown_step +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. +; +entry: + %cmp4 = icmp sgt i32 %n, 0 + br i1 %cmp4, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %i.05 = phi i32 [ %add, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05 + %0 = load i32, ptr %arrayidx, align 4 + %inc = add nuw i32 %0, 1 + store i32 %inc, ptr %arrayidx, align 4 + %add = add nuw i32 %i.05, %s + %cmp = icmp ne i32 %add, %n + br i1 %cmp, label %for.body, label %for.end + +for.end: ; preds = %for.body, %entry + ret void +} + +declare void @llvm.assume(i1) !8 = distinct !{!8, !9} !9 = !{!"llvm.loop.mustprogress"} -- GitLab From 3d44926712389945468a566c9058b2e715f8f2ad Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 5 Jun 2024 12:00:00 -0700 Subject: [PATCH 004/462] [bazel] Sync config with main (#94521) --- .../libc/test/src/math/libc_math_test_rules.bzl | 3 ++- .../llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel | 1 + .../llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl b/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl index 11d2e5aa340f..9ec3a5e5ad57 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl +++ b/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl @@ -25,6 +25,7 @@ def math_test(name, hdrs = [], deps = [], **kwargs): srcs = [test_name + ".cpp"] + hdrs, libc_function_deps = ["//libc:func_name".replace("func_name", name)], deps = [ + "//libc/test/UnitTest:fp_test_helpers", "//libc:__support_cpp_algorithm", "//libc:__support_fputil_basic_operations", "//libc:__support_fputil_fenv_impl", @@ -32,9 +33,9 @@ def math_test(name, hdrs = [], deps = [], **kwargs): "//libc:__support_fputil_manipulation_functions", "//libc:__support_fputil_nearest_integer_operations", "//libc:__support_fputil_normal_float", + "//libc:__support_macros_properties_architectures", "//libc:__support_math_extras", "//libc:__support_uint128", - "//libc/test/UnitTest:fp_test_helpers", "//libc:hdr_math_macros", ] + deps, **kwargs diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel index 7d4b9978db3f..eccea8faeebc 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel @@ -117,6 +117,7 @@ math_test( name = "llrintf128", hdrs = ["RoundToIntegerTest.h"], ) + math_test( name = "lroundf128", hdrs = ["RoundToIntegerTest.h"], diff --git a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel index 53a8c9b9476f..6dd1fc4679d4 100644 --- a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel @@ -46,6 +46,7 @@ libc_support_library( "//libc:__support_cpp_type_traits", "//libc:__support_fputil_fp_bits", "//libc:__support_fputil_fpbits_str", + "//libc:__support_macros_properties_types", "//libc:hdr_math_macros", "//libc/test/UnitTest:LibcUnitTest", "//libc/test/UnitTest:fp_test_helpers", -- GitLab From 1d8743359360d3d960d599e03e62537dc108aae2 Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Wed, 5 Jun 2024 15:01:33 -0400 Subject: [PATCH 005/462] [x86] Add tan intrinsic part 4 (#90503) This change is an implementation of #87367's investigation on supporting IEEE math operations as intrinsics. Which was discussed in this RFC: https://discourse.llvm.org/t/rfc-all-the-math-intrinsics/78294 Much of this change was following how G_FSIN and G_FCOS were used. Changes: - `llvm/docs/GlobalISel/GenericOpcode.rst` - Document the `G_FTAN` opcode - `llvm/docs/LangRef.rst` - Document the tan intrinsic - `llvm/include/llvm/Analysis/VecFuncs.def` - Associate the tan intrinsic as a vector function similar to the tanf libcall. - `llvm/include/llvm/CodeGen/BasicTTIImpl.h` - Map the tan intrinsic to `ISD::FTAN` - `llvm/include/llvm/CodeGen/ISDOpcodes.h` - Define ISD opcodes for `FTAN` and `STRICT_FTAN` - `llvm/include/llvm/IR/Intrinsics.td` - Create the tan intrinsic - `llvm/include/llvm/IR/RuntimeLibcalls.def` - Define tan libcall mappings - `llvm/include/llvm/Target/GenericOpcodes.td` - Define the `G_FTAN` Opcode - `llvm/include/llvm/Support/TargetOpcodes.def` - Create a `G_FTAN` Opcode handler - `llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td` - Map `G_FTAN` to `ftan` - `llvm/include/llvm/Target/TargetSelectionDAG.td` - Define `ftan`, `strict_ftan`, and `any_ftan` and map them to the ISD opcodes for `FTAN` and `STRICT_FTAN` - `llvm/lib/Analysis/VectorUtils.cpp` - Associate the tan intrinsic as a vector intrinsic - `llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp` Map the tan intrinsic to `G_FTAN` Opcode - `llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp` - Add `G_FTAN` to the list of floating point math operations also associate `G_FTAN` with the `TAN_F` runtime lib. - `llvm/lib/CodeGen/GlobalISel/Utils.cpp` - More floating point math operation common behaviors. - llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp - List the function expansion operations for `FTAN` and `STRICT_FTAN`. Also define both opcodes in `PromoteNode`. - `llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp` - More `FTAN` and `STRICT_FTAN` handling in the legalizer - `llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h` - Define `SoftenFloatRes_FTAN` and `ExpandFloatRes_FTAN`. - `llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp` - Define `FTAN` as a legal vector operation. - `llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp` - Define `FTAN` as a legal vector operation. - `llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp` - define tan as an intrinsic that doesn't return NaN. - `llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp` Map `LibFunc_tan`, `LibFunc_tanf`, and `LibFunc_tanl` to `ISD::FTAN`. Map `Intrinsic::tan` to `ISD::FTAN` and add selection dag handling for `Intrinsic::tan`. - `llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp` - Define `ftan` and `strict_ftan` names for the equivalent ISD opcodes. - `llvm/lib/CodeGen/TargetLoweringBase.cpp` -Define a Tan128 libcall and ISD::FTAN as a target lowering action. - `llvm/lib/Target/X86/X86ISelLowering.cpp` - Add x86_64 lowering for tan intrinsic resolves https://github.com/llvm/llvm-project/issues/70082 --- llvm/include/llvm/Analysis/VecFuncs.def | 36 ++++ llvm/include/llvm/CodeGen/BasicTTIImpl.h | 3 + llvm/include/llvm/CodeGen/ISDOpcodes.h | 2 + llvm/include/llvm/IR/RuntimeLibcalls.def | 5 + .../Target/GlobalISel/SelectionDAGCompat.td | 1 + .../include/llvm/Target/TargetSelectionDAG.td | 6 + llvm/lib/Analysis/VectorUtils.cpp | 1 + .../CodeGen/GlobalISel/LegalizerHelper.cpp | 5 + llvm/lib/CodeGen/GlobalISel/Utils.cpp | 2 + llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 7 + .../SelectionDAG/LegalizeFloatTypes.cpp | 24 ++- llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 4 + .../SelectionDAG/LegalizeVectorOps.cpp | 1 + .../SelectionDAG/LegalizeVectorTypes.cpp | 3 + .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 1 + .../SelectionDAG/SelectionDAGBuilder.cpp | 8 + .../SelectionDAG/SelectionDAGDumper.cpp | 2 + llvm/lib/CodeGen/TargetLoweringBase.cpp | 4 +- llvm/lib/Target/X86/X86ISelLowering.cpp | 18 +- llvm/test/CodeGen/X86/llvm.tan.ll | 70 ++++++ llvm/test/CodeGen/X86/vec-libcalls.ll | 202 ++++++++++++++++++ .../LoopVectorize/X86/amdlibm-calls.ll | 113 ++++++++++ .../X86/libm-vector-calls-VF2-VF8.ll | 115 ++++++++++ .../LoopVectorize/X86/svml-calls.ll | 46 ++++ .../LoopVectorize/X86/veclib-calls.ll | 25 +++ 25 files changed, 700 insertions(+), 4 deletions(-) create mode 100644 llvm/test/CodeGen/X86/llvm.tan.ll diff --git a/llvm/include/llvm/Analysis/VecFuncs.def b/llvm/include/llvm/Analysis/VecFuncs.def index e12eb7095b90..402189769c20 100644 --- a/llvm/include/llvm/Analysis/VecFuncs.def +++ b/llvm/include/llvm/Analysis/VecFuncs.def @@ -49,6 +49,7 @@ TLI_DEFINE_VECFUNC("llvm.sin.f32", "vsinf", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("cosf", "vcosf", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("llvm.cos.f32", "vcosf", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("tanf", "vtanf", FIXED(4), "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("llvm.tan.f32", "vtanf", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("asinf", "vasinf", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("acosf", "vacosf", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("atanf", "vatanf", FIXED(4), "_ZGV_LLVM_N4v") @@ -142,6 +143,18 @@ TLI_DEFINE_VECFUNC("llvm.cos.f64", "_ZGVdN4v_cos", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("llvm.cos.f32", "_ZGVbN4v_cosf", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("llvm.cos.f32", "_ZGVdN8v_cosf", FIXED(8), "_ZGV_LLVM_N8v") +TLI_DEFINE_VECFUNC("tan", "_ZGVbN2v_tan", FIXED(2), "_ZGV_LLVM_N2v") +TLI_DEFINE_VECFUNC("tan", "_ZGVdN4v_tan", FIXED(4), "_ZGV_LLVM_N4v") + +TLI_DEFINE_VECFUNC("tanf", "_ZGVbN4v_tanf", FIXED(4), "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("tanf", "_ZGVdN8v_tanf", FIXED(8), "_ZGV_LLVM_N8v") + +TLI_DEFINE_VECFUNC("llvm.tan.f64", "_ZGVbN2v_tan", FIXED(2), "_ZGV_LLVM_N2v") +TLI_DEFINE_VECFUNC("llvm.tan.f64", "_ZGVdN4v_tan", FIXED(4), "_ZGV_LLVM_N4v") + +TLI_DEFINE_VECFUNC("llvm.tan.f32", "_ZGVbN4v_tanf", FIXED(4), "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("llvm.tan.f32", "_ZGVdN8v_tanf", FIXED(8), "_ZGV_LLVM_N8v") + TLI_DEFINE_VECFUNC("pow", "_ZGVbN2vv_pow", FIXED(2), "_ZGV_LLVM_N2vv") TLI_DEFINE_VECFUNC("pow", "_ZGVdN4vv_pow", FIXED(4), "_ZGV_LLVM_N4vv") @@ -303,6 +316,22 @@ TLI_DEFINE_VECFUNC("llvm.cos.f32", "__svml_cosf4", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("llvm.cos.f32", "__svml_cosf8", FIXED(8), "_ZGV_LLVM_N8v") TLI_DEFINE_VECFUNC("llvm.cos.f32", "__svml_cosf16", FIXED(16), "_ZGV_LLVM_N16v") +TLI_DEFINE_VECFUNC("tan", "__svml_tan2", FIXED(2), "_ZGV_LLVM_N2v") +TLI_DEFINE_VECFUNC("tan", "__svml_tan4", FIXED(4), "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("tan", "__svml_tan8", FIXED(8), "_ZGV_LLVM_N8v") + +TLI_DEFINE_VECFUNC("tanf", "__svml_tanf4", FIXED(4), "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("tanf", "__svml_tanf8", FIXED(8), "_ZGV_LLVM_N8v") +TLI_DEFINE_VECFUNC("tanf", "__svml_tanf16", FIXED(16), "_ZGV_LLVM_N16v") + +TLI_DEFINE_VECFUNC("llvm.tan.f64", "__svml_tan2", FIXED(2), "_ZGV_LLVM_N2v") +TLI_DEFINE_VECFUNC("llvm.tan.f64", "__svml_tan4", FIXED(4), "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("llvm.tan.f64", "__svml_tan8", FIXED(8), "_ZGV_LLVM_N8v") + +TLI_DEFINE_VECFUNC("llvm.tan.f32", "__svml_tanf4", FIXED(4), "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("llvm.tan.f32", "__svml_tanf8", FIXED(8), "_ZGV_LLVM_N8v") +TLI_DEFINE_VECFUNC("llvm.tan.f32", "__svml_tanf16", FIXED(16), "_ZGV_LLVM_N16v") + TLI_DEFINE_VECFUNC("pow", "__svml_pow2", FIXED(2), "_ZGV_LLVM_N2vv") TLI_DEFINE_VECFUNC("pow", "__svml_pow4", FIXED(4), "_ZGV_LLVM_N4vv") TLI_DEFINE_VECFUNC("pow", "__svml_pow8", FIXED(8), "_ZGV_LLVM_N8vv") @@ -1237,6 +1266,13 @@ TLI_DEFINE_VECFUNC("tanf", "amd_vrs4_tanf", FIXED(4), NOMASK, "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("tanf", "amd_vrs8_tanf", FIXED(8), NOMASK, "_ZGV_LLVM_N8v") TLI_DEFINE_VECFUNC("tanf", "amd_vrs16_tanf", FIXED(16), NOMASK, "_ZGV_LLVM_N16v") +TLI_DEFINE_VECFUNC("llvm.tan.f32", "amd_vrs16_tanf", FIXED(16), NOMASK, "_ZGV_LLVM_N16v") +TLI_DEFINE_VECFUNC("llvm.tan.f32", "amd_vrs8_tanf", FIXED(8), NOMASK, "_ZGV_LLVM_N8v") +TLI_DEFINE_VECFUNC("llvm.tan.f32", "amd_vrs4_tanf", FIXED(4), NOMASK, "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("llvm.tan.f64", "amd_vrd8_tan", FIXED(8), NOMASK, "_ZGV_LLVM_N8v") +TLI_DEFINE_VECFUNC("llvm.tan.f64", "amd_vrd4_tan", FIXED(4), NOMASK, "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("llvm.tan.f64", "amd_vrd2_tan", FIXED(2), NOMASK, "_ZGV_LLVM_N2v") + TLI_DEFINE_VECFUNC("asin", "amd_vrd8_asin", FIXED(8), NOMASK, "_ZGV_LLVM_N8v") TLI_DEFINE_VECFUNC("asinf", "amd_vrs4_asinf", FIXED(4), NOMASK, "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("asinf", "amd_vrs8_asinf", FIXED(8), NOMASK, "_ZGV_LLVM_N8v") diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index ef4e0fd29768..9f8d3ded9b3c 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -1977,6 +1977,9 @@ public: case Intrinsic::cos: ISD = ISD::FCOS; break; + case Intrinsic::tan: + ISD = ISD::FTAN; + break; case Intrinsic::exp: ISD = ISD::FEXP; break; diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index 0f87e062e2da..c8c86ed5eef2 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -415,6 +415,7 @@ enum NodeType { STRICT_FLDEXP, STRICT_FSIN, STRICT_FCOS, + STRICT_FTAN, STRICT_FEXP, STRICT_FEXP2, STRICT_FLOG, @@ -934,6 +935,7 @@ enum NodeType { FCBRT, FSIN, FCOS, + FTAN, FPOW, FPOWI, /// FLDEXP - ldexp, inspired by libm (op0 * 2**op1). diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def index a5a72884a6bf..e900bcdbcdd0 100644 --- a/llvm/include/llvm/IR/RuntimeLibcalls.def +++ b/llvm/include/llvm/IR/RuntimeLibcalls.def @@ -197,6 +197,11 @@ HANDLE_LIBCALL(COS_F64, "cos") HANDLE_LIBCALL(COS_F80, "cosl") HANDLE_LIBCALL(COS_F128, "cosl") HANDLE_LIBCALL(COS_PPCF128, "cosl") +HANDLE_LIBCALL(TAN_F32, "tanf") +HANDLE_LIBCALL(TAN_F64, "tan") +HANDLE_LIBCALL(TAN_F80, "tanl") +HANDLE_LIBCALL(TAN_F128,"tanl") +HANDLE_LIBCALL(TAN_PPCF128, "tanl") HANDLE_LIBCALL(SINCOS_F32, nullptr) HANDLE_LIBCALL(SINCOS_F64, nullptr) HANDLE_LIBCALL(SINCOS_F80, nullptr) diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index 8fa0e4b86d6d..560d3b434d07 100644 --- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -148,6 +148,7 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index 1c95a6090984..15e02eb49271 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -509,6 +509,7 @@ def fneg : SDNode<"ISD::FNEG" , SDTFPUnaryOp>; def fsqrt : SDNode<"ISD::FSQRT" , SDTFPUnaryOp>; def fsin : SDNode<"ISD::FSIN" , SDTFPUnaryOp>; def fcos : SDNode<"ISD::FCOS" , SDTFPUnaryOp>; +def ftan : SDNode<"ISD::FTAN" , SDTFPUnaryOp>; def fexp2 : SDNode<"ISD::FEXP2" , SDTFPUnaryOp>; def fexp10 : SDNode<"ISD::FEXP10" , SDTFPUnaryOp>; def fpow : SDNode<"ISD::FPOW" , SDTFPBinOp>; @@ -562,6 +563,8 @@ def strict_fsin : SDNode<"ISD::STRICT_FSIN", SDTFPUnaryOp, [SDNPHasChain]>; def strict_fcos : SDNode<"ISD::STRICT_FCOS", SDTFPUnaryOp, [SDNPHasChain]>; +def strict_ftan : SDNode<"ISD::STRICT_FTAN", + SDTFPUnaryOp, [SDNPHasChain]>; def strict_fexp2 : SDNode<"ISD::STRICT_FEXP2", SDTFPUnaryOp, [SDNPHasChain]>; def strict_fpow : SDNode<"ISD::STRICT_FPOW", @@ -1517,6 +1520,9 @@ def any_fsin : PatFrags<(ops node:$src), def any_fcos : PatFrags<(ops node:$src), [(strict_fcos node:$src), (fcos node:$src)]>; +def any_ftan : PatFrags<(ops node:$src), + [(strict_ftan node:$src), + (ftan node:$src)]>; def any_fexp2 : PatFrags<(ops node:$src), [(strict_fexp2 node:$src), (fexp2 node:$src)]>; diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp index 917094267d05..30728ed58750 100644 --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -68,6 +68,7 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) { case Intrinsic::sqrt: // Begin floating-point. case Intrinsic::sin: case Intrinsic::cos: + case Intrinsic::tan: case Intrinsic::exp: case Intrinsic::exp2: case Intrinsic::log: diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 6f0cae2edab1..9830b521797c 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -449,6 +449,8 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) { RTLIBCASE(SIN_F); case TargetOpcode::G_FCOS: RTLIBCASE(COS_F); + case TargetOpcode::G_FTAN: + RTLIBCASE(TAN_F); case TargetOpcode::G_FLOG10: RTLIBCASE(LOG10_F); case TargetOpcode::G_FLOG: @@ -1037,6 +1039,7 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) { case TargetOpcode::G_FREM: case TargetOpcode::G_FCOS: case TargetOpcode::G_FSIN: + case TargetOpcode::G_FTAN: case TargetOpcode::G_FLOG10: case TargetOpcode::G_FLOG: case TargetOpcode::G_FLOG2: @@ -2893,6 +2896,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { case TargetOpcode::G_FFLOOR: case TargetOpcode::G_FCOS: case TargetOpcode::G_FSIN: + case TargetOpcode::G_FTAN: case TargetOpcode::G_FLOG10: case TargetOpcode::G_FLOG: case TargetOpcode::G_FLOG2: @@ -4659,6 +4663,7 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, case G_INTRINSIC_TRUNC: case G_FCOS: case G_FSIN: + case G_FTAN: case G_FSQRT: case G_BSWAP: case G_BITREVERSE: diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp index e8438be94b3c..129e6963aef3 100644 --- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp +++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp @@ -833,6 +833,7 @@ bool llvm::isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI, case TargetOpcode::G_FREM: case TargetOpcode::G_FSIN: case TargetOpcode::G_FCOS: + case TargetOpcode::G_FTAN: case TargetOpcode::G_FMA: case TargetOpcode::G_FMAD: if (SNaN) @@ -1713,6 +1714,7 @@ bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) { case TargetOpcode::G_FREM: case TargetOpcode::G_FRINT: case TargetOpcode::G_FSIN: + case TargetOpcode::G_FTAN: case TargetOpcode::G_FSQRT: case TargetOpcode::G_FSUB: case TargetOpcode::G_INTRINSIC_ROUND: diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 8cd2bb60d81f..27c45cab2e0d 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -4514,6 +4514,11 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { RTLIB::COS_F80, RTLIB::COS_F128, RTLIB::COS_PPCF128, Results); break; + case ISD::FTAN: + case ISD::STRICT_FTAN: + ExpandFPLibCall(Node, RTLIB::TAN_F32, RTLIB::TAN_F64, RTLIB::TAN_F80, + RTLIB::TAN_F128, RTLIB::TAN_PPCF128, Results); + break; case ISD::FSINCOS: // Expand into sincos libcall. ExpandSinCosLibCall(Node, Results); @@ -5468,6 +5473,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { case ISD::FSQRT: case ISD::FSIN: case ISD::FCOS: + case ISD::FTAN: case ISD::FLOG: case ISD::FLOG2: case ISD::FLOG10: @@ -5492,6 +5498,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { case ISD::STRICT_FSQRT: case ISD::STRICT_FSIN: case ISD::STRICT_FCOS: + case ISD::STRICT_FTAN: case ISD::STRICT_FLOG: case ISD::STRICT_FLOG2: case ISD::STRICT_FLOG10: diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index fb1424f75e09..aa116c9de5d8 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -131,6 +131,8 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) { case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break; case ISD::STRICT_FSUB: case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break; + case ISD::STRICT_FTAN: + case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break; case ISD::STRICT_FTRUNC: case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break; case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break; @@ -774,6 +776,12 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) { RTLIB::SUB_PPCF128)); } +SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) { + return SoftenFloatRes_Unary( + N, GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32, RTLIB::TAN_F64, + RTLIB::TAN_F80, RTLIB::TAN_F128, RTLIB::TAN_PPCF128)); +} + SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) { return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::TRUNC_F32, @@ -1330,7 +1338,7 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { #endif report_fatal_error("Do not know how to expand the result of this " "operator!"); - + // clang-format off case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; @@ -1399,6 +1407,8 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break; case ISD::STRICT_FSUB: case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break; + case ISD::STRICT_FTAN: + case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break; case ISD::STRICT_FTRUNC: case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break; case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break; @@ -1408,6 +1418,7 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break; case ISD::STRICT_FREM: case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break; + // clang-format on } // If Lo/Hi is null, the sub-method took care of registering results etc. @@ -1768,6 +1779,15 @@ void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo, RTLIB::SUB_PPCF128), Lo, Hi); } +void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo, + SDValue &Hi) { + ExpandFloatRes_Unary(N, + GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32, + RTLIB::TAN_F64, RTLIB::TAN_F80, + RTLIB::TAN_F128, RTLIB::TAN_PPCF128), + Lo, Hi); +} + void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N, SDValue &Lo, SDValue &Hi) { ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), @@ -2479,6 +2499,7 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { case ISD::FSIN: case ISD::FSQRT: case ISD::FTRUNC: + case ISD::FTAN: case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break; // Binary FP Operations @@ -2914,6 +2935,7 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { case ISD::FSIN: case ISD::FSQRT: case ISD::FTRUNC: + case ISD::FTAN: case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break; // Binary FP Operations diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index bec9cb49b586..2350b562a034 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -586,6 +586,7 @@ private: SDValue SoftenFloatRes_FSIN(SDNode *N); SDValue SoftenFloatRes_FSQRT(SDNode *N); SDValue SoftenFloatRes_FSUB(SDNode *N); + SDValue SoftenFloatRes_FTAN(SDNode *N); SDValue SoftenFloatRes_FTRUNC(SDNode *N); SDValue SoftenFloatRes_LOAD(SDNode *N); SDValue SoftenFloatRes_ATOMIC_LOAD(SDNode *N); @@ -635,6 +636,7 @@ private: SDValue &Lo, SDValue &Hi); void ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC, SDValue &Lo, SDValue &Hi); + // clang-format off void ExpandFloatRes_FABS (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_FMINNUM (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_FMAXNUM (SDNode *N, SDValue &Lo, SDValue &Hi); @@ -667,9 +669,11 @@ private: void ExpandFloatRes_FSIN (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_FSQRT (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_FSUB (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandFloatRes_FTAN (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_FTRUNC (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, SDValue &Hi); + // clang-format on // Float Operand Expansion. bool ExpandFloatOperand(SDNode *N, unsigned OpNo); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index 6acbc044d673..5c5347b7f314 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -397,6 +397,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { case ISD::FSQRT: case ISD::FSIN: case ISD::FCOS: + case ISD::FTAN: case ISD::FLDEXP: case ISD::FPOWI: case ISD::FPOW: diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 361416edb554..92ce3b17ed6c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -108,6 +108,7 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { case ISD::FROUNDEVEN: case ISD::FSIN: case ISD::FSQRT: + case ISD::FTAN: case ISD::FTRUNC: case ISD::SIGN_EXTEND: case ISD::SINT_TO_FP: @@ -1140,6 +1141,7 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { case ISD::VP_FROUNDEVEN: case ISD::FSIN: case ISD::FSQRT: case ISD::VP_SQRT: + case ISD::FTAN: case ISD::FTRUNC: case ISD::VP_FROUNDTOZERO: case ISD::SINT_TO_FP: @@ -4400,6 +4402,7 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { case ISD::FROUNDEVEN: case ISD::FSIN: case ISD::FSQRT: + case ISD::FTAN: case ISD::FTRUNC: if (unrollExpandedOp()) break; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 4a6a431696b5..09cdec8adb27 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -5375,6 +5375,7 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const case ISD::FREM: case ISD::FSIN: case ISD::FCOS: + case ISD::FTAN: case ISD::FMA: case ISD::FMAD: { if (SNaN) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index ba76456b5836..64d6fd458c62 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -6742,6 +6742,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, case Intrinsic::fabs: case Intrinsic::sin: case Intrinsic::cos: + case Intrinsic::tan: case Intrinsic::exp10: case Intrinsic::floor: case Intrinsic::ceil: @@ -6759,6 +6760,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, case Intrinsic::fabs: Opcode = ISD::FABS; break; case Intrinsic::sin: Opcode = ISD::FSIN; break; case Intrinsic::cos: Opcode = ISD::FCOS; break; + case Intrinsic::tan: Opcode = ISD::FTAN; break; case Intrinsic::exp10: Opcode = ISD::FEXP10; break; case Intrinsic::floor: Opcode = ISD::FFLOOR; break; case Intrinsic::ceil: Opcode = ISD::FCEIL; break; @@ -9160,6 +9162,12 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { if (visitUnaryFloatCall(I, ISD::FCOS)) return; break; + case LibFunc_tan: + case LibFunc_tanf: + case LibFunc_tanl: + if (visitUnaryFloatCall(I, ISD::FTAN)) + return; + break; case LibFunc_sqrt: case LibFunc_sqrtf: case LibFunc_sqrtl: diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index 2198c2354483..52da24b59451 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -210,6 +210,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::FCOS: return "fcos"; case ISD::STRICT_FCOS: return "strict_fcos"; case ISD::FSINCOS: return "fsincos"; + case ISD::FTAN: return "ftan"; + case ISD::STRICT_FTAN: return "strict_ftan"; case ISD::FTRUNC: return "ftrunc"; case ISD::STRICT_FTRUNC: return "strict_ftrunc"; case ISD::FFLOOR: return "ffloor"; diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp index 3aec7049e0cc..8240a1fd7e2f 100644 --- a/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -141,6 +141,7 @@ void TargetLoweringBase::InitLibcalls(const Triple &TT) { setLibcallName(RTLIB::EXP10_F128, "exp10f128"); setLibcallName(RTLIB::SIN_F128, "sinf128"); setLibcallName(RTLIB::COS_F128, "cosf128"); + setLibcallName(RTLIB::TAN_F128, "tanf128"); setLibcallName(RTLIB::SINCOS_F128, "sincosf128"); setLibcallName(RTLIB::POW_F128, "powf128"); setLibcallName(RTLIB::POW_FINITE_F128, "__powf128_finite"); @@ -1015,7 +1016,8 @@ void TargetLoweringBase::initActions() { setOperationAction({ISD::FCBRT, ISD::FLOG, ISD::FLOG2, ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10, ISD::FFLOOR, ISD::FNEARBYINT, ISD::FCEIL, ISD::FRINT, ISD::FTRUNC, ISD::LROUND, - ISD::LLROUND, ISD::LRINT, ISD::LLRINT, ISD::FROUNDEVEN}, + ISD::LLROUND, ISD::LRINT, ISD::LLRINT, ISD::FROUNDEVEN, + ISD::FTAN}, {MVT::f32, MVT::f64, MVT::f128}, Expand); // Default ISD::TRAP to expand (which turns it into abort). diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 7d30de15f84d..af1e45d25aac 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -615,6 +615,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::FSIN, VT, Action); setOperationAction(ISD::FCOS, VT, Action); setOperationAction(ISD::FSINCOS, VT, Action); + setOperationAction(ISD::FTAN, VT, Action); setOperationAction(ISD::FSQRT, VT, Action); setOperationAction(ISD::FPOW, VT, Action); setOperationAction(ISD::FLOG, VT, Action); @@ -833,9 +834,12 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, } // Always expand sin/cos functions even though x87 has an instruction. + // clang-format off setOperationAction(ISD::FSIN , MVT::f80, Expand); setOperationAction(ISD::FCOS , MVT::f80, Expand); setOperationAction(ISD::FSINCOS, MVT::f80, Expand); + setOperationAction(ISD::FTAN , MVT::f80, Expand); + // clang-format on setOperationAction(ISD::FFLOOR, MVT::f80, Expand); setOperationAction(ISD::FCEIL, MVT::f80, Expand); @@ -888,11 +892,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::FNEG, MVT::f128, Custom); setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom); + // clang-format off setOperationAction(ISD::FSIN, MVT::f128, LibCall); setOperationAction(ISD::STRICT_FSIN, MVT::f128, LibCall); setOperationAction(ISD::FCOS, MVT::f128, LibCall); setOperationAction(ISD::STRICT_FCOS, MVT::f128, LibCall); setOperationAction(ISD::FSINCOS, MVT::f128, LibCall); + setOperationAction(ISD::FTAN, MVT::f128, LibCall); + setOperationAction(ISD::STRICT_FTAN, MVT::f128, LibCall); + // clang-format on // No STRICT_FSINCOS setOperationAction(ISD::FSQRT, MVT::f128, LibCall); setOperationAction(ISD::STRICT_FSQRT, MVT::f128, LibCall); @@ -944,9 +952,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16, MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v2f64, MVT::v4f64, MVT::v8f64 }) { + // clang-format off setOperationAction(ISD::FSIN, VT, Expand); setOperationAction(ISD::FSINCOS, VT, Expand); setOperationAction(ISD::FCOS, VT, Expand); + setOperationAction(ISD::FTAN, VT, Expand); setOperationAction(ISD::FREM, VT, Expand); setOperationAction(ISD::FCOPYSIGN, VT, Expand); setOperationAction(ISD::FPOW, VT, Expand); @@ -956,6 +966,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::FEXP, VT, Expand); setOperationAction(ISD::FEXP2, VT, Expand); setOperationAction(ISD::FEXP10, VT, Expand); + // clang-format on } // First set operation action for all vector types to either promote @@ -2473,7 +2484,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, // function casting to f64 and calling `fmod`. if (Subtarget.is32Bit() && (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium())) - for (ISD::NodeType Op : + // clang-format off + for (ISD::NodeType Op : {ISD::FCEIL, ISD::STRICT_FCEIL, ISD::FCOS, ISD::STRICT_FCOS, ISD::FEXP, ISD::STRICT_FEXP, @@ -2482,9 +2494,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, ISD::FLOG, ISD::STRICT_FLOG, ISD::FLOG10, ISD::STRICT_FLOG10, ISD::FPOW, ISD::STRICT_FPOW, - ISD::FSIN, ISD::STRICT_FSIN}) + ISD::FSIN, ISD::STRICT_FSIN, + ISD::FTAN, ISD::STRICT_FTAN}) if (isOperationExpand(Op, MVT::f32)) setOperationAction(Op, MVT::f32, Promote); + // clang-format on // We have target-specific dag combine patterns for the following nodes: setTargetDAGCombine({ISD::VECTOR_SHUFFLE, diff --git a/llvm/test/CodeGen/X86/llvm.tan.ll b/llvm/test/CodeGen/X86/llvm.tan.ll new file mode 100644 index 000000000000..24b30038687f --- /dev/null +++ b/llvm/test/CodeGen/X86/llvm.tan.ll @@ -0,0 +1,70 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s + +define half @use_tanf16(half %a) nounwind { +; CHECK-LABEL: use_tanf16: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rax +; CHECK-NEXT: callq __extendhfsf2@PLT +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: callq __truncsfhf2@PLT +; CHECK-NEXT: popq %rax +; CHECK-NEXT: retq + %x = call half @llvm.tan.f16(half %a) + ret half %x +} + +define float @use_tanf32(float %a) nounwind { +; CHECK-LABEL: use_tanf32: +; CHECK: # %bb.0: +; CHECK-NEXT: jmp tanf@PLT # TAILCALL + %x = call float @llvm.tan.f32(float %a) + ret float %x +} + +define double @use_tanf64(double %a) nounwind { +; CHECK-LABEL: use_tanf64: +; CHECK: # %bb.0: +; CHECK-NEXT: jmp tan@PLT # TAILCALL + %x = call double @llvm.tan.f64(double %a) + ret double %x +} + +define x86_fp80 @use_tanf80(x86_fp80 %a) nounwind { +; CHECK-LABEL: use_tanf80: +; CHECK: # %bb.0: +; CHECK-NEXT: subq $24, %rsp +; CHECK-NEXT: fldt 32(%rsp) +; CHECK-NEXT: fstpt (%rsp) +; CHECK-NEXT: callq tanl@PLT +; CHECK-NEXT: addq $24, %rsp +; CHECK-NEXT: retq + %x = call x86_fp80 @llvm.tan.f80(x86_fp80 %a) + ret x86_fp80 %x +} + +define fp128 @use_tanfp128(fp128 %a) nounwind { +; CHECK-LABEL: use_tanfp128: +; CHECK: # %bb.0: +; CHECK-NEXT: jmp tanf128@PLT # TAILCALL + %x = call fp128 @llvm.tan.f128(fp128 %a) + ret fp128 %x +} + +define ppc_fp128 @use_tanppc_fp128(ppc_fp128 %a) nounwind { +; CHECK-LABEL: use_tanppc_fp128: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rax +; CHECK-NEXT: callq tanl@PLT +; CHECK-NEXT: popq %rax +; CHECK-NEXT: retq + %x = call ppc_fp128 @llvm.tan.ppcf128(ppc_fp128 %a) + ret ppc_fp128 %x +} + +declare half @llvm.tan.f16(half) +declare float @llvm.tan.f32(float) +declare double @llvm.tan.f64(double) +declare x86_fp80 @llvm.tan.f80(x86_fp80) +declare fp128 @llvm.tan.f128(fp128) +declare ppc_fp128 @llvm.tan.ppcf128(ppc_fp128) diff --git a/llvm/test/CodeGen/X86/vec-libcalls.ll b/llvm/test/CodeGen/X86/vec-libcalls.ll index 3a1315446d7a..6857101d3d75 100644 --- a/llvm/test/CodeGen/X86/vec-libcalls.ll +++ b/llvm/test/CodeGen/X86/vec-libcalls.ll @@ -17,6 +17,14 @@ declare <5 x float> @llvm.sin.v5f32(<5 x float>) declare <6 x float> @llvm.sin.v6f32(<6 x float>) declare <3 x double> @llvm.sin.v3f64(<3 x double>) +declare <1 x float> @llvm.tan.v1f32(<1 x float>) +declare <2 x float> @llvm.tan.v2f32(<2 x float>) +declare <3 x float> @llvm.tan.v3f32(<3 x float>) +declare <4 x float> @llvm.tan.v4f32(<4 x float>) +declare <5 x float> @llvm.tan.v5f32(<5 x float>) +declare <6 x float> @llvm.tan.v6f32(<6 x float>) +declare <3 x double> @llvm.tan.v3f64(<3 x double>) + ; Verify that all of the potential libcall candidates are handled. ; Some of these have custom lowering, so those cases won't have ; libcalls. @@ -230,6 +238,200 @@ define <3 x double> @sin_v3f64(<3 x double> %x) nounwind { ret <3 x double> %r } +define <1 x float> @tan_v1f32(<1 x float> %x) nounwind { +; CHECK-LABEL: tan_v1f32: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rax +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: popq %rax +; CHECK-NEXT: retq + %r = call <1 x float> @llvm.tan.v1f32(<1 x float> %x) + ret <1 x float> %r +} + +define <2 x float> @tan_v2f32(<2 x float> %x) nounwind { +; CHECK-LABEL: tan_v2f32: +; CHECK: # %bb.0: +; CHECK-NEXT: subq $40, %rsp +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: vmovshdup (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,1,3,3] +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; CHECK-NEXT: addq $40, %rsp +; CHECK-NEXT: retq + %r = call <2 x float> @llvm.tan.v2f32(<2 x float> %x) + ret <2 x float> %r +} + +define <3 x float> @tan_v3f32(<3 x float> %x) nounwind { +; CHECK-LABEL: tan_v3f32: +; CHECK: # %bb.0: +; CHECK-NEXT: subq $40, %rsp +; CHECK-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,1,3,3] +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,0] +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3] +; CHECK-NEXT: addq $40, %rsp +; CHECK-NEXT: retq + %r = call <3 x float> @llvm.tan.v3f32(<3 x float> %x) + ret <3 x float> %r +} + +define <4 x float> @tan_v4f32(<4 x float> %x) nounwind { +; CHECK-LABEL: tan_v4f32: +; CHECK: # %bb.0: +; CHECK-NEXT: subq $40, %rsp +; CHECK-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,1,3,3] +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,0] +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3] +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[3,3,3,3] +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0] +; CHECK-NEXT: addq $40, %rsp +; CHECK-NEXT: retq + %r = call <4 x float> @llvm.tan.v4f32(<4 x float> %x) + ret <4 x float> %r +} + +define <5 x float> @tan_v5f32(<5 x float> %x) nounwind { +; CHECK-LABEL: tan_v5f32: +; CHECK: # %bb.0: +; CHECK-NEXT: subq $72, %rsp +; CHECK-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,1,3,3] +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,0] +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3] +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[3,3,3,3] +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0] +; CHECK-NEXT: vmovups %ymm0, (%rsp) # 32-byte Spill +; CHECK-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm0 +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovups (%rsp), %ymm1 # 32-byte Reload +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; CHECK-NEXT: addq $72, %rsp +; CHECK-NEXT: retq + %r = call <5 x float> @llvm.tan.v5f32(<5 x float> %x) + ret <5 x float> %r +} + +define <6 x float> @tan_v6f32(<6 x float> %x) nounwind { +; CHECK-LABEL: tan_v6f32: +; CHECK: # %bb.0: +; CHECK-NEXT: subq $72, %rsp +; CHECK-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm0 +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: vmovshdup (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,1,3,3] +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; CHECK-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,1,3,3] +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,0] +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3] +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[3,3,3,3] +; CHECK-NEXT: callq tanf@PLT +; CHECK-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0] +; CHECK-NEXT: vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload +; CHECK-NEXT: addq $72, %rsp +; CHECK-NEXT: retq + %r = call <6 x float> @llvm.tan.v6f32(<6 x float> %x) + ret <6 x float> %r +} + +define <3 x double> @tan_v3f64(<3 x double> %x) nounwind { +; CHECK-LABEL: tan_v3f64: +; CHECK: # %bb.0: +; CHECK-NEXT: subq $72, %rsp +; CHECK-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: callq tan@PLT +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,0] +; CHECK-NEXT: callq tan@PLT +; CHECK-NEXT: vmovapd (%rsp), %xmm1 # 16-byte Reload +; CHECK-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; CHECK-NEXT: vmovupd %ymm0, (%rsp) # 32-byte Spill +; CHECK-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm0 +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: callq tan@PLT +; CHECK-NEXT: vmovups (%rsp), %ymm1 # 32-byte Reload +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; CHECK-NEXT: addq $72, %rsp +; CHECK-NEXT: retq + %r = call <3 x double> @llvm.tan.v3f64(<3 x double> %x) + ret <3 x double> %r +} + define <2 x float> @fabs_v2f32(<2 x float> %x) nounwind { ; CHECK-LABEL: fabs_v2f32: ; CHECK: # %bb.0: diff --git a/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls.ll b/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls.ll index 8d2820a245d9..1627292732b6 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/amdlibm-calls.ll @@ -15,6 +15,11 @@ declare float @cosf(float) #0 declare double @llvm.cos.f64(double) #0 declare float @llvm.cos.f32(float) #0 +declare double @tan(double) #0 +declare float @tanf(float) #0 +declare double @llvm.tan.f64(double) #0 +declare float @llvm.tan.f32(float) #0 + declare double @pow(double, double) #0 declare float @powf(float, float) #0 declare double @llvm.pow.f64(double, double) #0 @@ -264,6 +269,114 @@ for.end: ret void } +define void @tan_f64(ptr nocapture %varray) { +; CHECK-LABEL: @tan_f64( +; CHECK: [[TMP5:%.*]] = call <4 x double> @amd_vrd4_tan(<4 x double> [[TMP4:%.*]]) +; CHECK: ret void +; +; CHECK-AVX512-VF8-LABEL: @tan_f64( +; CHECK-AVX512-VF8: [[TMP5:%.*]] = call <8 x double> @amd_vrd8_tan(<8 x double> [[TMP4:%.*]]) +; CHECK-AVX512-VF8: ret void +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %tmp = trunc i64 %iv to i32 + %conv = sitofp i32 %tmp to double + %call = tail call double @tan(double %conv) + %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv + store double %call, ptr %arrayidx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + +define void @tan_f32(ptr nocapture %varray) { +; CHECK-LABEL: @tan_f32( +; CHECK: [[TMP5:%.*]] = call <4 x float> @amd_vrs4_tanf(<4 x float> [[TMP4:%.*]]) +; CHECK: ret void +; +; CHECK-AVX512-VF16-LABEL: @tan_f32( +; CHECK-AVX512-VF16: [[TMP5:%.*]] = call <16 x float> @amd_vrs16_tanf(<16 x float> [[TMP4:%.*]]) +; CHECK-AVX512-VF16: ret void +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %tmp = trunc i64 %iv to i32 + %conv = sitofp i32 %tmp to float + %call = tail call float @tanf(float %conv) + %arrayidx = getelementptr inbounds float, ptr %varray, i64 %iv + store float %call, ptr %arrayidx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + +define void @tan_f64_intrinsic(ptr nocapture %varray) { +; CHECK-LABEL: @tan_f64_intrinsic( +; CHECK: [[TMP5:%.*]] = call <4 x double> @amd_vrd4_tan(<4 x double> [[TMP4:%.*]]) +; CHECK: ret void +; +; CHECK-AVX512-VF8-LABEL: @tan_f64_intrinsic( +; CHECK-AVX512-VF8: [[TMP5:%.*]] = call <8 x double> @amd_vrd8_tan(<8 x double> [[TMP4:%.*]]) +; CHECK-AVX512-VF8: ret void +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %tmp = trunc i64 %iv to i32 + %conv = sitofp i32 %tmp to double + %call = tail call double @llvm.tan.f64(double %conv) + %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv + store double %call, ptr %arrayidx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + +define void @tan_f32_intrinsic(ptr nocapture %varray) { +; CHECK-LABEL: @tan_f32_intrinsic( +; CHECK: [[TMP5:%.*]] = call <4 x float> @amd_vrs4_tanf(<4 x float> [[TMP4:%.*]]) +; CHECK: ret void +; +; CHECK-AVX512-VF16-LABEL: @tan_f32_intrinsic( +; CHECK-AVX512-VF16: [[TMP5:%.*]] = call <16 x float> @amd_vrs16_tanf(<16 x float> [[TMP4:%.*]]) +; CHECK-AVX512-VF16: ret void +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %tmp = trunc i64 %iv to i32 + %conv = sitofp i32 %tmp to float + %call = tail call float @llvm.tan.f32(float %conv) + %arrayidx = getelementptr inbounds float, ptr %varray, i64 %iv + store float %call, ptr %arrayidx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + define void @pow_f64(ptr nocapture %varray, ptr nocapture readonly %exp) { ; CHECK-LABEL: @pow_f64( ; CHECK: [[TMP8:%.*]] = call <4 x double> @amd_vrd4_pow(<4 x double> [[TMP4:%.*]], <4 x double> [[WIDE_LOAD:%.*]]) diff --git a/llvm/test/Transforms/LoopVectorize/X86/libm-vector-calls-VF2-VF8.ll b/llvm/test/Transforms/LoopVectorize/X86/libm-vector-calls-VF2-VF8.ll index 038852f55f45..67a2cf2b80e7 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/libm-vector-calls-VF2-VF8.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/libm-vector-calls-VF2-VF8.ll @@ -356,6 +356,117 @@ for.end: ; preds = %for.body !132 = !{!"llvm.loop.vectorize.width", i32 8} !133 = !{!"llvm.loop.vectorize.enable", i1 true} +define void @tan_f64(ptr nocapture %varray) { +; CHECK-LABEL: @tan_f64( +; CHECK-LABEL: vector.body +; CHECK: [[TMP5:%.*]] = call <2 x double> @_ZGVbN2v_tan(<2 x double> [[TMP4:%.*]]) +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %tmp = trunc i64 %iv to i32 + %conv = sitofp i32 %tmp to double + %call = tail call double @tan(double %conv) + %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv + store double %call, ptr %arrayidx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1 + +for.end: + ret void +} + +!141 = distinct !{!141, !142, !143} +!142 = !{!"llvm.loop.vectorize.width", i32 2} +!143 = !{!"llvm.loop.vectorize.enable", i1 true} + + +define void @tan_f32(ptr nocapture %varray) { +; CHECK-LABEL: @tan_f32( +; CHECK-LABEL: vector.body +; CHECK: [[TMP5:%.*]] = call <8 x float> @_ZGVdN8v_tanf(<8 x float> [[TMP4:%.*]]) +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %tmp = trunc i64 %iv to i32 + %conv = sitofp i32 %tmp to float + %call = tail call float @tanf(float %conv) + %arrayidx = getelementptr inbounds float, ptr %varray, i64 %iv + store float %call, ptr %arrayidx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !21 + +for.end: + ret void +} + +!151 = distinct !{!151, !152, !153} +!152 = !{!"llvm.loop.vectorize.width", i32 8} +!153 = !{!"llvm.loop.vectorize.enable", i1 true} + +define void @tan_f64_intrinsic(ptr nocapture %varray) { +; CHECK-LABEL: @tan_f64_intrinsic( +; CHECK-LABEL: vector.body +; CHECK: [[TMP5:%.*]] = call <2 x double> @_ZGVbN2v_tan(<2 x double> [[TMP4:%.*]]) +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %tmp = trunc i64 %iv to i32 + %conv = sitofp i32 %tmp to double + %call = tail call double @llvm.tan.f64(double %conv) + %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv + store double %call, ptr %arrayidx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !31 + +for.end: + ret void +} + +!161 = distinct !{!161, !162, !163} +!162 = !{!"llvm.loop.vectorize.width", i32 2} +!163 = !{!"llvm.loop.vectorize.enable", i1 true} + +define void @tan_f32_intrinsic(ptr nocapture %varray) { +; CHECK-LABEL: @tan_f32_intrinsic( +; CHECK-LABEL: vector.body +; CHECK: [[TMP5:%.*]] = call <8 x float> @_ZGVdN8v_tanf(<8 x float> [[TMP4:%.*]]) +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %tmp = trunc i64 %iv to i32 + %conv = sitofp i32 %tmp to float + %call = tail call float @llvm.tan.f32(float %conv) + %arrayidx = getelementptr inbounds float, ptr %varray, i64 %iv + store float %call, ptr %arrayidx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !41 + +for.end: + ret void +} + + + +!171 = distinct !{!171, !172, !173} +!172 = !{!"llvm.loop.vectorize.width", i32 8} +!173 = !{!"llvm.loop.vectorize.enable", i1 true} + attributes #0 = { nounwind readnone } declare double @sin(double) #0 @@ -366,6 +477,10 @@ declare double @cos(double) #0 declare float @cosf(float) #0 declare double @llvm.cos.f64(double) #0 declare float @llvm.cos.f32(float) #0 +declare double @tan(double) #0 +declare float @tanf(float) #0 +declare double @llvm.tan.f64(double) #0 +declare float @llvm.tan.f32(float) #0 declare float @expf(float) #0 declare float @powf(float, float) #0 declare float @llvm.exp.f32(float) #0 diff --git a/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll b/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll index 005557d7445c..2e78e3632fea 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll @@ -230,6 +230,52 @@ for.end: ret void } +define void @tan_f64_intrinsic(ptr nocapture %varray) { +; CHECK-LABEL: @tan_f64_intrinsic( +; CHECK: [[TMP5:%.*]] = call <4 x double> @__svml_tan4(<4 x double> [[TMP4:%.*]]) +; CHECK: ret void +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %tmp = trunc i64 %iv to i32 + %conv = sitofp i32 %tmp to double + %call = tail call double @llvm.tan.f64(double %conv) + %arrayidx = getelementptr inbounds double, ptr %varray, i64 %iv + store double %call, ptr %arrayidx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + +define void @tan_f32_intrinsic(ptr nocapture %varray) { +; CHECK-LABEL: @tan_f32_intrinsic( +; CHECK: [[TMP5:%.*]] = call <4 x float> @__svml_tanf4(<4 x float> [[TMP4:%.*]]) +; CHECK: ret void +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %tmp = trunc i64 %iv to i32 + %conv = sitofp i32 %tmp to float + %call = tail call float @llvm.tan.f32(float %conv) + %arrayidx = getelementptr inbounds float, ptr %varray, i64 %iv + store float %call, ptr %arrayidx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + define void @pow_f64(ptr nocapture %varray, ptr nocapture readonly %exp) { ; CHECK-LABEL: @pow_f64( ; CHECK: [[TMP8:%.*]] = call <4 x double> @__svml_pow4(<4 x double> [[TMP4:%.*]], <4 x double> [[WIDE_LOAD:%.*]]) diff --git a/llvm/test/Transforms/LoopVectorize/X86/veclib-calls.ll b/llvm/test/Transforms/LoopVectorize/X86/veclib-calls.ll index 2e78a96a44b7..27038f3a24b6 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/veclib-calls.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/veclib-calls.ll @@ -406,6 +406,31 @@ for.end: ; preds = %for.body, %entry ret void } +;CHECK-LABEL: @tan_f32_intrinsic( +;CHECK: vtanf{{.*}}<4 x float> +;CHECK: ret void +declare float @llvm.tan.f32(float) nounwind readnone +define void @tan_f32_intrinsic(i32 %n, ptr noalias %y, ptr noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float, ptr %y, i64 %indvars.iv + %0 = load float, ptr %arrayidx, align 4 + %call = tail call float @llvm.tan.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float, ptr %x, i64 %indvars.iv + store float %call, ptr %arrayidx2, align 4 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + ;CHECK-LABEL: @asin_f32( ;CHECK: vasinf{{.*}}<4 x float> ;CHECK: ret void -- GitLab From 3ac6a646d40cb16aebb3467007efa2e499383a3d Mon Sep 17 00:00:00 2001 From: Alex Bradbury Date: Wed, 5 Jun 2024 20:58:41 +0200 Subject: [PATCH 006/462] [RISCV][test] Precommit LSR test that partially motivates #89927 --- .../LoopStrengthReduce/RISCV/many-geps.ll | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 llvm/test/Transforms/LoopStrengthReduce/RISCV/many-geps.ll diff --git a/llvm/test/Transforms/LoopStrengthReduce/RISCV/many-geps.ll b/llvm/test/Transforms/LoopStrengthReduce/RISCV/many-geps.ll new file mode 100644 index 000000000000..4914bb72d894 --- /dev/null +++ b/llvm/test/Transforms/LoopStrengthReduce/RISCV/many-geps.ll @@ -0,0 +1,109 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt < %s -loop-reduce -S | FileCheck %s + +target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128" +target triple = "riscv64" + +; This test was added as example motivation for the changes in #89927, which +; causes LSR to drop solutions if deemed to be less profitable than the +; starting point. At the time of adding this test, LSR's search heuristics +; best identified solution was an unprofitable one. This could of course +; change with future LSR improvements. + +%struct = type { i64, i32, i32, i32, i32, i32, i32, i32, i32, i64, i32, i64, i64, i32, i64 } + +define i32 @main() { +; CHECK-LABEL: define i32 @main() { +; CHECK-NEXT: [[CALL:%.*]] = tail call ptr null(i64 0) +; CHECK-NEXT: br label %[[BB2:.*]] +; CHECK: [[BB1:.*:]] +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[CALL]], align 4 +; CHECK-NEXT: ret i32 0 +; CHECK: [[BB2]]: +; CHECK-NEXT: [[LSR_IV30:%.*]] = phi i64 [ [[LSR_IV_NEXT31:%.*]], %[[BB2]] ], [ 8, [[BB:%.*]] ] +; CHECK-NEXT: [[LSR_IV27:%.*]] = phi i64 [ [[LSR_IV_NEXT28:%.*]], %[[BB2]] ], [ 12, [[BB]] ] +; CHECK-NEXT: [[LSR_IV24:%.*]] = phi i64 [ [[LSR_IV_NEXT25:%.*]], %[[BB2]] ], [ 16, [[BB]] ] +; CHECK-NEXT: [[LSR_IV21:%.*]] = phi i64 [ [[LSR_IV_NEXT22:%.*]], %[[BB2]] ], [ 20, [[BB]] ] +; CHECK-NEXT: [[LSR_IV18:%.*]] = phi i64 [ [[LSR_IV_NEXT19:%.*]], %[[BB2]] ], [ 24, [[BB]] ] +; CHECK-NEXT: [[LSR_IV15:%.*]] = phi i64 [ [[LSR_IV_NEXT16:%.*]], %[[BB2]] ], [ 28, [[BB]] ] +; CHECK-NEXT: [[LSR_IV12:%.*]] = phi i64 [ [[LSR_IV_NEXT13:%.*]], %[[BB2]] ], [ 32, [[BB]] ] +; CHECK-NEXT: [[LSR_IV9:%.*]] = phi i64 [ [[LSR_IV_NEXT10:%.*]], %[[BB2]] ], [ 36, [[BB]] ] +; CHECK-NEXT: [[LSR_IV4:%.*]] = phi i64 [ [[LSR_IV_NEXT5:%.*]], %[[BB2]] ], [ 40, [[BB]] ] +; CHECK-NEXT: [[LSR_IV1:%.*]] = phi i64 [ [[LSR_IV_NEXT2:%.*]], %[[BB2]] ], [ 48, [[BB]] ] +; CHECK-NEXT: [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], %[[BB2]] ], [ 72, [[BB]] ] +; CHECK-NEXT: [[SCEVGEP32:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV30]] +; CHECK-NEXT: store i32 0, ptr [[SCEVGEP32]], align 8 +; CHECK-NEXT: [[SCEVGEP29:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV27]] +; CHECK-NEXT: store i32 0, ptr [[SCEVGEP29]], align 4 +; CHECK-NEXT: [[SCEVGEP26:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV24]] +; CHECK-NEXT: store i32 0, ptr [[SCEVGEP26]], align 8 +; CHECK-NEXT: [[SCEVGEP23:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV21]] +; CHECK-NEXT: store i32 0, ptr [[SCEVGEP23]], align 4 +; CHECK-NEXT: [[SCEVGEP20:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV18]] +; CHECK-NEXT: store i32 0, ptr [[SCEVGEP20]], align 8 +; CHECK-NEXT: [[SCEVGEP17:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV15]] +; CHECK-NEXT: store i32 0, ptr [[SCEVGEP17]], align 4 +; CHECK-NEXT: [[SCEVGEP14:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV12]] +; CHECK-NEXT: store i32 0, ptr [[SCEVGEP14]], align 8 +; CHECK-NEXT: [[SCEVGEP11:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV9]] +; CHECK-NEXT: store i32 0, ptr [[SCEVGEP11]], align 4 +; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV4]] +; CHECK-NEXT: store i64 0, ptr [[SCEVGEP6]], align 8 +; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV1]] +; CHECK-NEXT: store i32 0, ptr [[SCEVGEP3]], align 8 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV]] +; CHECK-NEXT: store i32 0, ptr [[SCEVGEP]], align 8 +; CHECK-NEXT: [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[CALL]], i64 [[LSR_IV4]] +; CHECK-NEXT: [[SCEVGEP8:%.*]] = getelementptr i8, ptr [[SCEVGEP7]], i64 40 +; CHECK-NEXT: store i64 0, ptr [[SCEVGEP8]], align 8 +; CHECK-NEXT: [[LSR_IV_NEXT]] = add i64 [[LSR_IV]], 88 +; CHECK-NEXT: [[LSR_IV_NEXT2]] = add i64 [[LSR_IV1]], 88 +; CHECK-NEXT: [[LSR_IV_NEXT5]] = add i64 [[LSR_IV4]], 88 +; CHECK-NEXT: [[LSR_IV_NEXT10]] = add i64 [[LSR_IV9]], 88 +; CHECK-NEXT: [[LSR_IV_NEXT13]] = add i64 [[LSR_IV12]], 88 +; CHECK-NEXT: [[LSR_IV_NEXT16]] = add i64 [[LSR_IV15]], 88 +; CHECK-NEXT: [[LSR_IV_NEXT19]] = add i64 [[LSR_IV18]], 88 +; CHECK-NEXT: [[LSR_IV_NEXT22]] = add i64 [[LSR_IV21]], 88 +; CHECK-NEXT: [[LSR_IV_NEXT25]] = add i64 [[LSR_IV24]], 88 +; CHECK-NEXT: [[LSR_IV_NEXT28]] = add i64 [[LSR_IV27]], 88 +; CHECK-NEXT: [[LSR_IV_NEXT31]] = add i64 [[LSR_IV30]], 88 +; CHECK-NEXT: br label %[[BB2]] +; +0: + %call = tail call ptr null(i64 0) + br label %2 + +1: + %load = load i32, ptr %call, align 4 + ret i32 0 + +2: + %phi = phi i64 [ 0, %0 ], [ %add, %2 ] + %getelementptr = getelementptr %struct, ptr %call, i64 %phi + %getelementptr3 = getelementptr i8, ptr %getelementptr, i64 8 + store i32 0, ptr %getelementptr3, align 8 + %getelementptr4 = getelementptr i8, ptr %getelementptr, i64 12 + store i32 0, ptr %getelementptr4, align 4 + %getelementptr5 = getelementptr i8, ptr %getelementptr, i64 16 + store i32 0, ptr %getelementptr5, align 8 + %getelementptr6 = getelementptr i8, ptr %getelementptr, i64 20 + store i32 0, ptr %getelementptr6, align 4 + %getelementptr7 = getelementptr i8, ptr %getelementptr, i64 24 + store i32 0, ptr %getelementptr7, align 8 + %getelementptr8 = getelementptr i8, ptr %getelementptr, i64 28 + store i32 0, ptr %getelementptr8, align 4 + %getelementptr9 = getelementptr i8, ptr %getelementptr, i64 32 + store i32 0, ptr %getelementptr9, align 8 + %getelementptr10 = getelementptr i8, ptr %getelementptr, i64 36 + store i32 0, ptr %getelementptr10, align 4 + %getelementptr11 = getelementptr i8, ptr %getelementptr, i64 40 + store i64 0, ptr %getelementptr11, align 8 + %getelementptr12 = getelementptr i8, ptr %getelementptr, i64 48 + store i32 0, ptr %getelementptr12, align 8 + %getelementptr13 = getelementptr i8, ptr %getelementptr, i64 72 + store i32 0, ptr %getelementptr13, align 8 + %getelementptr14 = getelementptr i8, ptr %getelementptr, i64 80 + store i64 0, ptr %getelementptr14, align 8 + %add = add i64 %phi, 1 + br label %2 +} -- GitLab From 21711f89b9d85028160611f725bd33d7832d1d46 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 5 Jun 2024 12:22:03 -0700 Subject: [PATCH 007/462] [LegalizeVectorOps] Move VP_STORE legalization from LegalizeDAG to LegalizeVectorOps. 705636a1130551ab105aec95b909a35a0305fc9f moved reductions from LegalizeVectorOps to LegalizeDAG, but the way it was done inadvertently moved stores from LegalizeVectorOps to LegalizeDAG too. This was not intended or desired. Found when this was pulled into my downstream which has other changes that make the distinction important. --- llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index 5c5347b7f314..042684a434fd 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -507,7 +507,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { break; \ } \ /* Defer non-vector results to LegalizeDAG. */ \ - if (!Node->getValueType(0).isVector()) { \ + if (!Node->getValueType(0).isVector() && \ + Node->getValueType(0) != MVT::Other) { \ Action = TargetLowering::Legal; \ break; \ } \ -- GitLab From 73a2fd470c58eb10e2e51466a053dd30b63bbe7a Mon Sep 17 00:00:00 2001 From: Artem Kroviakov <71938912+akroviakov@users.noreply.github.com> Date: Wed, 5 Jun 2024 21:39:05 +0200 Subject: [PATCH 008/462] [mlir][xegpu] Patch dynamic descriptor creation (#93580) fixes the bug in XeGPU's `CreateNdDescOp` tensor creation with dynamic offset and strides. --- mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp index 22959224d56c..5ef47fbbe1ce 100644 --- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp +++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp @@ -110,7 +110,7 @@ void CreateNdDescOp::build(OpBuilder &builder, OperationState &state, dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets); dispatchIndexOpFoldResults(shape, dynamicShape, staticShape); - dispatchIndexOpFoldResults(strides, dynamicStrides, staticOffsets); + dispatchIndexOpFoldResults(strides, dynamicStrides, staticStrides); auto staticOffsetsAttr = builder.getDenseI64ArrayAttr(staticOffsets); auto staticShapeAttr = builder.getDenseI64ArrayAttr(staticShape); -- GitLab From 3beb232fb4fd1943155174d58a454493828cd719 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Wed, 5 Jun 2024 21:42:44 +0200 Subject: [PATCH 009/462] Fix clang reject valid C++ code after d999ce0302f06d250f6d496b56a5a5f (#94471) The incremental processing mode doesn't seem to work well for C++, see the https://github.com/llvm/llvm-project/pull/89804#issuecomment-2149840711 for details. --- clang/lib/Interpreter/IncrementalParser.cpp | 3 ++- clang/lib/Sema/SemaDecl.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/clang/lib/Interpreter/IncrementalParser.cpp b/clang/lib/Interpreter/IncrementalParser.cpp index 5bc8385d874a..a8d0294fb615 100644 --- a/clang/lib/Interpreter/IncrementalParser.cpp +++ b/clang/lib/Interpreter/IncrementalParser.cpp @@ -413,7 +413,8 @@ void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) { if (!ND) continue; // Check if we need to clean up the IdResolver chain. - if (ND->getDeclName().getFETokenInfo()) + if (ND->getDeclName().getFETokenInfo() && !D->getLangOpts().ObjC && + !D->getLangOpts().CPlusPlus) getCI()->getSema().IdResolver.RemoveDecl(ND); } } diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index a6734ef8c30a..4b9b735f1cfb 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -2288,7 +2288,8 @@ void Sema::ActOnPopScope(SourceLocation Loc, Scope *S) { // Partial translation units that are created in incremental processing must // not clean up the IdResolver because PTUs should take into account the // declarations that came from previous PTUs. - if (!PP.isIncrementalProcessingEnabled() || getLangOpts().ObjC) + if (!PP.isIncrementalProcessingEnabled() || getLangOpts().ObjC || + getLangOpts().CPlusPlus) IdResolver.RemoveDecl(D); // Warn on it if we are shadowing a declaration. -- GitLab From a4b32c25761e3de55d42a4799a303f36aa198fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Wed, 5 Jun 2024 22:49:58 +0300 Subject: [PATCH 010/462] Revert "[compiler-rt][builtins] Switch libatomic locks to pthread_mutex_t (#94374)" This reverts commit b62b7a42bbee4a3bbf9094808f460fdc9c119bd7 and a5729b71d844c1444f7d348dc2d4ea5b98de5ec5. This commit broke compilation for systems that lack pthreads. --- compiler-rt/lib/builtins/atomic.c | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/compiler-rt/lib/builtins/atomic.c b/compiler-rt/lib/builtins/atomic.c index c3a36a9aaba6..852bb20f0867 100644 --- a/compiler-rt/lib/builtins/atomic.c +++ b/compiler-rt/lib/builtins/atomic.c @@ -51,14 +51,6 @@ #endif static const long SPINLOCK_MASK = SPINLOCK_COUNT - 1; -#ifndef CACHE_LINE_SIZE -#define CACHE_LINE_SIZE 64 -#endif - -#ifdef __clang__ -#pragma clang diagnostic ignored "-Wgnu-designator" -#endif - //////////////////////////////////////////////////////////////////////////////// // Platform-specific lock implementation. Falls back to spinlocks if none is // defined. Each platform should define the Lock type, and corresponding @@ -102,18 +94,21 @@ static Lock locks[SPINLOCK_COUNT]; // initialized to OS_SPINLOCK_INIT which is 0 #else _Static_assert(__atomic_always_lock_free(sizeof(uintptr_t), 0), "Implementation assumes lock-free pointer-size cmpxchg"); -#include -#include -typedef struct { - alignas(CACHE_LINE_SIZE) pthread_mutex_t m; -} Lock; +typedef _Atomic(uintptr_t) Lock; /// Unlock a lock. This is a release operation. -__inline static void unlock(Lock *l) { pthread_mutex_unlock(&l->m); } -/// Locks a lock. -__inline static void lock(Lock *l) { pthread_mutex_lock(&l->m); } +__inline static void unlock(Lock *l) { + __c11_atomic_store(l, 0, __ATOMIC_RELEASE); +} +/// Locks a lock. In the current implementation, this is potentially +/// unbounded in the contended case. +__inline static void lock(Lock *l) { + uintptr_t old = 0; + while (!__c11_atomic_compare_exchange_weak(l, &old, 1, __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED)) + old = 0; +} /// locks for atomic operations -static Lock locks[SPINLOCK_COUNT] = { - [0 ... SPINLOCK_COUNT - 1] = {PTHREAD_MUTEX_INITIALIZER}}; +static Lock locks[SPINLOCK_COUNT]; #endif /// Returns a lock to use for a given pointer. -- GitLab From b477d1de6e3a5e5860fe7fdf272e8bbb6a330f48 Mon Sep 17 00:00:00 2001 From: PiJoules <6019989+PiJoules@users.noreply.github.com> Date: Wed, 5 Jun 2024 12:59:39 -0700 Subject: [PATCH 011/462] [libc][type_traits] Add aligned_storage (#94074) --- libc/src/__support/CPP/CMakeLists.txt | 1 + libc/src/__support/CPP/type_traits.h | 1 + .../CPP/type_traits/aligned_storage.h | 27 +++++++++++++++++++ .../src/__support/CPP/type_traits_test.cpp | 9 +++++++ 4 files changed, 38 insertions(+) create mode 100644 libc/src/__support/CPP/type_traits/aligned_storage.h diff --git a/libc/src/__support/CPP/CMakeLists.txt b/libc/src/__support/CPP/CMakeLists.txt index 08661aba5b6b..e6f58b7cd822 100644 --- a/libc/src/__support/CPP/CMakeLists.txt +++ b/libc/src/__support/CPP/CMakeLists.txt @@ -111,6 +111,7 @@ add_header_library( type_traits/add_lvalue_reference.h type_traits/add_pointer.h type_traits/add_rvalue_reference.h + type_traits/aligned_storage.h type_traits/always_false.h type_traits/bool_constant.h type_traits/conditional.h diff --git a/libc/src/__support/CPP/type_traits.h b/libc/src/__support/CPP/type_traits.h index 1494aeb905e0..d50b6612656d 100644 --- a/libc/src/__support/CPP/type_traits.h +++ b/libc/src/__support/CPP/type_traits.h @@ -12,6 +12,7 @@ #include "src/__support/CPP/type_traits/add_lvalue_reference.h" #include "src/__support/CPP/type_traits/add_pointer.h" #include "src/__support/CPP/type_traits/add_rvalue_reference.h" +#include "src/__support/CPP/type_traits/aligned_storage.h" #include "src/__support/CPP/type_traits/bool_constant.h" #include "src/__support/CPP/type_traits/conditional.h" #include "src/__support/CPP/type_traits/decay.h" diff --git a/libc/src/__support/CPP/type_traits/aligned_storage.h b/libc/src/__support/CPP/type_traits/aligned_storage.h new file mode 100644 index 000000000000..574b1146f6b2 --- /dev/null +++ b/libc/src/__support/CPP/type_traits/aligned_storage.h @@ -0,0 +1,27 @@ +//===-- aligned_storage type_traits --------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_ALIGNED_STORAGE_H +#define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_ALIGNED_STORAGE_H + +#include // size_t + +namespace LIBC_NAMESPACE::cpp { + +template struct aligned_storage { + struct type { + alignas(Align) unsigned char data[Len]; + }; +}; + +template +using aligned_storage_t = typename aligned_storage::type; + +} // namespace LIBC_NAMESPACE::cpp + +#endif // LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_ALIGNED_STORAGE_H diff --git a/libc/test/src/__support/CPP/type_traits_test.cpp b/libc/test/src/__support/CPP/type_traits_test.cpp index a2051f380275..1c428e93a2fd 100644 --- a/libc/test/src/__support/CPP/type_traits_test.cpp +++ b/libc/test/src/__support/CPP/type_traits_test.cpp @@ -112,6 +112,15 @@ TEST(LlvmLibcTypeTraitsTest, add_rvalue_reference_void) { const volatile void>)); } +TEST(LlvmLibcTypeTraitsTest, aligned_storage) { + struct S { + int a, b; + }; + aligned_storage_t buf; + EXPECT_EQ(alignof(buf), alignof(S)); + EXPECT_EQ(sizeof(buf), sizeof(S)); +} + TEST(LlvmLibcTypeTraitsTest, bool_constant) { EXPECT_TRUE((bool_constant::value)); EXPECT_FALSE((bool_constant::value)); -- GitLab From f8afa763c6194f5bf485480e1fb94b953942f876 Mon Sep 17 00:00:00 2001 From: Med Ismail Bennani Date: Wed, 5 Jun 2024 13:21:27 -0700 Subject: [PATCH 012/462] [lldb/crashlog] Use environment variable to manually set dsymForUUIDBinary (#94517) In lldb, users can change the `dsymForUUID` binary using the `LLDB_APPLE_DSYMFORUUID_EXECUTABLE` environment variable. This patch changes the crashlog to support the same behaviour as lldb and uses this environment variable to disable `dsymForUUID` lookups in crashlog test by having it be empty. Since CI bots shoudn't have access to images on build records, it doesn't make sense to make use of `dsymForUUID` in tests. rdar://128953725 Signed-off-by: Med Ismail Bennani --- lldb/examples/python/crashlog.py | 4 +++- .../Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lldb/examples/python/crashlog.py b/lldb/examples/python/crashlog.py index 641b2e64d53b..c874cb4d32e6 100755 --- a/lldb/examples/python/crashlog.py +++ b/lldb/examples/python/crashlog.py @@ -284,7 +284,9 @@ class CrashLog(symbolication.Symbolicator): """Class that represents a binary images in a darwin crash log""" dsymForUUIDBinary = "/usr/local/bin/dsymForUUID" - if not os.path.exists(dsymForUUIDBinary): + if "LLDB_APPLE_DSYMFORUUID_EXECUTABLE" in os.environ: + dsymForUUIDBinary = os.environ["LLDB_APPLE_DSYMFORUUID_EXECUTABLE"] + elif not os.path.exists(dsymForUUIDBinary): try: dsymForUUIDBinary = ( subprocess.check_output("which dsymForUUID", shell=True) diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg index 417069653d68..3da9265b3553 100644 --- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg +++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg @@ -3,3 +3,5 @@ if 'system-darwin' not in config.available_features: if 'lldb-repro' in config.available_features: config.unsupported = True + +config.environment["LLDB_APPLE_DSYMFORUUID_EXECUTABLE"] = "" -- GitLab From e675d0d5cd4047bcf4ec368d2b4c19a57e5347f3 Mon Sep 17 00:00:00 2001 From: erichkeane Date: Tue, 4 Jun 2024 10:57:17 -0700 Subject: [PATCH 013/462] [OpenACC][NFC] Refactor SemaOpenACC to use a visitor This code was getting unwieldy, so a quick patch to refactor it a bit. --- clang/include/clang/AST/OpenACCClause.h | 2 +- clang/include/clang/Basic/OpenACCClauses.def | 22 +- clang/lib/AST/OpenACCClause.cpp | 2 +- clang/lib/Sema/SemaOpenACC.cpp | 1193 ++++++++++-------- 4 files changed, 669 insertions(+), 550 deletions(-) diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h index a4c82cdad87b..ea1ffbc7fd08 100644 --- a/clang/include/clang/AST/OpenACCClause.h +++ b/clang/include/clang/AST/OpenACCClause.h @@ -867,7 +867,7 @@ public: case OpenACCClauseKind::CLAUSE_NAME: \ Visit##CLAUSE_NAME##Clause(*cast(C)); \ return; -#define CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME) \ +#define CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME, DEPRECATED) \ case OpenACCClauseKind::ALIAS_NAME: \ Visit##CLAUSE_NAME##Clause(*cast(C)); \ return; diff --git a/clang/include/clang/Basic/OpenACCClauses.def b/clang/include/clang/Basic/OpenACCClauses.def index 53f4cd103d14..85f4859925f0 100644 --- a/clang/include/clang/Basic/OpenACCClauses.def +++ b/clang/include/clang/Basic/OpenACCClauses.def @@ -15,31 +15,31 @@ // // VISIT_CLAUSE(CLAUSE_NAME) // -// CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME) +// CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME, DEPRECATED) #ifndef CLAUSE_ALIAS -#define CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME) +#define CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME, false) #endif VISIT_CLAUSE(Auto) VISIT_CLAUSE(Async) VISIT_CLAUSE(Attach) VISIT_CLAUSE(Copy) -CLAUSE_ALIAS(PCopy, Copy) -CLAUSE_ALIAS(PresentOrCopy, Copy) +CLAUSE_ALIAS(PCopy, Copy, true) +CLAUSE_ALIAS(PresentOrCopy, Copy, true) VISIT_CLAUSE(CopyIn) -CLAUSE_ALIAS(PCopyIn, CopyIn) -CLAUSE_ALIAS(PresentOrCopyIn, CopyIn) +CLAUSE_ALIAS(PCopyIn, CopyIn, true) +CLAUSE_ALIAS(PresentOrCopyIn, CopyIn, true) VISIT_CLAUSE(CopyOut) -CLAUSE_ALIAS(PCopyOut, CopyOut) -CLAUSE_ALIAS(PresentOrCopyOut, CopyOut) +CLAUSE_ALIAS(PCopyOut, CopyOut, true) +CLAUSE_ALIAS(PresentOrCopyOut, CopyOut, true) VISIT_CLAUSE(Create) -CLAUSE_ALIAS(PCreate, Create) -CLAUSE_ALIAS(PresentOrCreate, Create) +CLAUSE_ALIAS(PCreate, Create, true) +CLAUSE_ALIAS(PresentOrCreate, Create, true) VISIT_CLAUSE(Default) VISIT_CLAUSE(DevicePtr) VISIT_CLAUSE(DeviceType) -CLAUSE_ALIAS(DType, DeviceType) +CLAUSE_ALIAS(DType, DeviceType, false) VISIT_CLAUSE(FirstPrivate) VISIT_CLAUSE(If) VISIT_CLAUSE(Independent) diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp index 403ce9a22157..95089a9b79e2 100644 --- a/clang/lib/AST/OpenACCClause.cpp +++ b/clang/lib/AST/OpenACCClause.cpp @@ -104,7 +104,7 @@ OpenACCClause::child_range OpenACCClause::children() { #define VISIT_CLAUSE(CLAUSE_NAME) \ case OpenACCClauseKind::CLAUSE_NAME: \ return cast(this)->children(); -#define CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME) \ +#define CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME, DEPRECATED) \ case OpenACCClauseKind::ALIAS_NAME: \ return cast(this)->children(); diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp index 1320fd90b440..97586a037eee 100644 --- a/clang/lib/Sema/SemaOpenACC.cpp +++ b/clang/lib/Sema/SemaOpenACC.cpp @@ -424,617 +424,736 @@ bool checkValidAfterDeviceType( S.Diag(DeviceTypeClause.getBeginLoc(), diag::note_acc_previous_clause_here); return true; } -} // namespace -SemaOpenACC::SemaOpenACC(Sema &S) : SemaBase(S) {} +class SemaOpenACCClauseVisitor { + SemaOpenACC &SemaRef; + ASTContext &Ctx; + ArrayRef ExistingClauses; + bool NotImplemented = false; -SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(SemaOpenACC &S, - OpenACCDirectiveKind DK) - : SemaRef(S), WasInsideComputeConstruct(S.InsideComputeConstruct), - DirKind(DK) { - // Compute constructs end up taking their 'loop'. - if (DirKind == OpenACCDirectiveKind::Parallel || - DirKind == OpenACCDirectiveKind::Serial || - DirKind == OpenACCDirectiveKind::Kernels) { - SemaRef.InsideComputeConstruct = true; - SemaRef.ParentlessLoopConstructs.swap(ParentlessLoopConstructs); + OpenACCClause *isNotImplemented() { + NotImplemented = true; + return nullptr; } -} -SemaOpenACC::AssociatedStmtRAII::~AssociatedStmtRAII() { - SemaRef.InsideComputeConstruct = WasInsideComputeConstruct; - if (DirKind == OpenACCDirectiveKind::Parallel || - DirKind == OpenACCDirectiveKind::Serial || - DirKind == OpenACCDirectiveKind::Kernels) { - assert(SemaRef.ParentlessLoopConstructs.empty() && - "Didn't consume loop construct list?"); - SemaRef.ParentlessLoopConstructs.swap(ParentlessLoopConstructs); - } -} +public: + SemaOpenACCClauseVisitor(SemaOpenACC &S, + ArrayRef ExistingClauses) + : SemaRef(S), Ctx(S.getASTContext()), ExistingClauses(ExistingClauses) {} + // Once we've implemented everything, we shouldn't need this infrastructure. + // But in the meantime, we use this to help decide whether the clause was + // handled for this directive. + bool diagNotImplemented() { return NotImplemented; } + + OpenACCClause *Visit(SemaOpenACC::OpenACCParsedClause &Clause) { + switch (Clause.getClauseKind()) { + case OpenACCClauseKind::Gang: + case OpenACCClauseKind::Worker: + case OpenACCClauseKind::Vector: { + // TODO OpenACC: These are only implemented enough for the 'seq' diagnostic, + // otherwise treats itself as unimplemented. When we implement these, we + // can remove them from here. -OpenACCClause * -SemaOpenACC::ActOnClause(ArrayRef ExistingClauses, - OpenACCParsedClause &Clause) { - if (Clause.getClauseKind() == OpenACCClauseKind::Invalid) - return nullptr; + // OpenACC 3.3 2.9: + // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause + // appears. + const auto *Itr = + llvm::find_if(ExistingClauses, llvm::IsaPred); - // Diagnose that we don't support this clause on this directive. - if (!doesClauseApplyToDirective(Clause.getDirectiveKind(), - Clause.getClauseKind())) { - Diag(Clause.getBeginLoc(), diag::err_acc_clause_appertainment) - << Clause.getDirectiveKind() << Clause.getClauseKind(); - return nullptr; + if (Itr != ExistingClauses.end()) { + SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine) + << Clause.getClauseKind() << (*Itr)->getClauseKind(); + SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); + } + return isNotImplemented(); } - if (const auto *DevTypeClause = - llvm::find_if(ExistingClauses, - [&](const OpenACCClause *C) { - return isa(C); - }); - DevTypeClause != ExistingClauses.end()) { - if (checkValidAfterDeviceType( - *this, *cast(*DevTypeClause), Clause)) - return nullptr; +#define VISIT_CLAUSE(CLAUSE_NAME) \ + case OpenACCClauseKind::CLAUSE_NAME: \ + return Visit##CLAUSE_NAME##Clause(Clause); +#define CLAUSE_ALIAS(ALIAS, CLAUSE_NAME, DEPRECATED) \ + case OpenACCClauseKind::ALIAS: \ + if (DEPRECATED) \ + SemaRef.Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name) \ + << Clause.getClauseKind() << OpenACCClauseKind::CLAUSE_NAME; \ + return Visit##CLAUSE_NAME##Clause(Clause); +#include "clang/Basic/OpenACCClauses.def" + default: + return isNotImplemented(); + } + llvm_unreachable("Invalid clause kind"); } - switch (Clause.getClauseKind()) { - case OpenACCClauseKind::Default: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; +#define VISIT_CLAUSE(CLAUSE_NAME) \ + OpenACCClause *Visit##CLAUSE_NAME##Clause( \ + SemaOpenACC::OpenACCParsedClause &Clause); +#include "clang/Basic/OpenACCClauses.def" +}; - // Don't add an invalid clause to the AST. - if (Clause.getDefaultClauseKind() == OpenACCDefaultClauseKind::Invalid) - return nullptr; - - // OpenACC 3.3, Section 2.5.4: - // At most one 'default' clause may appear, and it must have a value of - // either 'none' or 'present'. - // Second half of the sentence is diagnosed during parsing. - if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause)) - return nullptr; - - return OpenACCDefaultClause::Create( - getASTContext(), Clause.getDefaultClauseKind(), Clause.getBeginLoc(), - Clause.getLParenLoc(), Clause.getEndLoc()); - } +OpenACCClause *SemaOpenACCClauseVisitor::VisitDefaultClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + + // Don't add an invalid clause to the AST. + if (Clause.getDefaultClauseKind() == OpenACCDefaultClauseKind::Invalid) + return nullptr; - case OpenACCClauseKind::If: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; + // OpenACC 3.3, Section 2.5.4: + // At most one 'default' clause may appear, and it must have a value of + // either 'none' or 'present'. + // Second half of the sentence is diagnosed during parsing. + if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause)) + return nullptr; - // There is no prose in the standard that says duplicates aren't allowed, - // but this diagnostic is present in other compilers, as well as makes - // sense. - if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause)) - return nullptr; + return OpenACCDefaultClause::Create( + Ctx, Clause.getDefaultClauseKind(), Clause.getBeginLoc(), + Clause.getLParenLoc(), Clause.getEndLoc()); +} - // The parser has ensured that we have a proper condition expr, so there - // isn't really much to do here. +OpenACCClause *SemaOpenACCClauseVisitor::VisitIfClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + + // There is no prose in the standard that says duplicates aren't allowed, + // but this diagnostic is present in other compilers, as well as makes + // sense. + if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause)) + return nullptr; - // If the 'if' clause is true, it makes the 'self' clause have no effect, - // diagnose that here. - // TODO OpenACC: When we add these two to other constructs, we might not - // want to warn on this (for example, 'update'). - const auto *Itr = - llvm::find_if(ExistingClauses, llvm::IsaPred); - if (Itr != ExistingClauses.end()) { - Diag(Clause.getBeginLoc(), diag::warn_acc_if_self_conflict); - Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); - } + // The parser has ensured that we have a proper condition expr, so there + // isn't really much to do here. - return OpenACCIfClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(), - Clause.getConditionExpr(), Clause.getEndLoc()); + // If the 'if' clause is true, it makes the 'self' clause have no effect, + // diagnose that here. + // TODO OpenACC: When we add these two to other constructs, we might not + // want to warn on this (for example, 'update'). + const auto *Itr = + llvm::find_if(ExistingClauses, llvm::IsaPred); + if (Itr != ExistingClauses.end()) { + SemaRef.Diag(Clause.getBeginLoc(), diag::warn_acc_if_self_conflict); + SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); } - case OpenACCClauseKind::Self: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; - - // TODO OpenACC: When we implement this for 'update', this takes a - // 'var-list' instead of a condition expression, so semantics/handling has - // to happen differently here. - - // There is no prose in the standard that says duplicates aren't allowed, - // but this diagnostic is present in other compilers, as well as makes - // sense. - if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause)) - return nullptr; + return OpenACCIfClause::Create(Ctx, Clause.getBeginLoc(), + Clause.getLParenLoc(), + Clause.getConditionExpr(), Clause.getEndLoc()); +} - // If the 'if' clause is true, it makes the 'self' clause have no effect, - // diagnose that here. - // TODO OpenACC: When we add these two to other constructs, we might not - // want to warn on this (for example, 'update'). - const auto *Itr = - llvm::find_if(ExistingClauses, llvm::IsaPred); - if (Itr != ExistingClauses.end()) { - Diag(Clause.getBeginLoc(), diag::warn_acc_if_self_conflict); - Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); - } +OpenACCClause *SemaOpenACCClauseVisitor::VisitSelfClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + + // TODO OpenACC: When we implement this for 'update', this takes a + // 'var-list' instead of a condition expression, so semantics/handling has + // to happen differently here. + + // There is no prose in the standard that says duplicates aren't allowed, + // but this diagnostic is present in other compilers, as well as makes + // sense. + if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause)) + return nullptr; - return OpenACCSelfClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(), - Clause.getConditionExpr(), Clause.getEndLoc()); + // If the 'if' clause is true, it makes the 'self' clause have no effect, + // diagnose that here. + // TODO OpenACC: When we add these two to other constructs, we might not + // want to warn on this (for example, 'update'). + const auto *Itr = + llvm::find_if(ExistingClauses, llvm::IsaPred); + if (Itr != ExistingClauses.end()) { + SemaRef.Diag(Clause.getBeginLoc(), diag::warn_acc_if_self_conflict); + SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); } - case OpenACCClauseKind::NumGangs: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; + return OpenACCSelfClause::Create( + Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), + Clause.getConditionExpr(), Clause.getEndLoc()); +} - // There is no prose in the standard that says duplicates aren't allowed, - // but this diagnostic is present in other compilers, as well as makes - // sense. - if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause)) - return nullptr; +OpenACCClause *SemaOpenACCClauseVisitor::VisitNumGangsClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + + // There is no prose in the standard that says duplicates aren't allowed, + // but this diagnostic is present in other compilers, as well as makes + // sense. + if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause)) + return nullptr; - if (Clause.getIntExprs().empty()) - Diag(Clause.getBeginLoc(), diag::err_acc_num_gangs_num_args) - << /*NoArgs=*/0; - - unsigned MaxArgs = - (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel || - Clause.getDirectiveKind() == OpenACCDirectiveKind::ParallelLoop) - ? 3 - : 1; - if (Clause.getIntExprs().size() > MaxArgs) - Diag(Clause.getBeginLoc(), diag::err_acc_num_gangs_num_args) - << /*NoArgs=*/1 << Clause.getDirectiveKind() << MaxArgs + // num_gangs requires at least 1 int expr in all forms. Diagnose here, but + // allow us to continue, an empty clause might be useful for future + // diagnostics. + if (Clause.getIntExprs().empty()) + SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_num_gangs_num_args) + << /*NoArgs=*/0; + + unsigned MaxArgs = + (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel || + Clause.getDirectiveKind() == OpenACCDirectiveKind::ParallelLoop) + ? 3 + : 1; + // The max number of args differs between parallel and other constructs. + // Again, allow us to continue for the purposes of future diagnostics. + if (Clause.getIntExprs().size() > MaxArgs) + SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_num_gangs_num_args) + << /*NoArgs=*/1 << Clause.getDirectiveKind() << MaxArgs + << Clause.getIntExprs().size(); + + // OpenACC 3.3 Section 2.5.4: + // A reduction clause may not appear on a parallel construct with a + // num_gangs clause that has more than one argument. + if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel && + Clause.getIntExprs().size() > 1) { + auto *Parallel = + llvm::find_if(ExistingClauses, llvm::IsaPred); + + if (Parallel != ExistingClauses.end()) { + SemaRef.Diag(Clause.getBeginLoc(), + diag::err_acc_reduction_num_gangs_conflict) << Clause.getIntExprs().size(); - - // OpenACC 3.3 Section 2.5.4: - // A reduction clause may not appear on a parallel construct with a - // num_gangs clause that has more than one argument. - if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel && - Clause.getIntExprs().size() > 1) { - auto *Parallel = - llvm::find_if(ExistingClauses, llvm::IsaPred); - - if (Parallel != ExistingClauses.end()) { - Diag(Clause.getBeginLoc(), diag::err_acc_reduction_num_gangs_conflict) - << Clause.getIntExprs().size(); - Diag((*Parallel)->getBeginLoc(), diag::note_acc_previous_clause_here); - return nullptr; - } + SemaRef.Diag((*Parallel)->getBeginLoc(), + diag::note_acc_previous_clause_here); + return nullptr; } - - // Create the AST node for the clause even if the number of expressions is - // incorrect. - return OpenACCNumGangsClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(), - Clause.getIntExprs(), Clause.getEndLoc()); - break; } - case OpenACCClauseKind::NumWorkers: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; + return OpenACCNumGangsClause::Create( + Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getIntExprs(), + Clause.getEndLoc()); +} - // There is no prose in the standard that says duplicates aren't allowed, - // but this diagnostic is present in other compilers, as well as makes - // sense. - if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause)) - return nullptr; +OpenACCClause *SemaOpenACCClauseVisitor::VisitNumWorkersClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + + // There is no prose in the standard that says duplicates aren't allowed, + // but this diagnostic is present in other compilers, as well as makes + // sense. + if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause)) + return nullptr; - assert(Clause.getIntExprs().size() == 1 && - "Invalid number of expressions for NumWorkers"); - return OpenACCNumWorkersClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(), - Clause.getIntExprs()[0], Clause.getEndLoc()); - } - case OpenACCClauseKind::VectorLength: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; + assert(Clause.getIntExprs().size() == 1 && + "Invalid number of expressions for NumWorkers"); + return OpenACCNumWorkersClause::Create( + Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getIntExprs()[0], + Clause.getEndLoc()); +} - // There is no prose in the standard that says duplicates aren't allowed, - // but this diagnostic is present in other compilers, as well as makes - // sense. - if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause)) - return nullptr; +OpenACCClause *SemaOpenACCClauseVisitor::VisitVectorLengthClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + + // There is no prose in the standard that says duplicates aren't allowed, + // but this diagnostic is present in other compilers, as well as makes + // sense. + if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause)) + return nullptr; - assert(Clause.getIntExprs().size() == 1 && - "Invalid number of expressions for VectorLength"); - return OpenACCVectorLengthClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(), - Clause.getIntExprs()[0], Clause.getEndLoc()); - } - case OpenACCClauseKind::Async: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; + assert(Clause.getIntExprs().size() == 1 && + "Invalid number of expressions for NumWorkers"); + return OpenACCVectorLengthClause::Create( + Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getIntExprs()[0], + Clause.getEndLoc()); +} - // There is no prose in the standard that says duplicates aren't allowed, - // but this diagnostic is present in other compilers, as well as makes - // sense. - if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause)) - return nullptr; +OpenACCClause *SemaOpenACCClauseVisitor::VisitAsyncClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + + // There is no prose in the standard that says duplicates aren't allowed, + // but this diagnostic is present in other compilers, as well as makes + // sense. + if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause)) + return nullptr; - assert(Clause.getNumIntExprs() < 2 && - "Invalid number of expressions for Async"); + assert(Clause.getNumIntExprs() < 2 && + "Invalid number of expressions for Async"); + return OpenACCAsyncClause::Create( + Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), + Clause.getNumIntExprs() != 0 ? Clause.getIntExprs()[0] : nullptr, + Clause.getEndLoc()); +} - return OpenACCAsyncClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(), - Clause.getNumIntExprs() != 0 ? Clause.getIntExprs()[0] : nullptr, - Clause.getEndLoc()); - } - case OpenACCClauseKind::Private: { - // Restrictions only properly implemented on 'compute' and 'loop' - // constructs, and 'compute'/'loop' constructs are the only construct that - // can do anything with this yet, so skip/treat as unimplemented in this - // case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) && - Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop) - break; +OpenACCClause *SemaOpenACCClauseVisitor::VisitPrivateClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' and 'loop' + // constructs, and 'compute'/'loop' constructs are the only construct that + // can do anything with this yet, so skip/treat as unimplemented in this + // case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) && + Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop) + return isNotImplemented(); + + // ActOnVar ensured that everything is a valid variable reference, so there + // really isn't anything to do here. GCC does some duplicate-finding, though + // it isn't apparent in the standard where this is justified. + + return OpenACCPrivateClause::Create(Ctx, Clause.getBeginLoc(), + Clause.getLParenLoc(), + Clause.getVarList(), Clause.getEndLoc()); +} - // ActOnVar ensured that everything is a valid variable reference, so there - // really isn't anything to do here. GCC does some duplicate-finding, though - // it isn't apparent in the standard where this is justified. +OpenACCClause *SemaOpenACCClauseVisitor::VisitFirstPrivateClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + + // ActOnVar ensured that everything is a valid variable reference, so there + // really isn't anything to do here. GCC does some duplicate-finding, though + // it isn't apparent in the standard where this is justified. + + return OpenACCFirstPrivateClause::Create( + Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getVarList(), + Clause.getEndLoc()); +} - return OpenACCPrivateClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(), - Clause.getVarList(), Clause.getEndLoc()); - } - case OpenACCClauseKind::FirstPrivate: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; +OpenACCClause *SemaOpenACCClauseVisitor::VisitNoCreateClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + // ActOnVar ensured that everything is a valid variable reference, so there + // really isn't anything to do here. GCC does some duplicate-finding, though + // it isn't apparent in the standard where this is justified. + + return OpenACCNoCreateClause::Create(Ctx, Clause.getBeginLoc(), + Clause.getLParenLoc(), + Clause.getVarList(), Clause.getEndLoc()); +} - // ActOnVar ensured that everything is a valid variable reference, so there - // really isn't anything to do here. GCC does some duplicate-finding, though - // it isn't apparent in the standard where this is justified. +OpenACCClause *SemaOpenACCClauseVisitor::VisitPresentClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + // ActOnVar ensured that everything is a valid variable reference, so there + // really isn't anything to do here. GCC does some duplicate-finding, though + // it isn't apparent in the standard where this is justified. + + return OpenACCPresentClause::Create(Ctx, Clause.getBeginLoc(), + Clause.getLParenLoc(), + Clause.getVarList(), Clause.getEndLoc()); +} - return OpenACCFirstPrivateClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(), - Clause.getVarList(), Clause.getEndLoc()); - } - case OpenACCClauseKind::NoCreate: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; +OpenACCClause *SemaOpenACCClauseVisitor::VisitCopyClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + // ActOnVar ensured that everything is a valid variable reference, so there + // really isn't anything to do here. GCC does some duplicate-finding, though + // it isn't apparent in the standard where this is justified. + + return OpenACCCopyClause::Create( + Ctx, Clause.getClauseKind(), Clause.getBeginLoc(), Clause.getLParenLoc(), + Clause.getVarList(), Clause.getEndLoc()); +} - // ActOnVar ensured that everything is a valid variable reference, so there - // really isn't anything to do here. GCC does some duplicate-finding, though - // it isn't apparent in the standard where this is justified. +OpenACCClause *SemaOpenACCClauseVisitor::VisitCopyInClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + // ActOnVar ensured that everything is a valid variable reference, so there + // really isn't anything to do here. GCC does some duplicate-finding, though + // it isn't apparent in the standard where this is justified. + + return OpenACCCopyInClause::Create( + Ctx, Clause.getClauseKind(), Clause.getBeginLoc(), Clause.getLParenLoc(), + Clause.isReadOnly(), Clause.getVarList(), Clause.getEndLoc()); +} - return OpenACCNoCreateClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(), - Clause.getVarList(), Clause.getEndLoc()); - } - case OpenACCClauseKind::Present: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; +OpenACCClause *SemaOpenACCClauseVisitor::VisitCopyOutClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + // ActOnVar ensured that everything is a valid variable reference, so there + // really isn't anything to do here. GCC does some duplicate-finding, though + // it isn't apparent in the standard where this is justified. + + return OpenACCCopyOutClause::Create( + Ctx, Clause.getClauseKind(), Clause.getBeginLoc(), Clause.getLParenLoc(), + Clause.isZero(), Clause.getVarList(), Clause.getEndLoc()); +} - // ActOnVar ensured that everything is a valid variable reference, so there - // really isn't anything to do here. GCC does some duplicate-finding, though - // it isn't apparent in the standard where this is justified. +OpenACCClause *SemaOpenACCClauseVisitor::VisitCreateClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + // ActOnVar ensured that everything is a valid variable reference, so there + // really isn't anything to do here. GCC does some duplicate-finding, though + // it isn't apparent in the standard where this is justified. + + return OpenACCCreateClause::Create( + Ctx, Clause.getClauseKind(), Clause.getBeginLoc(), Clause.getLParenLoc(), + Clause.isZero(), Clause.getVarList(), Clause.getEndLoc()); +} - return OpenACCPresentClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(), - Clause.getVarList(), Clause.getEndLoc()); - } - case OpenACCClauseKind::PresentOrCopy: - case OpenACCClauseKind::PCopy: - Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name) - << Clause.getClauseKind() << OpenACCClauseKind::Copy; - LLVM_FALLTHROUGH; - case OpenACCClauseKind::Copy: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; +OpenACCClause *SemaOpenACCClauseVisitor::VisitAttachClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + + // ActOnVar ensured that everything is a valid variable reference, but we + // still have to make sure it is a pointer type. + llvm::SmallVector VarList{Clause.getVarList().begin(), + Clause.getVarList().end()}; + VarList.erase(std::remove_if(VarList.begin(), VarList.end(), + [&](Expr *E) { + return SemaRef.CheckVarIsPointerType( + OpenACCClauseKind::Attach, E); + }), + VarList.end()); + Clause.setVarListDetails(VarList, + /*IsReadOnly=*/false, /*IsZero=*/false); + return OpenACCAttachClause::Create(Ctx, Clause.getBeginLoc(), + Clause.getLParenLoc(), Clause.getVarList(), + Clause.getEndLoc()); +} - // ActOnVar ensured that everything is a valid variable reference, so there - // really isn't anything to do here. GCC does some duplicate-finding, though - // it isn't apparent in the standard where this is justified. +OpenACCClause *SemaOpenACCClauseVisitor::VisitDevicePtrClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + + // ActOnVar ensured that everything is a valid variable reference, but we + // still have to make sure it is a pointer type. + llvm::SmallVector VarList{Clause.getVarList().begin(), + Clause.getVarList().end()}; + VarList.erase(std::remove_if(VarList.begin(), VarList.end(), + [&](Expr *E) { + return SemaRef.CheckVarIsPointerType( + OpenACCClauseKind::DevicePtr, E); + }), + VarList.end()); + Clause.setVarListDetails(VarList, + /*IsReadOnly=*/false, /*IsZero=*/false); + + return OpenACCDevicePtrClause::Create( + Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getVarList(), + Clause.getEndLoc()); +} - return OpenACCCopyClause::Create( - getASTContext(), Clause.getClauseKind(), Clause.getBeginLoc(), - Clause.getLParenLoc(), Clause.getVarList(), Clause.getEndLoc()); - } - case OpenACCClauseKind::PresentOrCopyIn: - case OpenACCClauseKind::PCopyIn: - Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name) - << Clause.getClauseKind() << OpenACCClauseKind::CopyIn; - LLVM_FALLTHROUGH; - case OpenACCClauseKind::CopyIn: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; +OpenACCClause *SemaOpenACCClauseVisitor::VisitWaitClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + + return OpenACCWaitClause::Create( + Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getDevNumExpr(), + Clause.getQueuesLoc(), Clause.getQueueIdExprs(), Clause.getEndLoc()); +} - // ActOnVar ensured that everything is a valid variable reference, so there - // really isn't anything to do here. GCC does some duplicate-finding, though - // it isn't apparent in the standard where this is justified. +OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceTypeClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' and 'loop' + // constructs, and 'compute'/'loop' constructs are the only construct that + // can do anything with this yet, so skip/treat as unimplemented in this + // case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) && + Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop) + return isNotImplemented(); + + // TODO OpenACC: Once we get enough of the CodeGen implemented that we have + // a source for the list of valid architectures, we need to warn on unknown + // identifiers here. + + return OpenACCDeviceTypeClause::Create( + Ctx, Clause.getClauseKind(), Clause.getBeginLoc(), Clause.getLParenLoc(), + Clause.getDeviceTypeArchitectures(), Clause.getEndLoc()); +} - return OpenACCCopyInClause::Create( - getASTContext(), Clause.getClauseKind(), Clause.getBeginLoc(), - Clause.getLParenLoc(), Clause.isReadOnly(), Clause.getVarList(), - Clause.getEndLoc()); +OpenACCClause *SemaOpenACCClauseVisitor::VisitAutoClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'loop' constructs, and it is + // the only construct that can do anything with this, so skip/treat as + // unimplemented for the combined constructs. + if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop) + return isNotImplemented(); + + // OpenACC 3.3 2.9: + // Only one of the seq, independent, and auto clauses may appear. + const auto *Itr = + llvm::find_if(ExistingClauses, + llvm::IsaPred); + if (Itr != ExistingClauses.end()) { + SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict) + << Clause.getClauseKind() << Clause.getDirectiveKind(); + SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); + return nullptr; } - case OpenACCClauseKind::PresentOrCopyOut: - case OpenACCClauseKind::PCopyOut: - Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name) - << Clause.getClauseKind() << OpenACCClauseKind::CopyOut; - LLVM_FALLTHROUGH; - case OpenACCClauseKind::CopyOut: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; - // ActOnVar ensured that everything is a valid variable reference, so there - // really isn't anything to do here. GCC does some duplicate-finding, though - // it isn't apparent in the standard where this is justified. + return OpenACCAutoClause::Create(Ctx, Clause.getBeginLoc(), + Clause.getEndLoc()); +} - return OpenACCCopyOutClause::Create( - getASTContext(), Clause.getClauseKind(), Clause.getBeginLoc(), - Clause.getLParenLoc(), Clause.isZero(), Clause.getVarList(), - Clause.getEndLoc()); +OpenACCClause *SemaOpenACCClauseVisitor::VisitIndependentClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'loop' constructs, and it is + // the only construct that can do anything with this, so skip/treat as + // unimplemented for the combined constructs. + if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop) + return isNotImplemented(); + + // OpenACC 3.3 2.9: + // Only one of the seq, independent, and auto clauses may appear. + const auto *Itr = llvm::find_if( + ExistingClauses, llvm::IsaPred); + if (Itr != ExistingClauses.end()) { + SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict) + << Clause.getClauseKind() << Clause.getDirectiveKind(); + SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); + return nullptr; } - case OpenACCClauseKind::PresentOrCreate: - case OpenACCClauseKind::PCreate: - Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name) - << Clause.getClauseKind() << OpenACCClauseKind::Create; - LLVM_FALLTHROUGH; - case OpenACCClauseKind::Create: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; - - // ActOnVar ensured that everything is a valid variable reference, so there - // really isn't anything to do here. GCC does some duplicate-finding, though - // it isn't apparent in the standard where this is justified. - return OpenACCCreateClause::Create(getASTContext(), Clause.getClauseKind(), - Clause.getBeginLoc(), - Clause.getLParenLoc(), Clause.isZero(), - Clause.getVarList(), Clause.getEndLoc()); - } - case OpenACCClauseKind::Attach: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; + return OpenACCIndependentClause::Create(Ctx, Clause.getBeginLoc(), + Clause.getEndLoc()); +} - // ActOnVar ensured that everything is a valid variable reference, but we - // still have to make sure it is a pointer type. - llvm::SmallVector VarList{Clause.getVarList().begin(), - Clause.getVarList().end()}; - VarList.erase(std::remove_if(VarList.begin(), VarList.end(), [&](Expr *E) { - return CheckVarIsPointerType(OpenACCClauseKind::Attach, E); - }), VarList.end()); - Clause.setVarListDetails(VarList, - /*IsReadOnly=*/false, /*IsZero=*/false); - - return OpenACCAttachClause::Create(getASTContext(), Clause.getBeginLoc(), - Clause.getLParenLoc(), - Clause.getVarList(), Clause.getEndLoc()); +OpenACCClause *SemaOpenACCClauseVisitor::VisitSeqClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'loop' constructs, and it is + // the only construct that can do anything with this, so skip/treat as + // unimplemented for the combined constructs. + if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop) + return isNotImplemented(); + + // OpenACC 3.3 2.9: + // Only one of the seq, independent, and auto clauses may appear. + const auto *Itr = + llvm::find_if(ExistingClauses, + llvm::IsaPred); + if (Itr != ExistingClauses.end()) { + SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict) + << Clause.getClauseKind() << Clause.getDirectiveKind(); + SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); + return nullptr; } - case OpenACCClauseKind::DevicePtr: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; - // ActOnVar ensured that everything is a valid variable reference, but we - // still have to make sure it is a pointer type. - llvm::SmallVector VarList{Clause.getVarList().begin(), - Clause.getVarList().end()}; - VarList.erase(std::remove_if(VarList.begin(), VarList.end(), [&](Expr *E) { - return CheckVarIsPointerType(OpenACCClauseKind::DevicePtr, E); - }), VarList.end()); - Clause.setVarListDetails(VarList, - /*IsReadOnly=*/false, /*IsZero=*/false); - - return OpenACCDevicePtrClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(), - Clause.getVarList(), Clause.getEndLoc()); - } - case OpenACCClauseKind::Wait: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; + // OpenACC 3.3 2.9: + // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause + // appears. + Itr = llvm::find_if(ExistingClauses, + llvm::IsaPred); - return OpenACCWaitClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(), - Clause.getDevNumExpr(), Clause.getQueuesLoc(), Clause.getQueueIdExprs(), - Clause.getEndLoc()); + if (Itr != ExistingClauses.end()) { + SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine) + << Clause.getClauseKind() << (*Itr)->getClauseKind(); + SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); + return nullptr; } - case OpenACCClauseKind::DType: - case OpenACCClauseKind::DeviceType: { - // Restrictions only properly implemented on 'compute' and 'loop' - // constructs, and 'compute'/'loop' constructs are the only construct that - // can do anything with this yet, so skip/treat as unimplemented in this - // case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()) && - Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop) - break; - // TODO OpenACC: Once we get enough of the CodeGen implemented that we have - // a source for the list of valid architectures, we need to warn on unknown - // identifiers here. - - return OpenACCDeviceTypeClause::Create( - getASTContext(), Clause.getClauseKind(), Clause.getBeginLoc(), - Clause.getLParenLoc(), Clause.getDeviceTypeArchitectures(), - Clause.getEndLoc()); - } - case OpenACCClauseKind::Auto: { - // Restrictions only properly implemented on 'loop' constructs, and it is - // the only construct that can do anything with this, so skip/treat as - // unimplemented for the combined constructs. - if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop) - break; + // TODO OpenACC: 2.9 ~ line 2010 specifies that the associated loop has some + // restrictions when there is a 'seq' clause in place. We probably need to + // implement that. + return OpenACCSeqClause::Create(Ctx, Clause.getBeginLoc(), + Clause.getEndLoc()); +} - // OpenACC 3.3 2.9: - // Only one of the seq, independent, and auto clauses may appear. - const auto *Itr = llvm::find_if( - ExistingClauses, - llvm::IsaPred); - if (Itr != ExistingClauses.end()) { - Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict) - << Clause.getClauseKind() << Clause.getDirectiveKind(); - Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); +OpenACCClause *SemaOpenACCClauseVisitor::VisitReductionClause( + SemaOpenACC::OpenACCParsedClause &Clause) { + // Restrictions only properly implemented on 'compute' constructs, and + // 'compute' constructs are the only construct that can do anything with + // this yet, so skip/treat as unimplemented in this case. + if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) + return isNotImplemented(); + + // OpenACC 3.3 Section 2.5.4: + // A reduction clause may not appear on a parallel construct with a + // num_gangs clause that has more than one argument. + if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel) { + auto NumGangsClauses = llvm::make_filter_range( + ExistingClauses, llvm::IsaPred); + + for (auto *NGC : NumGangsClauses) { + unsigned NumExprs = + cast(NGC)->getIntExprs().size(); + + if (NumExprs > 1) { + SemaRef.Diag(Clause.getBeginLoc(), + diag::err_acc_reduction_num_gangs_conflict) + << NumExprs; + SemaRef.Diag(NGC->getBeginLoc(), diag::note_acc_previous_clause_here); + return nullptr; + } } - - return OpenACCAutoClause::Create(getASTContext(), Clause.getBeginLoc(), - Clause.getEndLoc()); } - case OpenACCClauseKind::Independent: { - // Restrictions only properly implemented on 'loop' constructs, and it is - // the only construct that can do anything with this, so skip/treat as - // unimplemented for the combined constructs. - if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop) - break; - // OpenACC 3.3 2.9: - // Only one of the seq, independent, and auto clauses may appear. - const auto *Itr = llvm::find_if( - ExistingClauses, llvm::IsaPred); - if (Itr != ExistingClauses.end()) { - Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict) - << Clause.getClauseKind() << Clause.getDirectiveKind(); - Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); - } + SmallVector ValidVars; + + for (Expr *Var : Clause.getVarList()) { + ExprResult Res = SemaRef.CheckReductionVar(Var); - return OpenACCIndependentClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getEndLoc()); + if (Res.isUsable()) + ValidVars.push_back(Res.get()); } - case OpenACCClauseKind::Seq: { - // Restrictions only properly implemented on 'loop' constructs, and it is - // the only construct that can do anything with this, so skip/treat as - // unimplemented for the combined constructs. - if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop) - break; - // OpenACC 3.3 2.9: - // Only one of the seq, independent, and auto clauses may appear. - const auto *Itr = llvm::find_if( - ExistingClauses, - llvm::IsaPred); - if (Itr != ExistingClauses.end()) { - Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict) - << Clause.getClauseKind() << Clause.getDirectiveKind(); - Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); - } + return OpenACCReductionClause::Create( + Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getReductionOp(), + ValidVars, Clause.getEndLoc()); +} - // OpenACC 3.3 2.9: - // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause - // appears. - Itr = llvm::find_if(ExistingClauses, - llvm::IsaPred); +} // namespace - if (Itr != ExistingClauses.end()) { - Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine) - << Clause.getClauseKind() << (*Itr)->getClauseKind(); - Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); - } +SemaOpenACC::SemaOpenACC(Sema &S) : SemaBase(S) {} - // TODO OpenACC: 2.9 ~ line 2010 specifies that the associated loop has some - // restrictions when there is a 'seq' clause in place. We probably need to - // implement that. - return OpenACCSeqClause::Create(getASTContext(), Clause.getBeginLoc(), - Clause.getEndLoc()); +SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(SemaOpenACC &S, + OpenACCDirectiveKind DK) + : SemaRef(S), WasInsideComputeConstruct(S.InsideComputeConstruct), + DirKind(DK) { + // Compute constructs end up taking their 'loop'. + if (DirKind == OpenACCDirectiveKind::Parallel || + DirKind == OpenACCDirectiveKind::Serial || + DirKind == OpenACCDirectiveKind::Kernels) { + SemaRef.InsideComputeConstruct = true; + SemaRef.ParentlessLoopConstructs.swap(ParentlessLoopConstructs); } - case OpenACCClauseKind::Gang: - case OpenACCClauseKind::Worker: - case OpenACCClauseKind::Vector: { - // OpenACC 3.3 2.9: - // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause - // appears. - const auto *Itr = - llvm::find_if(ExistingClauses, llvm::IsaPred); +} - if (Itr != ExistingClauses.end()) { - Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine) - << Clause.getClauseKind() << (*Itr)->getClauseKind(); - Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); - } - // Not yet implemented, so immediately drop to the 'not yet implemented' - // diagnostic. - break; +SemaOpenACC::AssociatedStmtRAII::~AssociatedStmtRAII() { + SemaRef.InsideComputeConstruct = WasInsideComputeConstruct; + if (DirKind == OpenACCDirectiveKind::Parallel || + DirKind == OpenACCDirectiveKind::Serial || + DirKind == OpenACCDirectiveKind::Kernels) { + assert(SemaRef.ParentlessLoopConstructs.empty() && + "Didn't consume loop construct list?"); + SemaRef.ParentlessLoopConstructs.swap(ParentlessLoopConstructs); } - case OpenACCClauseKind::Reduction: { - // Restrictions only properly implemented on 'compute' constructs, and - // 'compute' constructs are the only construct that can do anything with - // this yet, so skip/treat as unimplemented in this case. - if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind())) - break; - - // OpenACC 3.3 Section 2.5.4: - // A reduction clause may not appear on a parallel construct with a - // num_gangs clause that has more than one argument. - if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Parallel) { - auto NumGangsClauses = llvm::make_filter_range( - ExistingClauses, llvm::IsaPred); - - for (auto *NGC : NumGangsClauses) { - unsigned NumExprs = - cast(NGC)->getIntExprs().size(); - - if (NumExprs > 1) { - Diag(Clause.getBeginLoc(), diag::err_acc_reduction_num_gangs_conflict) - << NumExprs; - Diag(NGC->getBeginLoc(), diag::note_acc_previous_clause_here); - return nullptr; - } - } - } - - SmallVector ValidVars; - - for (Expr *Var : Clause.getVarList()) { - ExprResult Res = CheckReductionVar(Var); +} - if (Res.isUsable()) - ValidVars.push_back(Res.get()); - } +OpenACCClause * +SemaOpenACC::ActOnClause(ArrayRef ExistingClauses, + OpenACCParsedClause &Clause) { + if (Clause.getClauseKind() == OpenACCClauseKind::Invalid) + return nullptr; - return OpenACCReductionClause::Create( - getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(), - Clause.getReductionOp(), ValidVars, Clause.getEndLoc()); + // Diagnose that we don't support this clause on this directive. + if (!doesClauseApplyToDirective(Clause.getDirectiveKind(), + Clause.getClauseKind())) { + Diag(Clause.getBeginLoc(), diag::err_acc_clause_appertainment) + << Clause.getDirectiveKind() << Clause.getClauseKind(); + return nullptr; } - default: - break; + + if (const auto *DevTypeClause = + llvm::find_if(ExistingClauses, + [&](const OpenACCClause *C) { + return isa(C); + }); + DevTypeClause != ExistingClauses.end()) { + if (checkValidAfterDeviceType( + *this, *cast(*DevTypeClause), Clause)) + return nullptr; } - Diag(Clause.getBeginLoc(), diag::warn_acc_clause_unimplemented) - << Clause.getClauseKind(); - return nullptr; + SemaOpenACCClauseVisitor Visitor{*this, ExistingClauses}; + OpenACCClause *Result = Visitor.Visit(Clause); + assert((!Result || Result->getClauseKind() == Clause.getClauseKind()) && + "Created wrong clause?"); + + if (Visitor.diagNotImplemented()) + Diag(Clause.getBeginLoc(), diag::warn_acc_clause_unimplemented) + << Clause.getClauseKind(); + + return Result; + + // switch (Clause.getClauseKind()) { + // case OpenACCClauseKind::PresentOrCopy: + // case OpenACCClauseKind::PCopy: + // Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name) + // << Clause.getClauseKind() << OpenACCClauseKind::Copy; + // LLVM_FALLTHROUGH; + // case OpenACCClauseKind::PresentOrCreate: + // case OpenACCClauseKind::PCreate: + // Diag(Clause.getBeginLoc(), diag::warn_acc_deprecated_alias_name) + // << Clause.getClauseKind() << OpenACCClauseKind::Create; + // LLVM_FALLTHROUGH; + // + // + // + // + // case OpenACCClauseKind::DType: + // + // + // + // + // + // + // + // + // case OpenACCClauseKind::Gang: + // case OpenACCClauseKind::Worker: + // case OpenACCClauseKind::Vector: { + // // OpenACC 3.3 2.9: + // // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' + // clause + // // appears. + // const auto *Itr = + // llvm::find_if(ExistingClauses, llvm::IsaPred); + // + // if (Itr != ExistingClauses.end()) { + // Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine) + // << Clause.getClauseKind() << (*Itr)->getClauseKind(); + // Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here); + // } + // // Not yet implemented, so immediately drop to the 'not yet implemented' + // // diagnostic. + // break; + // } + // */ + } /// OpenACC 3.3 section 2.5.15: -- GitLab From d99cf1b916ddcfad60c706840ddfd5d2f1409f2a Mon Sep 17 00:00:00 2001 From: jeanPerier Date: Wed, 5 Jun 2024 23:16:20 +0200 Subject: [PATCH 014/462] [flang] lower assumed-rank inquiries - part 1 (#94475) Lower assumed-rank inquiries, except SIZE/SHAPE/UBOUND/LBOUND. This is mainly about adding tests, and removing a couple TODOs in helpers. --- flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 4 - flang/lib/Optimizer/Builder/MutableBox.cpp | 6 +- .../Lower/HLFIR/assumed-rank-inquiries.f90 | 383 ++++++++++++++++++ 3 files changed, 386 insertions(+), 7 deletions(-) create mode 100644 flang/test/Lower/HLFIR/assumed-rank-inquiries.f90 diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp index 1cd3976d0afb..072899751bc8 100644 --- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -4965,10 +4965,6 @@ fir::ExtendedValue IntrinsicLibrary::genIsContiguous(mlir::Type resultType, llvm::ArrayRef args) { assert(args.size() == 1); - if (const auto *boxValue = args[0].getBoxOf()) - if (boxValue->hasAssumedRank()) - TODO(loc, "intrinsic: is_contiguous with assumed rank argument"); - return builder.createConvert( loc, resultType, fir::runtime::genIsContiguous(builder, loc, fir::getBase(args[0]))); diff --git a/flang/lib/Optimizer/Builder/MutableBox.cpp b/flang/lib/Optimizer/Builder/MutableBox.cpp index 76b920dba869..16e543fe86a7 100644 --- a/flang/lib/Optimizer/Builder/MutableBox.cpp +++ b/flang/lib/Optimizer/Builder/MutableBox.cpp @@ -394,6 +394,8 @@ static bool readToBoxValue(const fir::MutableBoxValue &box, // Track value as fir.box if ((box.isDerived() && mayBePolymorphic) || box.isUnlimitedPolymorphic()) return true; + if (box.hasAssumedRank()) + return true; // Intrinsic allocatables are contiguous, no need to track the value by // fir.box. if (box.isAllocatable() || box.rank() == 0) @@ -409,14 +411,12 @@ fir::factory::genMutableBoxRead(fir::FirOpBuilder &builder, mlir::Location loc, const fir::MutableBoxValue &box, bool mayBePolymorphic, bool preserveLowerBounds) { - if (box.hasAssumedRank()) - TODO(loc, "assumed rank allocatables or pointers"); llvm::SmallVector lbounds; llvm::SmallVector extents; llvm::SmallVector lengths; if (readToBoxValue(box, mayBePolymorphic)) { auto reader = MutablePropertyReader(builder, loc, box); - if (preserveLowerBounds) + if (preserveLowerBounds && !box.hasAssumedRank()) reader.getLowerBounds(lbounds); return fir::BoxValue{reader.getIrBox(), lbounds, box.nonDeferredLenParams()}; diff --git a/flang/test/Lower/HLFIR/assumed-rank-inquiries.f90 b/flang/test/Lower/HLFIR/assumed-rank-inquiries.f90 new file mode 100644 index 000000000000..e8610aa7d0e7 --- /dev/null +++ b/flang/test/Lower/HLFIR/assumed-rank-inquiries.f90 @@ -0,0 +1,383 @@ +! Test lowering of inquiry intrinsics with assumed-ranks arguments. +! RUN: bbc -emit-hlfir -o - %s -allow-assumed-rank | FileCheck %s + +subroutine test_allocated(x) + real, allocatable :: x(..) + call takes_logical(allocated(x)) +end subroutine + +subroutine test_associated_1(x) + real, pointer :: x(..) + call takes_logical(associated(x)) +end subroutine + +subroutine test_associated_2(x, y) + real, pointer :: x(..) + real, target :: y(:) + call takes_logical(associated(x, y)) +end subroutine + +subroutine test_associated_3(x, y) + real, pointer :: x(..) + real, pointer :: y(..) + call takes_logical(associated(x, y)) +end subroutine + +subroutine test_len_1(x) + character(*) :: x(..) + call takes_integer(len(x)) +end subroutine + +subroutine test_len_2(x) + character(*), pointer :: x(..) + call takes_integer(len(x)) +end subroutine + +subroutine test_storage_size_1(x) + class(*) :: x(..) + call takes_integer(storage_size(x)) +end subroutine + +subroutine test_storage_size_2(x) + class(*), pointer :: x(..) + call takes_integer(storage_size(x)) +end subroutine + +subroutine test_present_1(x) + class(*), optional :: x(..) + call takes_logical(present(x)) +end subroutine + +subroutine test_present_2(x) + class(*), optional, pointer :: x(..) + call takes_logical(present(x)) +end subroutine + +subroutine test_is_contiguous_1(x) + class(*) :: x(..) + call takes_logical(is_contiguous(x)) +end subroutine + +subroutine test_is_contiguous_2(x) + class(*), pointer :: x(..) + call takes_logical(is_contiguous(x)) +end subroutine + +subroutine test_same_type_as_1(x, y) + class(*) :: x(..), y(..) + call takes_logical(same_type_as(x, y)) +end subroutine + +subroutine test_same_type_as_2(x, y) + class(*), pointer :: x(..), y(..) + call takes_logical(same_type_as(x, y)) +end subroutine + +subroutine test_extends_type_of_1(x, y) + class(*) :: x(..), y(..) + call takes_logical(extends_type_of(x, y)) +end subroutine + +subroutine test_extends_type_of_2(x, y) + class(*), pointer :: x(..), y(..) + call takes_logical(extends_type_of(x, y)) +end subroutine + +subroutine c_loc_1(x) + use iso_c_binding, only : c_loc + real, target :: x(..) + call takes_cloc(c_loc(x)) +end subroutine + +subroutine c_loc_2(x) + use iso_c_binding, only : c_loc + real, pointer :: x(..) + call takes_cloc(c_loc(x)) +end subroutine + +! CHECK-LABEL: func.func @_QPtest_allocated( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_allocatedEx"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]]#1 : !fir.ref>>> +! CHECK: %[[VAL_4:.*]] = fir.box_addr %[[VAL_3]] : (!fir.box>>) -> !fir.heap> +! CHECK: %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.heap>) -> i64 +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i64 +! CHECK: %[[VAL_7:.*]] = arith.cmpi ne, %[[VAL_5]], %[[VAL_6]] : i64 +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i1) -> !fir.logical<4> +! CHECK: %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_logical(%[[VAL_9]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_9]]#1, %[[VAL_9]]#2 : !fir.ref>, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_associated_1( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_associated_1Ex"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]]#1 : !fir.ref>>> +! CHECK: %[[VAL_4:.*]] = fir.box_addr %[[VAL_3]] : (!fir.box>>) -> !fir.ptr> +! CHECK: %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.ptr>) -> i64 +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i64 +! CHECK: %[[VAL_7:.*]] = arith.cmpi ne, %[[VAL_5]], %[[VAL_6]] : i64 +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i1) -> !fir.logical<4> +! CHECK: %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_logical(%[[VAL_9]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_9]]#1, %[[VAL_9]]#2 : !fir.ref>, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_associated_2( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.bindc_name = "x"}, +! CHECK-SAME: %[[VAL_1:.*]]: !fir.box> {fir.bindc_name = "y", fir.target}) { +! CHECK: %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_associated_2Ex"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_associated_2Ey"} : (!fir.box>, !fir.dscope) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_5:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref>>> +! CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (!fir.box>>) -> !fir.box +! CHECK: %[[VAL_7:.*]] = fir.convert %[[VAL_4]]#1 : (!fir.box>) -> !fir.box +! CHECK: %[[VAL_8:.*]] = fir.call @_FortranAPointerIsAssociatedWith(%[[VAL_6]], %[[VAL_7]]) fastmath : (!fir.box, !fir.box) -> i1 +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4> +! CHECK: %[[VAL_10:.*]]:3 = hlfir.associate %[[VAL_9]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_logical(%[[VAL_10]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_10]]#1, %[[VAL_10]]#2 : !fir.ref>, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_associated_3( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.bindc_name = "x"}, +! CHECK-SAME: %[[VAL_1:.*]]: !fir.ref>>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_associated_3Ex"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_associated_3Ey"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_5:.*]] = fir.load %[[VAL_4]]#1 : !fir.ref>>> +! CHECK: %[[VAL_6:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref>>> +! CHECK: %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (!fir.box>>) -> !fir.box +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_5]] : (!fir.box>>) -> !fir.box +! CHECK: %[[VAL_9:.*]] = fir.call @_FortranAPointerIsAssociatedWith(%[[VAL_7]], %[[VAL_8]]) fastmath : (!fir.box, !fir.box) -> i1 +! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i1) -> !fir.logical<4> +! CHECK: %[[VAL_11:.*]]:3 = hlfir.associate %[[VAL_10]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_logical(%[[VAL_11]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_11]]#1, %[[VAL_11]]#2 : !fir.ref>, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_len_1( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box>> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFtest_len_1Ex"} : (!fir.box>>, !fir.dscope) -> (!fir.box>>, !fir.box>>) +! CHECK: %[[VAL_3:.*]] = fir.box_elesize %[[VAL_2]]#1 : (!fir.box>>) -> index +! CHECK: %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (index) -> i32 +! CHECK: %[[VAL_5:.*]]:3 = hlfir.associate %[[VAL_4]] {adapt.valuebyref} : (i32) -> (!fir.ref, !fir.ref, i1) +! CHECK: fir.call @_QPtakes_integer(%[[VAL_5]]#1) fastmath : (!fir.ref) -> () +! CHECK: hlfir.end_associate %[[VAL_5]]#1, %[[VAL_5]]#2 : !fir.ref, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_len_2( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>>> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref>>>> +! CHECK: %[[VAL_3:.*]] = fir.box_elesize %[[VAL_2]] : (!fir.box>>>) -> index +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_3]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_len_2Ex"} : (!fir.ref>>>>, index, !fir.dscope) -> (!fir.ref>>>>, !fir.ref>>>>) +! CHECK: %[[VAL_5:.*]] = fir.convert %[[VAL_3]] : (index) -> i32 +! CHECK: %[[VAL_6:.*]]:3 = hlfir.associate %[[VAL_5]] {adapt.valuebyref} : (i32) -> (!fir.ref, !fir.ref, i1) +! CHECK: fir.call @_QPtakes_integer(%[[VAL_6]]#1) fastmath : (!fir.ref) -> () +! CHECK: hlfir.end_associate %[[VAL_6]]#1, %[[VAL_6]]#2 : !fir.ref, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_storage_size_1( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.class> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFtest_storage_size_1Ex"} : (!fir.class>, !fir.dscope) -> (!fir.class>, !fir.class>) +! CHECK: %[[VAL_3:.*]] = fir.box_elesize %[[VAL_2]]#1 : (!fir.class>) -> i32 +! CHECK: %[[VAL_4:.*]] = arith.constant 8 : i32 +! CHECK: %[[VAL_5:.*]] = arith.muli %[[VAL_3]], %[[VAL_4]] : i32 +! CHECK: %[[VAL_6:.*]]:3 = hlfir.associate %[[VAL_5]] {adapt.valuebyref} : (i32) -> (!fir.ref, !fir.ref, i1) +! CHECK: fir.call @_QPtakes_integer(%[[VAL_6]]#1) fastmath : (!fir.ref) -> () +! CHECK: hlfir.end_associate %[[VAL_6]]#1, %[[VAL_6]]#2 : !fir.ref, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_storage_size_2( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_storage_size_2Ex"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]]#1 : !fir.ref>>> +! CHECK: %[[VAL_4:.*]] = fir.box_addr %[[VAL_3]] : (!fir.class>>) -> !fir.ptr> +! CHECK: %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.ptr>) -> i64 +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i64 +! CHECK: %[[VAL_7:.*]] = arith.cmpi eq, %[[VAL_5]], %[[VAL_6]] : i64 +! CHECK: fir.if %[[VAL_7]] { +! CHECK: %[[VAL_13:.*]] = fir.call @_FortranAReportFatalUserError +! CHECK: } +! CHECK: %[[VAL_14:.*]] = fir.load %[[VAL_2]]#1 : !fir.ref>>> +! CHECK: %[[VAL_15:.*]] = fir.box_elesize %[[VAL_14]] : (!fir.class>>) -> i32 +! CHECK: %[[VAL_16:.*]] = arith.constant 8 : i32 +! CHECK: %[[VAL_17:.*]] = arith.muli %[[VAL_15]], %[[VAL_16]] : i32 +! CHECK: %[[VAL_18:.*]]:3 = hlfir.associate %[[VAL_17]] {adapt.valuebyref} : (i32) -> (!fir.ref, !fir.ref, i1) +! CHECK: fir.call @_QPtakes_integer(%[[VAL_18]]#1) fastmath : (!fir.ref) -> () +! CHECK: hlfir.end_associate %[[VAL_18]]#1, %[[VAL_18]]#2 : !fir.ref, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_present_1( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.class> {fir.bindc_name = "x", fir.optional}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_present_1Ex"} : (!fir.class>, !fir.dscope) -> (!fir.class>, !fir.class>) +! CHECK: %[[VAL_3:.*]] = fir.is_present %[[VAL_2]]#1 : (!fir.class>) -> i1 +! CHECK: %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4> +! CHECK: %[[VAL_5:.*]]:3 = hlfir.associate %[[VAL_4]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_logical(%[[VAL_5]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_5]]#1, %[[VAL_5]]#2 : !fir.ref>, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_present_2( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.bindc_name = "x", fir.optional}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_present_2Ex"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_3:.*]] = fir.is_present %[[VAL_2]]#1 : (!fir.ref>>>) -> i1 +! CHECK: %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4> +! CHECK: %[[VAL_5:.*]]:3 = hlfir.associate %[[VAL_4]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_logical(%[[VAL_5]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_5]]#1, %[[VAL_5]]#2 : !fir.ref>, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_is_contiguous_1( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.class> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFtest_is_contiguous_1Ex"} : (!fir.class>, !fir.dscope) -> (!fir.class>, !fir.class>) +! CHECK: %[[VAL_3:.*]] = fir.convert %[[VAL_2]]#1 : (!fir.class>) -> !fir.box +! CHECK: %[[VAL_4:.*]] = fir.call @_FortranAIsContiguous(%[[VAL_3]]) fastmath : (!fir.box) -> i1 +! CHECK: %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (i1) -> !fir.logical<4> +! CHECK: %[[VAL_6:.*]]:3 = hlfir.associate %[[VAL_5]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_logical(%[[VAL_6]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_6]]#1, %[[VAL_6]]#2 : !fir.ref>, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_is_contiguous_2( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_is_contiguous_2Ex"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]]#1 : !fir.ref>>> +! CHECK: %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.class>>) -> !fir.box +! CHECK: %[[VAL_5:.*]] = fir.call @_FortranAIsContiguous(%[[VAL_4]]) fastmath : (!fir.box) -> i1 +! CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i1) -> !fir.logical<4> +! CHECK: %[[VAL_7:.*]]:3 = hlfir.associate %[[VAL_6]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_logical(%[[VAL_7]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_7]]#1, %[[VAL_7]]#2 : !fir.ref>, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_same_type_as_1( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.class> {fir.bindc_name = "x"}, +! CHECK-SAME: %[[VAL_1:.*]]: !fir.class> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {uniq_name = "_QFtest_same_type_as_1Ex"} : (!fir.class>, !fir.dscope) -> (!fir.class>, !fir.class>) +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {uniq_name = "_QFtest_same_type_as_1Ey"} : (!fir.class>, !fir.dscope) -> (!fir.class>, !fir.class>) +! CHECK: %[[VAL_5:.*]] = fir.convert %[[VAL_3]]#1 : (!fir.class>) -> !fir.box +! CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_4]]#1 : (!fir.class>) -> !fir.box +! CHECK: %[[VAL_7:.*]] = fir.call @_FortranASameTypeAs(%[[VAL_5]], %[[VAL_6]]) fastmath : (!fir.box, !fir.box) -> i1 +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i1) -> !fir.logical<4> +! CHECK: %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_logical(%[[VAL_9]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_9]]#1, %[[VAL_9]]#2 : !fir.ref>, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_same_type_as_2( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.bindc_name = "x"}, +! CHECK-SAME: %[[VAL_1:.*]]: !fir.ref>>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_same_type_as_2Ex"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_same_type_as_2Ey"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_5:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref>>> +! CHECK: %[[VAL_6:.*]] = fir.load %[[VAL_4]]#1 : !fir.ref>>> +! CHECK: %[[VAL_7:.*]] = fir.convert %[[VAL_5]] : (!fir.class>>) -> !fir.box +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_6]] : (!fir.class>>) -> !fir.box +! CHECK: %[[VAL_9:.*]] = fir.call @_FortranASameTypeAs(%[[VAL_7]], %[[VAL_8]]) fastmath : (!fir.box, !fir.box) -> i1 +! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i1) -> !fir.logical<4> +! CHECK: %[[VAL_11:.*]]:3 = hlfir.associate %[[VAL_10]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_logical(%[[VAL_11]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_11]]#1, %[[VAL_11]]#2 : !fir.ref>, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_extends_type_of_1( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.class> {fir.bindc_name = "x"}, +! CHECK-SAME: %[[VAL_1:.*]]: !fir.class> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {uniq_name = "_QFtest_extends_type_of_1Ex"} : (!fir.class>, !fir.dscope) -> (!fir.class>, !fir.class>) +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {uniq_name = "_QFtest_extends_type_of_1Ey"} : (!fir.class>, !fir.dscope) -> (!fir.class>, !fir.class>) +! CHECK: %[[VAL_5:.*]] = fir.convert %[[VAL_3]]#1 : (!fir.class>) -> !fir.box +! CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_4]]#1 : (!fir.class>) -> !fir.box +! CHECK: %[[VAL_7:.*]] = fir.call @_FortranAExtendsTypeOf(%[[VAL_5]], %[[VAL_6]]) fastmath : (!fir.box, !fir.box) -> i1 +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i1) -> !fir.logical<4> +! CHECK: %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_logical(%[[VAL_9]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_9]]#1, %[[VAL_9]]#2 : !fir.ref>, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_extends_type_of_2( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.bindc_name = "x"}, +! CHECK-SAME: %[[VAL_1:.*]]: !fir.ref>>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_extends_type_of_2Ex"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_extends_type_of_2Ey"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_5:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref>>> +! CHECK: %[[VAL_6:.*]] = fir.load %[[VAL_4]]#1 : !fir.ref>>> +! CHECK: %[[VAL_7:.*]] = fir.convert %[[VAL_5]] : (!fir.class>>) -> !fir.box +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_6]] : (!fir.class>>) -> !fir.box +! CHECK: %[[VAL_9:.*]] = fir.call @_FortranAExtendsTypeOf(%[[VAL_7]], %[[VAL_8]]) fastmath : (!fir.box, !fir.box) -> i1 +! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i1) -> !fir.logical<4> +! CHECK: %[[VAL_11:.*]]:3 = hlfir.associate %[[VAL_10]] {adapt.valuebyref} : (!fir.logical<4>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_logical(%[[VAL_11]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_11]]#1, %[[VAL_11]]#2 : !fir.ref>, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPc_loc_1( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "x", fir.target}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFc_loc_1Ex"} : (!fir.box>, !fir.dscope) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}> +! CHECK: %[[VAL_4:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}> +! CHECK: %[[VAL_5:.*]] = fir.coordinate_of %[[VAL_3]], %[[VAL_4]] : (!fir.ref>, !fir.field) -> !fir.ref +! CHECK: %[[VAL_6:.*]] = fir.box_addr %[[VAL_2]]#1 : (!fir.box>) -> !fir.ref> +! CHECK: %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (!fir.ref>) -> i64 +! CHECK: fir.store %[[VAL_7]] to %[[VAL_5]] : !fir.ref +! CHECK: %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = ".tmp.intrinsic_result"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_9:.*]] = arith.constant false +! CHECK: %[[VAL_10:.*]] = hlfir.as_expr %[[VAL_8]]#0 move %[[VAL_9]] : (!fir.ref>, i1) -> !hlfir.expr> +! CHECK: %[[VAL_11:.*]]:3 = hlfir.associate %[[VAL_10]] {adapt.valuebyref} : (!hlfir.expr>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_cloc(%[[VAL_11]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_11]]#1, %[[VAL_11]]#2 : !fir.ref>, i1 +! CHECK: hlfir.destroy %[[VAL_10]] : !hlfir.expr> +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPc_loc_2( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFc_loc_2Ex"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]]#1 : !fir.ref>>> +! CHECK: %[[VAL_4:.*]] = fir.alloca !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}> +! CHECK: %[[VAL_5:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}> +! CHECK: %[[VAL_6:.*]] = fir.coordinate_of %[[VAL_4]], %[[VAL_5]] : (!fir.ref>, !fir.field) -> !fir.ref +! CHECK: %[[VAL_7:.*]] = fir.box_addr %[[VAL_3]] : (!fir.box>>) -> !fir.ptr> +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (!fir.ptr>) -> i64 +! CHECK: fir.store %[[VAL_8]] to %[[VAL_6]] : !fir.ref +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = ".tmp.intrinsic_result"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_10:.*]] = arith.constant false +! CHECK: %[[VAL_11:.*]] = hlfir.as_expr %[[VAL_9]]#0 move %[[VAL_10]] : (!fir.ref>, i1) -> !hlfir.expr> +! CHECK: %[[VAL_12:.*]]:3 = hlfir.associate %[[VAL_11]] {adapt.valuebyref} : (!hlfir.expr>) -> (!fir.ref>, !fir.ref>, i1) +! CHECK: fir.call @_QPtakes_cloc(%[[VAL_12]]#1) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.end_associate %[[VAL_12]]#1, %[[VAL_12]]#2 : !fir.ref>, i1 +! CHECK: hlfir.destroy %[[VAL_11]] : !hlfir.expr> +! CHECK: return +! CHECK: } -- GitLab From c599894aa63747f338544eaa3038eb9f132a5bbd Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 5 Jun 2024 16:20:15 -0500 Subject: [PATCH 015/462] [RISCV] Remove policy and merge operand from unmasked vmsbf/vmsif/vmsof.m. (#94530) These instructions always update the destination under a tail agnostic policy. --- .../Target/RISCV/RISCVInstrInfoVPseudos.td | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index 5fe5a7a5bd5c..5db7cb4dc618 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -1033,6 +1033,22 @@ class VPseudoUnaryNoMask : + Pseudo<(outs RetClass:$rd), + (ins OpClass:$rs2, AVL:$vl, ixlenimm:$sew), []>, + RISCVVPseudo { + let mayLoad = 0; + let mayStore = 0; + let hasSideEffects = 0; + let Constraints = Constraint; + let TargetOverlapConstraintType = TargetConstraintType; + let HasVLOp = 1; + let HasSEWOp = 1; +} + class VPseudoUnaryNoMaskRoundingMode, + def "_M_" # mti.BX : VPseudoUnaryNoMaskNoPolicy, SchedUnary<"WriteVMSFSV", "ReadVMSFSV", mx, forceMergeOpRead=true>; def "_M_" # mti.BX # "_MASK" : VPseudoUnaryMask, @@ -4078,9 +4094,8 @@ class VPatMaskUnaryNoMask(inst#"_M_"#mti.BX) - (mti.Mask (IMPLICIT_DEF)), (mti.Mask VR:$rs2), - GPR:$vl, mti.Log2SEW, TA_MA)>; + GPR:$vl, mti.Log2SEW)>; class VPatMaskUnaryMask Date: Wed, 5 Jun 2024 16:20:38 -0500 Subject: [PATCH 016/462] [RISCV] Use ForceTailAgnostic for masked vmsbf/vmsif/vmsof.m. (#94532) These instructions use the mask policy, but always update the destination under tail agnostic policy. --- llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td | 1 + llvm/test/CodeGen/RISCV/rvv/vmsbf.ll | 16 ++++++++-------- llvm/test/CodeGen/RISCV/rvv/vmsif.ll | 16 ++++++++-------- llvm/test/CodeGen/RISCV/rvv/vmsof.ll | 16 ++++++++-------- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index 5db7cb4dc618..a85a8bb87d11 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -2075,6 +2075,7 @@ multiclass VPseudoVSFS_M { def "_M_" # mti.BX : VPseudoUnaryNoMaskNoPolicy, SchedUnary<"WriteVMSFSV", "ReadVMSFSV", mx, forceMergeOpRead=true>; + let ForceTailAgnostic = true in def "_M_" # mti.BX # "_MASK" : VPseudoUnaryMask, SchedUnary<"WriteVMSFSV", "ReadVMSFSV", mx, forceMergeOpRead=true>; diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsbf.ll b/llvm/test/CodeGen/RISCV/rvv/vmsbf.ll index 14a1f084c398..d1f344d52763 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vmsbf.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vmsbf.ll @@ -33,7 +33,7 @@ define @intrinsic_vmsbf_mask_m_nxv1i1_nxv1i1( ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmv1r.v v10, v0 ; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetvli zero, a0, e8, mf8, tu, mu +; CHECK-NEXT: vsetvli zero, a0, e8, mf8, ta, mu ; CHECK-NEXT: vmsbf.m v10, v8, v0.t ; CHECK-NEXT: vmv1r.v v0, v10 ; CHECK-NEXT: ret @@ -75,7 +75,7 @@ define @intrinsic_vmsbf_mask_m_nxv2i1_nxv2i1( ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmv1r.v v10, v0 ; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetvli zero, a0, e8, mf4, tu, mu +; CHECK-NEXT: vsetvli zero, a0, e8, mf4, ta, mu ; CHECK-NEXT: vmsbf.m v10, v8, v0.t ; CHECK-NEXT: vmv1r.v v0, v10 ; CHECK-NEXT: ret @@ -117,7 +117,7 @@ define @intrinsic_vmsbf_mask_m_nxv4i1_nxv4i1( ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmv1r.v v10, v0 ; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetvli zero, a0, e8, mf2, tu, mu +; CHECK-NEXT: vsetvli zero, a0, e8, mf2, ta, mu ; CHECK-NEXT: vmsbf.m v10, v8, v0.t ; CHECK-NEXT: vmv1r.v v0, v10 ; CHECK-NEXT: ret @@ -159,9 +159,9 @@ define @intrinsic_vmsbf_mask_m_nxv8i1_nxv8i1( ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmv1r.v v10, v0 ; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetvli zero, a0, e8, m1, tu, mu +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, mu ; CHECK-NEXT: vmsbf.m v10, v8, v0.t -; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmv.v.v v0, v10 ; CHECK-NEXT: ret entry: %a = call @llvm.riscv.vmsbf.mask.nxv8i1( @@ -201,7 +201,7 @@ define @intrinsic_vmsbf_mask_m_nxv16i1_nxv16i1( @intrinsic_vmsbf_mask_m_nxv32i1_nxv32i1( @intrinsic_vmsbf_mask_m_nxv64i1_nxv64i1( @intrinsic_vmsif_mask_m_nxv1i1_nxv1i1( ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmv1r.v v10, v0 ; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetvli zero, a0, e8, mf8, tu, mu +; CHECK-NEXT: vsetvli zero, a0, e8, mf8, ta, mu ; CHECK-NEXT: vmsif.m v10, v8, v0.t ; CHECK-NEXT: vmv1r.v v0, v10 ; CHECK-NEXT: ret @@ -75,7 +75,7 @@ define @intrinsic_vmsif_mask_m_nxv2i1_nxv2i1( ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmv1r.v v10, v0 ; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetvli zero, a0, e8, mf4, tu, mu +; CHECK-NEXT: vsetvli zero, a0, e8, mf4, ta, mu ; CHECK-NEXT: vmsif.m v10, v8, v0.t ; CHECK-NEXT: vmv1r.v v0, v10 ; CHECK-NEXT: ret @@ -117,7 +117,7 @@ define @intrinsic_vmsif_mask_m_nxv4i1_nxv4i1( ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmv1r.v v10, v0 ; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetvli zero, a0, e8, mf2, tu, mu +; CHECK-NEXT: vsetvli zero, a0, e8, mf2, ta, mu ; CHECK-NEXT: vmsif.m v10, v8, v0.t ; CHECK-NEXT: vmv1r.v v0, v10 ; CHECK-NEXT: ret @@ -159,9 +159,9 @@ define @intrinsic_vmsif_mask_m_nxv8i1_nxv8i1( ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmv1r.v v10, v0 ; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetvli zero, a0, e8, m1, tu, mu +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, mu ; CHECK-NEXT: vmsif.m v10, v8, v0.t -; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmv.v.v v0, v10 ; CHECK-NEXT: ret entry: %a = call @llvm.riscv.vmsif.mask.nxv8i1( @@ -201,7 +201,7 @@ define @intrinsic_vmsif_mask_m_nxv16i1_nxv16i1( @intrinsic_vmsif_mask_m_nxv32i1_nxv32i1( @intrinsic_vmsif_mask_m_nxv64i1_nxv64i1( @intrinsic_vmsof_mask_m_nxv1i1_nxv1i1( ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmv1r.v v10, v0 ; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetvli zero, a0, e8, mf8, tu, mu +; CHECK-NEXT: vsetvli zero, a0, e8, mf8, ta, mu ; CHECK-NEXT: vmsof.m v10, v8, v0.t ; CHECK-NEXT: vmv1r.v v0, v10 ; CHECK-NEXT: ret @@ -75,7 +75,7 @@ define @intrinsic_vmsof_mask_m_nxv2i1_nxv2i1( ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmv1r.v v10, v0 ; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetvli zero, a0, e8, mf4, tu, mu +; CHECK-NEXT: vsetvli zero, a0, e8, mf4, ta, mu ; CHECK-NEXT: vmsof.m v10, v8, v0.t ; CHECK-NEXT: vmv1r.v v0, v10 ; CHECK-NEXT: ret @@ -117,7 +117,7 @@ define @intrinsic_vmsof_mask_m_nxv4i1_nxv4i1( ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmv1r.v v10, v0 ; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetvli zero, a0, e8, mf2, tu, mu +; CHECK-NEXT: vsetvli zero, a0, e8, mf2, ta, mu ; CHECK-NEXT: vmsof.m v10, v8, v0.t ; CHECK-NEXT: vmv1r.v v0, v10 ; CHECK-NEXT: ret @@ -159,9 +159,9 @@ define @intrinsic_vmsof_mask_m_nxv8i1_nxv8i1( ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmv1r.v v10, v0 ; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetvli zero, a0, e8, m1, tu, mu +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, mu ; CHECK-NEXT: vmsof.m v10, v8, v0.t -; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmv.v.v v0, v10 ; CHECK-NEXT: ret entry: %a = call @llvm.riscv.vmsof.mask.nxv8i1( @@ -201,7 +201,7 @@ define @intrinsic_vmsof_mask_m_nxv16i1_nxv16i1( @intrinsic_vmsof_mask_m_nxv32i1_nxv32i1( @intrinsic_vmsof_mask_m_nxv64i1_nxv64i1( Date: Wed, 5 Jun 2024 15:02:28 -0700 Subject: [PATCH 017/462] [libc] Unpoison epoll structs (#94536) The epoll wait functions return structs via pointer, but those structs need to be unpoisoned before return. This patch adds that unpoisoning. --- libc/src/sys/epoll/linux/CMakeLists.txt | 3 +++ libc/src/sys/epoll/linux/epoll_pwait.cpp | 3 +++ libc/src/sys/epoll/linux/epoll_pwait2.cpp | 3 +++ libc/src/sys/epoll/linux/epoll_wait.cpp | 3 +++ utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 7 +++++-- 5 files changed, 17 insertions(+), 2 deletions(-) diff --git a/libc/src/sys/epoll/linux/CMakeLists.txt b/libc/src/sys/epoll/linux/CMakeLists.txt index 4e661b262b85..5ba89bd1af60 100644 --- a/libc/src/sys/epoll/linux/CMakeLists.txt +++ b/libc/src/sys/epoll/linux/CMakeLists.txt @@ -48,6 +48,7 @@ add_entrypoint_object( libc.hdr.types.struct_timespec libc.include.sys_syscall libc.src.__support.OSUtil.osutil + libc.src.__support.macros.sanitizer libc.src.errno.errno ) @@ -65,6 +66,7 @@ add_entrypoint_object( libc.hdr.signal_macros libc.include.sys_syscall libc.src.__support.OSUtil.osutil + libc.src.__support.macros.sanitizer libc.src.errno.errno ) @@ -82,5 +84,6 @@ add_entrypoint_object( libc.hdr.signal_macros libc.include.sys_syscall libc.src.__support.OSUtil.osutil + libc.src.__support.macros.sanitizer libc.src.errno.errno ) diff --git a/libc/src/sys/epoll/linux/epoll_pwait.cpp b/libc/src/sys/epoll/linux/epoll_pwait.cpp index 8f498d18d547..24b66f0721b3 100644 --- a/libc/src/sys/epoll/linux/epoll_pwait.cpp +++ b/libc/src/sys/epoll/linux/epoll_pwait.cpp @@ -13,6 +13,7 @@ #include "hdr/types/struct_epoll_event.h" #include "src/__support/OSUtil/syscall.h" // For internal syscall function. #include "src/__support/common.h" +#include "src/__support/macros/sanitizer.h" #include "src/errno/libc_errno.h" #include // For syscall numbers. @@ -33,6 +34,8 @@ LLVM_LIBC_FUNCTION(int, epoll_pwait, return -1; } + MSAN_UNPOISON(events, ret * sizeof(struct epoll_event)); + return ret; } diff --git a/libc/src/sys/epoll/linux/epoll_pwait2.cpp b/libc/src/sys/epoll/linux/epoll_pwait2.cpp index bd33cb6325ce..e13423a085a5 100644 --- a/libc/src/sys/epoll/linux/epoll_pwait2.cpp +++ b/libc/src/sys/epoll/linux/epoll_pwait2.cpp @@ -14,6 +14,7 @@ #include "hdr/types/struct_timespec.h" #include "src/__support/OSUtil/syscall.h" // For internal syscall function. #include "src/__support/common.h" +#include "src/__support/macros/sanitizer.h" #include "src/errno/libc_errno.h" #include // For syscall numbers. @@ -35,6 +36,8 @@ LLVM_LIBC_FUNCTION(int, epoll_pwait2, return -1; } + MSAN_UNPOISON(events, ret * sizeof(struct epoll_event)); + return ret; } diff --git a/libc/src/sys/epoll/linux/epoll_wait.cpp b/libc/src/sys/epoll/linux/epoll_wait.cpp index 95238d872d52..3ce4a92e7969 100644 --- a/libc/src/sys/epoll/linux/epoll_wait.cpp +++ b/libc/src/sys/epoll/linux/epoll_wait.cpp @@ -13,6 +13,7 @@ #include "hdr/types/struct_epoll_event.h" #include "src/__support/OSUtil/syscall.h" // For internal syscall function. #include "src/__support/common.h" +#include "src/__support/macros/sanitizer.h" #include "src/errno/libc_errno.h" #include // For syscall numbers. @@ -39,6 +40,8 @@ LLVM_LIBC_FUNCTION(int, epoll_wait, return -1; } + MSAN_UNPOISON(events, ret * sizeof(struct epoll_event)); + return ret; } diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index f3809bd74814..96cc89555931 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -1111,7 +1111,7 @@ libc_support_library( ], defines = [ "LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY", - "LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT" + "LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT", ], target_compatible_with = select({ "@platforms//os:linux": [], @@ -1119,9 +1119,9 @@ libc_support_library( }), deps = [ ":__support_cpp_optional", - ":__support_time_linux", ":__support_threads_linux_futex_utils", ":__support_threads_sleep", + ":__support_time_linux", ":types_pid_t", ], ) @@ -3580,6 +3580,7 @@ libc_function( }), weak = True, deps = [ + ":__support_macros_sanitizer", ":__support_osutil_syscall", ":errno", ":hdr_signal_macros", @@ -3599,6 +3600,7 @@ libc_function( }), weak = True, deps = [ + ":__support_macros_sanitizer", ":__support_osutil_syscall", ":errno", ":hdr_signal_macros", @@ -3620,6 +3622,7 @@ libc_function( # }), # weak = True, # deps = [ +# ":__support_macros_sanitizer", # ":__support_osutil_syscall", # ":errno", # ":hdr_signal_macros", -- GitLab From 0e743ecca018e9fa7482dc8bec62eae8cd0c929b Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Wed, 5 Jun 2024 23:02:57 +0100 Subject: [PATCH 018/462] VPlan/PatternMatch: introduce m_c_Mul (NFC) (#93950) Introduce a commutative version of m_Mul, and simplify a usage based on it. --- llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h | 7 +++++++ llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 +-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index 058746880743..d6b4acb2bdba 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -274,6 +274,13 @@ m_Mul(const Op0_t &Op0, const Op1_t &Op1) { return m_Binary(Op0, Op1); } +template +inline AllBinaryRecipe_match +m_c_Mul(const Op0_t &Op0, const Op1_t &Op1) { + return m_Binary(Op0, Op1); +} + /// Match a binary OR operation. Note that while conceptually the operands can /// be matched commutatively, \p Commutative defaults to false in line with the /// IR-based pattern matching infrastructure. Use m_c_BinaryOr for a commutative diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index ab3b5cf2b9da..8ec67eb2f54b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1037,8 +1037,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { return; } - if (match(&R, m_CombineOr(m_Mul(m_VPValue(A), m_SpecificInt(1)), - m_Mul(m_SpecificInt(1), m_VPValue(A))))) + if (match(&R, m_c_Mul(m_VPValue(A), m_SpecificInt(1)))) return R.getVPSingleValue()->replaceAllUsesWith(A); } -- GitLab From 53ddc87454669c0d595c0e3d3174e35cdc4b0a61 Mon Sep 17 00:00:00 2001 From: Han-Chung Wang Date: Wed, 5 Jun 2024 15:07:22 -0700 Subject: [PATCH 019/462] [mlir][vector] Improve flattening vector.transfer_write ops. (#94051) We can flatten the transfer ops even when the collapsed indices are not zeros. We can compute it. It is already supported in vector.transfer_read cases. The revision refactors the logic and reuse it in transfer_write cases. --- .../Transforms/VectorTransferOpTransforms.cpp | 143 ++++++++---------- .../Vector/vector-transfer-flatten.mlir | 43 +++++- 2 files changed, 102 insertions(+), 84 deletions(-) diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp index 997b56a1ce14..c131fde517f8 100644 --- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp @@ -505,25 +505,61 @@ static Value collapseInnerDims(PatternRewriter &rewriter, mlir::Location loc, return rewriter.create(loc, input, reassociation); } -/// Checks that the indices corresponding to dimensions starting at -/// `firstDimToCollapse` are constant 0, and writes to `outIndices` -/// the truncated indices where `firstDimToCollapse` is now the innermost dim. -/// TODO: Extract the logic that writes to outIndices so that this method -/// simply checks one pre-condition. -static LogicalResult -checkAndCollapseInnerZeroIndices(ValueRange indices, int64_t firstDimToCollapse, - SmallVector &outIndices) { - int64_t rank = indices.size(); - if (firstDimToCollapse >= rank) - return failure(); - for (int64_t i = firstDimToCollapse; i < rank; ++i) { - std::optional cst = getConstantIntValue(indices[i]); - if (!cst || cst.value() != 0) - return failure(); +/// Returns the new indices that collapses the inner dimensions starting from +/// the `firstDimToCollapse` dimension. +static SmallVector getCollapsedIndices(RewriterBase &rewriter, + Location loc, + ArrayRef shape, + ValueRange indices, + int64_t firstDimToCollapse) { + assert(firstDimToCollapse < static_cast(indices.size())); + + // If all the collapsed indices are zero then no extra logic is needed. + // Otherwise, a new offset/index has to be computed. + SmallVector indicesAfterCollapsing( + indices.begin(), indices.begin() + firstDimToCollapse); + SmallVector indicesToCollapse(indices.begin() + firstDimToCollapse, + indices.end()); + if (llvm::all_of(indicesToCollapse, isZeroIndex)) { + indicesAfterCollapsing.push_back(indicesToCollapse[0]); + return indicesAfterCollapsing; + } + + // Compute the remaining trailing index/offset required for reading from + // the collapsed memref: + // + // offset = 0 + // for (i = firstDimToCollapse; i < outputRank; ++i) + // offset += sourceType.getDimSize(i) * transferReadOp.indices[i] + // + // For this example: + // %2 = vector.transfer_read/write %arg4[%c0, %arg0, %c0] (...) : + // memref<1x43x2xi32>, vector<1x2xi32> + // which would be collapsed to: + // %1 = vector.transfer_read/write %collapse_shape[%c0, %offset] (...) : + // memref<1x86xi32>, vector<2xi32> + // one would get the following offset: + // %offset = %arg0 * 43 + OpFoldResult collapsedOffset = + rewriter.create(loc, 0).getResult(); + + auto collapsedStrides = computeSuffixProduct( + ArrayRef(shape.begin() + firstDimToCollapse, shape.end())); + + // Compute the collapsed offset. + auto &&[collapsedExpr, collapsedVals] = + computeLinearIndex(collapsedOffset, collapsedStrides, indicesToCollapse); + collapsedOffset = affine::makeComposedFoldedAffineApply( + rewriter, loc, collapsedExpr, collapsedVals); + + if (collapsedOffset.is()) { + indicesAfterCollapsing.push_back(collapsedOffset.get()); + } else { + indicesAfterCollapsing.push_back(rewriter.create( + loc, *getConstantIntValue(collapsedOffset))); } - outIndices = indices; - outIndices.resize(firstDimToCollapse + 1); - return success(); + + return indicesAfterCollapsing; } namespace { @@ -594,54 +630,9 @@ public: AffineMap::get(collapsedRank, 0, dimExprs, rewriter.getContext()); // 2.2 New indices - // If all the collapsed indices are zero then no extra logic is needed. - // Otherwise, a new offset/index has to be computed. - SmallVector collapsedIndices; - if (failed(checkAndCollapseInnerZeroIndices(transferReadOp.getIndices(), - firstDimToCollapse, - collapsedIndices))) { - // Copy all the leading indices. - SmallVector indices = transferReadOp.getIndices(); - collapsedIndices.append(indices.begin(), - indices.begin() + firstDimToCollapse); - - // Compute the remaining trailing index/offset required for reading from - // the collapsed memref: - // - // offset = 0 - // for (i = firstDimToCollapse; i < outputRank; ++i) - // offset += sourceType.getDimSize(i) * transferReadOp.indices[i] - // - // For this example: - // %2 = vector.transfer_read %arg4[%c0, %arg0, %c0] (...) : - // memref<1x43x2xi32>, vector<1x2xi32> - // which would be collapsed to: - // %1 = vector.transfer_read %collapse_shape[%c0, %offset] (...) : - // memref<1x86xi32>, vector<2xi32> - // one would get the following offset: - // %offset = %arg0 * 43 - OpFoldResult collapsedOffset = - rewriter.create(loc, 0).getResult(); - - auto sourceShape = sourceType.getShape(); - auto collapsedStrides = computeSuffixProduct(ArrayRef( - sourceShape.begin() + firstDimToCollapse, sourceShape.end())); - - // Compute the collapsed offset. - ArrayRef indicesToCollapse(indices.begin() + firstDimToCollapse, - indices.end()); - auto &&[collapsedExpr, collapsedVals] = computeLinearIndex( - collapsedOffset, collapsedStrides, indicesToCollapse); - collapsedOffset = affine::makeComposedFoldedAffineApply( - rewriter, loc, collapsedExpr, collapsedVals); - - if (collapsedOffset.is()) { - collapsedIndices.push_back(collapsedOffset.get()); - } else { - collapsedIndices.push_back(rewriter.create( - loc, *getConstantIntValue(collapsedOffset))); - } - } + SmallVector collapsedIndices = + getCollapsedIndices(rewriter, loc, sourceType.getShape(), + transferReadOp.getIndices(), firstDimToCollapse); // 3. Create new vector.transfer_read that reads from the collapsed memref VectorType flatVectorType = VectorType::get({vectorType.getNumElements()}, @@ -697,8 +688,7 @@ public: return failure(); if (!vector::isContiguousSlice(sourceType, vectorType)) return failure(); - int64_t firstContiguousInnerDim = - sourceType.getRank() - vectorType.getRank(); + int64_t firstDimToCollapse = sourceType.getRank() - vectorType.getRank(); // TODO: generalize this pattern, relax the requirements here. if (transferWriteOp.hasOutOfBoundsDim()) return failure(); @@ -706,22 +696,23 @@ public: return failure(); if (transferWriteOp.getMask()) return failure(); - SmallVector collapsedIndices; - if (failed(checkAndCollapseInnerZeroIndices(transferWriteOp.getIndices(), - firstContiguousInnerDim, - collapsedIndices))) - return failure(); + + SmallVector collapsedIndices = + getCollapsedIndices(rewriter, loc, sourceType.getShape(), + transferWriteOp.getIndices(), firstDimToCollapse); Value collapsedSource = - collapseInnerDims(rewriter, loc, source, firstContiguousInnerDim); + collapseInnerDims(rewriter, loc, source, firstDimToCollapse); MemRefType collapsedSourceType = cast(collapsedSource.getType()); int64_t collapsedRank = collapsedSourceType.getRank(); - assert(collapsedRank == firstContiguousInnerDim + 1); + assert(collapsedRank == firstDimToCollapse + 1); + SmallVector dimExprs{ - getAffineDimExpr(firstContiguousInnerDim, rewriter.getContext())}; + getAffineDimExpr(firstDimToCollapse, rewriter.getContext())}; auto collapsedMap = AffineMap::get(collapsedRank, 0, dimExprs, rewriter.getContext()); + VectorType flatVectorType = VectorType::get({vectorType.getNumElements()}, vectorType.getElementType()); Value flatVector = diff --git a/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir b/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir index 788ae9ac044e..65bf0b9335d2 100644 --- a/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir +++ b/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir @@ -471,16 +471,16 @@ func.func @regression_non_contiguous_dim_read(%subview : memref<1x3x3x2xf32, str } // CHECK: #[[$MAP:.+]] = affine_map<()[s0] -> (s0 * 2)> -// CHECK-LABEL: func.func @regression_non_contiguous_dim_read( -// CHECK: %[[COLLAPSE:.+]] = memref.collapse_shape %{{.*}} {{\[}}[0], [1], [2, 3]] : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>> into memref<1x3x6xf32, strided<[40, 10, 1], offset: ?>> -// CHECK: %[[APPLY:.*]] = affine.apply #[[$MAP]]() +// CHECK-LABEL: func.func @regression_non_contiguous_dim_read( +// CHECK: %[[COLLAPSE:.+]] = memref.collapse_shape %{{.*}} {{\[}}[0], [1], [2, 3]] : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>> into memref<1x3x6xf32, strided<[40, 10, 1], offset: ?>> +// CHECK: %[[APPLY:.*]] = affine.apply #[[$MAP]]() // CHECK-128B-LABEL: func @regression_non_contiguous_dim_read( // CHECK-128B: memref.collapse_shape // ----- -func.func @unsupported_non_contiguous_dim_write(%value : vector<2x2xf32>, +func.func @regression_non_contiguous_dim_write(%value : vector<2x2xf32>, %subview : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>>, %idx0 : index, %idx1 : index) { %c0 = arith.constant 0 : index @@ -488,8 +488,35 @@ func.func @unsupported_non_contiguous_dim_write(%value : vector<2x2xf32>, return } -// CHECK-LABEL: func.func @unsupported_non_contiguous_dim_write( -// CHECK-NOT: memref.collapse_shape +// CHECK: #[[$MAP:.+]] = affine_map<()[s0] -> (s0 * 2)> +// CHECK-LABEL: func.func @regression_non_contiguous_dim_write( +// CHECK: %[[APPLY:.*]] = affine.apply #[[$MAP]]() +// CHECK: %[[COLLAPSE:.+]] = memref.collapse_shape %{{.*}} {{\[}}[0], [1], [2, 3]] : memref<1x3x3x2xf32, strided<[40, 10, 2, 1], offset: ?>> into memref<1x3x6xf32, strided<[40, 10, 1], offset: ?>> -// CHECK-128B-LABEL: func @unsupported_non_contiguous_dim_write( -// CHECK-128B-NOT: memref.collapse_shape +// CHECK-128B-LABEL: func @regression_non_contiguous_dim_write( +// CHECK-128B: memref.collapse_shape + +// ----- + +func.func @negative_out_of_bound_transfer_read( + %arg : memref>) -> vector<5x4x3x2xi8> { + %c0 = arith.constant 0 : index + %cst = arith.constant 0 : i8 + %v = vector.transfer_read %arg[%c0, %c0, %c0, %c0], %cst {in_bounds = [false, true, true, true]} : + memref>, vector<5x4x3x2xi8> + return %v : vector<5x4x3x2xi8> +} +// CHECK: func.func @negative_out_of_bound_transfer_read +// CHECK-NOT: memref.collapse_shape + +// ----- + +func.func @negative_out_of_bound_transfer_write( + %arg : memref>, %vec : vector<1x1x3x2xi8>) { + %c0 = arith.constant 0 : index + vector.transfer_write %vec, %arg [%c0, %c0, %c0, %c0] {in_bounds = [false, true, true, true]} : + vector<1x1x3x2xi8>, memref> + return +} +// CHECK: func.func @negative_out_of_bound_transfer_write +// CHECK-NOT: memref.collapse_shape -- GitLab From 55d2fffdae5531759569e4ea8985c3de2e96bcc1 Mon Sep 17 00:00:00 2001 From: Sandeep Dasgupta Date: Wed, 5 Jun 2024 22:10:55 +0000 Subject: [PATCH 020/462] [mlir][python]Python Bindings for select edit operations on Block arguments (#94305) The PR implements MLIR Python Bindings for a few simple edit operations on Block arguments, namely, `add_argument`, `erase_argument`, and `erase_arguments`. --- mlir/include/mlir-c/IR.h | 3 +++ mlir/lib/Bindings/Python/IRCore.cpp | 13 ++++++++++++ mlir/lib/CAPI/IR/IR.cpp | 4 ++++ mlir/test/python/ir/blocks.py | 32 +++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/mlir/include/mlir-c/IR.h b/mlir/include/mlir-c/IR.h index 32abacf35313..e3d69b7708b3 100644 --- a/mlir/include/mlir-c/IR.h +++ b/mlir/include/mlir-c/IR.h @@ -858,6 +858,9 @@ MLIR_CAPI_EXPORTED MlirValue mlirBlockAddArgument(MlirBlock block, MlirType type, MlirLocation loc); +/// Erase the argument at 'index' and remove it from the argument list. +MLIR_CAPI_EXPORTED void mlirBlockEraseArgument(MlirBlock block, unsigned index); + /// Inserts an argument of the specified type at a specified index to the block. /// Returns the newly added argument. MLIR_CAPI_EXPORTED MlirValue mlirBlockInsertArgument(MlirBlock block, diff --git a/mlir/lib/Bindings/Python/IRCore.cpp b/mlir/lib/Bindings/Python/IRCore.cpp index de20632b4fb7..4b6b54dc1c11 100644 --- a/mlir/lib/Bindings/Python/IRCore.cpp +++ b/mlir/lib/Bindings/Python/IRCore.cpp @@ -3238,6 +3238,19 @@ void mlir::python::populateIRCore(py::module &m) { return PyBlockArgumentList(self.getParentOperation(), self.get()); }, "Returns a list of block arguments.") + .def( + "add_argument", + [](PyBlock &self, const PyType &type, const PyLocation &loc) { + return mlirBlockAddArgument(self.get(), type, loc); + }, + "Append an argument of the specified type to the block and returns " + "the newly added argument.") + .def( + "erase_argument", + [](PyBlock &self, unsigned index) { + return mlirBlockEraseArgument(self.get(), index); + }, + "Erase the argument at 'index' and remove it from the argument list.") .def_property_readonly( "operations", [](PyBlock &self) { diff --git a/mlir/lib/CAPI/IR/IR.cpp b/mlir/lib/CAPI/IR/IR.cpp index a72cd247e73f..4e823c866f43 100644 --- a/mlir/lib/CAPI/IR/IR.cpp +++ b/mlir/lib/CAPI/IR/IR.cpp @@ -906,6 +906,10 @@ MlirValue mlirBlockAddArgument(MlirBlock block, MlirType type, return wrap(unwrap(block)->addArgument(unwrap(type), unwrap(loc))); } +void mlirBlockEraseArgument(MlirBlock block, unsigned index) { + return unwrap(block)->eraseArgument(index); +} + MlirValue mlirBlockInsertArgument(MlirBlock block, intptr_t pos, MlirType type, MlirLocation loc) { return wrap(unwrap(block)->insertArgument(pos, unwrap(type), unwrap(loc))); diff --git a/mlir/test/python/ir/blocks.py b/mlir/test/python/ir/blocks.py index 8b4d946c97b8..70ccaeeb5435 100644 --- a/mlir/test/python/ir/blocks.py +++ b/mlir/test/python/ir/blocks.py @@ -145,3 +145,35 @@ def testBlockHash(): block1 = Block.create_at_start(dummy.operation.regions[0], [f32]) block2 = Block.create_at_start(dummy.operation.regions[0], [f32]) assert hash(block1) != hash(block2) + + +# CHECK-LABEL: TEST: testBlockAddArgs +@run +def testBlockAddArgs(): + with Context() as ctx, Location.unknown(ctx) as loc: + ctx.allow_unregistered_dialects = True + f32 = F32Type.get() + op = Operation.create("test", regions=1, loc=Location.unknown()) + blocks = op.regions[0].blocks + blocks.append() + # CHECK: ^bb0: + op.print(enable_debug_info=True) + blocks[0].add_argument(f32, loc) + # CHECK: ^bb0(%{{.+}}: f32 loc(unknown)): + op.print(enable_debug_info=True) + + +# CHECK-LABEL: TEST: testBlockEraseArgs +@run +def testBlockEraseArgs(): + with Context() as ctx, Location.unknown(ctx) as loc: + ctx.allow_unregistered_dialects = True + f32 = F32Type.get() + op = Operation.create("test", regions=1, loc=Location.unknown()) + blocks = op.regions[0].blocks + blocks.append(f32) + # CHECK: ^bb0(%{{.+}}: f32 loc(unknown)): + op.print(enable_debug_info=True) + blocks[0].erase_argument(0) + # CHECK: ^bb0: + op.print(enable_debug_info=True) -- GitLab From 1d75c59ace2c9fc4e9a94907748d9555cd4a7d14 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 5 Jun 2024 18:29:46 -0400 Subject: [PATCH 021/462] Fix detection of __datasizeof with Clang. (#94174) The `__has_extension` builtin macro is the same as __has_feature when -pedantic-errors is specified, which means we don't get the right information about __datasizeof being available. Using __has_keyword (really !__is_identifier) will tell the truth even when -pedantic-errors is specified. This means we always have __datasizeof under Clang --- libcxx/include/__type_traits/datasizeof.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcxx/include/__type_traits/datasizeof.h b/libcxx/include/__type_traits/datasizeof.h index 54fde242ebcd..35c12921e8ff 100644 --- a/libcxx/include/__type_traits/datasizeof.h +++ b/libcxx/include/__type_traits/datasizeof.h @@ -26,7 +26,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if __has_extension(datasizeof) +#if __has_keyword(__datasizeof) || __has_extension(datasizeof) template inline const size_t __datasizeof_v = __datasizeof(_Tp); #else -- GitLab From 19bce1702bd1e399bea76d0de2a649a14551b000 Mon Sep 17 00:00:00 2001 From: Jason Molenda Date: Wed, 5 Jun 2024 15:39:40 -0700 Subject: [PATCH 022/462] [lldb] [NFC] Rewrite TestRedefinitionsInInlines.py as an API test (#94539) Rewrite an inline test as an API test, to be a little easier to debug, and add some additional checks that we're in the inlined test1, then step and we are now in the inlined test2 functions. --- lldb/test/API/lang/c/inlines/Makefile | 3 + .../c/inlines/TestRedefinitionsInInlines.py | 72 +++++++++++++++---- lldb/test/API/lang/c/inlines/main.c | 23 +++--- 3 files changed, 73 insertions(+), 25 deletions(-) create mode 100644 lldb/test/API/lang/c/inlines/Makefile diff --git a/lldb/test/API/lang/c/inlines/Makefile b/lldb/test/API/lang/c/inlines/Makefile new file mode 100644 index 000000000000..f9555f92be8c --- /dev/null +++ b/lldb/test/API/lang/c/inlines/Makefile @@ -0,0 +1,3 @@ +C_SOURCES := main.c + +include Makefile.rules diff --git a/lldb/test/API/lang/c/inlines/TestRedefinitionsInInlines.py b/lldb/test/API/lang/c/inlines/TestRedefinitionsInInlines.py index 024b9dad9b0f..062fd88f7d27 100644 --- a/lldb/test/API/lang/c/inlines/TestRedefinitionsInInlines.py +++ b/lldb/test/API/lang/c/inlines/TestRedefinitionsInInlines.py @@ -1,14 +1,60 @@ -from lldbsuite.test import lldbinline -from lldbsuite.test import decorators - -lldbinline.MakeInlineTest( - __file__, - globals(), - [ - decorators.expectedFailureAll( - compiler="clang", - compiler_version=["<", "3.5"], - bugnumber="llvm.org/pr27845", +"""Test that inlined argument variables have their correct location in debuginfo""" + +import lldb +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil + + +class TestRedefinitionsInInlines(TestBase): + # https://github.com/llvm/llvm-project/issues/28219 + @skipIf(compiler="clang", compiler_version=["<", "3.5"]) + def test(self): + self.source = "main.c" + self.build() + (target, process, thread, bp1) = lldbutil.run_to_source_breakpoint( + self, "first breakpoint", lldb.SBFileSpec(self.source, False) + ) + + bp2 = target.BreakpointCreateBySourceRegex( + "second breakpoint", lldb.SBFileSpec(self.source, False) + ) + bp3 = target.BreakpointCreateBySourceRegex( + "third breakpoint", lldb.SBFileSpec(self.source, False) ) - ], -) + + # When called from main(), test2 is passed in the value of 42 in 'b' + self.expect("expression b", DATA_TYPES_DISPLAYED_CORRECTLY, substrs=["42"]) + + process.Continue() + + self.assertState(process.GetState(), lldb.eStateStopped) + thread = lldbutil.get_stopped_thread(process, lldb.eStopReasonBreakpoint) + self.assertIsNotNone(thread) + bp_id = thread.GetStopReasonDataAtIndex(0) + self.assertEqual(bp_id, bp2.GetID()) + + self.expect("expression b", DATA_TYPES_DISPLAYED_CORRECTLY, substrs=["42"]) + self.expect("expression c", DATA_TYPES_DISPLAYED_CORRECTLY, substrs=["84"]) + + process.Continue() + + # Now we're in test1(), and the first thing it does is call test2(24). "Step in" + # and check that we have the value 24 as the argument. + self.assertState(process.GetState(), lldb.eStateStopped) + thread = lldbutil.get_stopped_thread(process, lldb.eStopReasonBreakpoint) + self.assertIsNotNone(thread) + bp_id = thread.GetStopReasonDataAtIndex(0) + self.assertEqual(bp_id, bp3.GetID()) + + frame = thread.GetFrameAtIndex(0) + self.assertTrue(frame.IsInlined()) + self.assertEqual(frame.GetFunctionName(), "test1") + + thread.StepInto() + + frame = thread.GetFrameAtIndex(0) + self.assertTrue(frame.IsInlined()) + self.assertEqual(frame.GetFunctionName(), "test2") + + self.expect("expression b", DATA_TYPES_DISPLAYED_CORRECTLY, substrs=["24"]) diff --git a/lldb/test/API/lang/c/inlines/main.c b/lldb/test/API/lang/c/inlines/main.c index 8fe49180800b..6ecc04dd508f 100644 --- a/lldb/test/API/lang/c/inlines/main.c +++ b/lldb/test/API/lang/c/inlines/main.c @@ -3,23 +3,22 @@ inline void test1(int) __attribute__ ((always_inline)); inline void test2(int) __attribute__ ((always_inline)); +// Called once from main with b==42 then called from test1 with b==24. void test2(int b) { - printf("test2(%d)\n", b); //% self.expect("expression b", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["42"]) - { - int c = b * 2; - printf("c=%d\n", c); //% self.expect("expression b", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["42"]) - //% self.expect("expression c", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["84"]) - } + printf("test2(%d)\n", b); // first breakpoint + { + int c = b * 2; + printf("c=%d\n", c); // second breakpoint + } } void test1(int a) { printf("test1(%d)\n", a); - test2(a+1);//% self.runCmd("step") - //% self.expect("expression b", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["24"]) + test2(a + 1); // third breakpoint } -int main() { - test2(42); - test1(23); - return 0; +int main(int argc) { + test2(42); + test1(23); + return 0; } -- GitLab From 68a9cb799511506045ca26c04e7933f0e0ed46ec Mon Sep 17 00:00:00 2001 From: Med Ismail Bennani Date: Wed, 5 Jun 2024 15:45:42 -0700 Subject: [PATCH 023/462] [lldb/crashlog] Add `--no-parallel-image-loading` hidden flag (#94513) This patch adds the `--no-parallel-image-loading` to the crashlog command. By default, image loading will happen in parallel in the crashlog script however, sometimes, when running tests or debugging the crashlog script itself, it's better to load the images sequentially. As its name suggests, this flag will disable the default image loading behaviour to load all the images sequencially in the main thread. Signed-off-by: Med Ismail Bennani --- lldb/examples/python/crashlog.py | 18 +++++++++++++++--- .../python/crashlog_scripted_process.py | 8 ++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/lldb/examples/python/crashlog.py b/lldb/examples/python/crashlog.py index c874cb4d32e6..7c6c60e518d7 100755 --- a/lldb/examples/python/crashlog.py +++ b/lldb/examples/python/crashlog.py @@ -557,11 +557,15 @@ class CrashLog(symbolication.Symbolicator): futures = [] with tempfile.TemporaryDirectory() as obj_dir: - with concurrent.futures.ThreadPoolExecutor() as executor: - def add_module(image, target, obj_dir): - return image, image.add_module(target, obj_dir) + def add_module(image, target, obj_dir): + return image, image.add_module(target, obj_dir) + max_worker = None + if options.no_parallel_image_loading: + max_worker = 1 + + with concurrent.futures.ThreadPoolExecutor(max_worker) as executor: for image in images_to_load: if image not in loaded_images: if image.uuid == uuid.UUID(int=0): @@ -1530,6 +1534,7 @@ def load_crashlog_in_scripted_process(debugger, crashlog_path, options, result): "file_path": crashlog_path, "load_all_images": options.load_all_images, "crashed_only": options.crashed_only, + "no_parallel_image_loading": options.no_parallel_image_loading, } ) ) @@ -1722,6 +1727,13 @@ def CreateSymbolicateCrashLogOptions( help="show source for all threads, not just the crashed thread", default=False, ) + arg_parser.add_argument( + "--no-parallel-image-loading", + dest="no_parallel_image_loading", + action="store_true", + help=argparse.SUPPRESS, + default=False, + ) if add_interactive_options: arg_parser.add_argument( "-i", diff --git a/lldb/examples/python/crashlog_scripted_process.py b/lldb/examples/python/crashlog_scripted_process.py index 26c5c37b7371..2ee030239ee3 100644 --- a/lldb/examples/python/crashlog_scripted_process.py +++ b/lldb/examples/python/crashlog_scripted_process.py @@ -53,6 +53,7 @@ class CrashLogScriptedProcess(ScriptedProcess): class CrashLogOptions: load_all_images = False crashed_only = True + no_parallel_image_loading = False def __init__(self, exe_ctx: lldb.SBExecutionContext, args: lldb.SBStructuredData): super().__init__(exe_ctx, args) @@ -84,6 +85,13 @@ class CrashLogScriptedProcess(ScriptedProcess): if crashed_only.GetType() == lldb.eStructuredDataTypeBoolean: self.options.crashed_only = crashed_only.GetBooleanValue() + no_parallel_image_loading = args.GetValueForKey("no_parallel_image_loading") + if no_parallel_image_loading and no_parallel_image_loading.IsValid(): + if no_parallel_image_loading.GetType() == lldb.eStructuredDataTypeBoolean: + self.options.no_parallel_image_loading = ( + no_parallel_image_loading.GetBooleanValue() + ) + self.pid = super().get_process_id() self.crashed_thread_idx = 0 self.exception = None -- GitLab From a9342f2eab7993a3be1d1156070c0d2de21ed973 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 5 Jun 2024 17:46:16 -0500 Subject: [PATCH 024/462] [RISCV] Remove VPseudoBinaryMOutNoMask. NFC (#94531) It appears to be identical to VPseudoBinaryNoMask. The comment above it doesn't make sense since VPseudoBinaryNoMask doesn't restrict the destination register to not include V0. Looks like a copy/paste from VPseudoBinaryMOutMask. --- .../Target/RISCV/RISCVInstrInfoVPseudos.td | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index a85a8bb87d11..b0949f5fc1d7 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -1438,24 +1438,6 @@ class VPseudoTernaryMaskPolicyRoundingMode : - Pseudo<(outs RetClass:$rd), - (ins Op1Class:$rs2, Op2Class:$rs1, AVL:$vl, ixlenimm:$sew), []>, - RISCVVPseudo { - let mayLoad = 0; - let mayStore = 0; - let hasSideEffects = 0; - let Constraints = Constraint; - let TargetOverlapConstraintType = TargetConstraintType; - let HasVLOp = 1; - let HasSEWOp = 1; -} - // Like VPseudoBinaryMask, but output can be V0. class VPseudoBinaryMOutMask { let VLMul = MInfo.value, isCommutable = Commutable in { - def "_" # MInfo.MX : VPseudoBinaryMOutNoMask; + def "_" # MInfo.MX : VPseudoBinaryNoMask; let ForceTailAgnostic = true in def "_" # MInfo.MX # "_MASK" : VPseudoBinaryMOutMask, -- GitLab From dbc3e26c25587e5460ae12caed84cb09197c4ed7 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Wed, 5 Jun 2024 18:08:19 -0500 Subject: [PATCH 025/462] [libc][NFC] More verbose warning message on missing utilities Summary: The GPU needs both the loader utility and the architecture to be present to run tests. This simply makes it easier to detect which is missing in case of problems. --- libc/test/CMakeLists.txt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/libc/test/CMakeLists.txt b/libc/test/CMakeLists.txt index 5e26a1000633..b5c989aa1d43 100644 --- a/libc/test/CMakeLists.txt +++ b/libc/test/CMakeLists.txt @@ -8,10 +8,14 @@ add_custom_target(libc-long-running-tests) add_subdirectory(UnitTest) -if(LIBC_TARGET_OS_IS_GPU AND - (NOT TARGET libc.utils.gpu.loader OR LIBC_GPU_TESTS_DISABLED)) - message(WARNING "Cannot build libc GPU tests, missing loader or architecture") - return() +if(LIBC_TARGET_OS_IS_GPU) + if(NOT TARGET libc.utils.gpu.loader) + message(WARNING "Cannot build libc GPU tests, missing loader.") + return() + elseif(LIBC_GPU_TESTS_DISABLED) + message(WARNING "Cannot build libc GPU tests, missing target architecture.") + return() + endif() endif() add_subdirectory(include) -- GitLab From 2ec122d9662e8e7f034e822fcfb3fc3c372971ba Mon Sep 17 00:00:00 2001 From: Daniel Sanders Date: Wed, 5 Jun 2024 17:40:29 -0700 Subject: [PATCH 026/462] Test commit --- llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp index e5c6b6d60d06..399ce6ef91ac 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp @@ -6,6 +6,7 @@ // //===----------------------------------------------------------------------===// + #include "AArch64TargetObjectFile.h" #include "AArch64TargetMachine.h" #include "llvm/BinaryFormat/Dwarf.h" -- GitLab From 62c0eaeec9a65c73f3bee0b8b05461f1c628d27c Mon Sep 17 00:00:00 2001 From: Daniel Sanders Date: Wed, 5 Jun 2024 17:40:52 -0700 Subject: [PATCH 027/462] Revert "Test commit" This reverts commit 2ec122d9662e8e7f034e822fcfb3fc3c372971ba. --- llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp index 399ce6ef91ac..e5c6b6d60d06 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp @@ -6,7 +6,6 @@ // //===----------------------------------------------------------------------===// - #include "AArch64TargetObjectFile.h" #include "AArch64TargetMachine.h" #include "llvm/BinaryFormat/Dwarf.h" -- GitLab From 798f2019a2fb608450c5ad5dc261d6496cf1eb15 Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Wed, 5 Jun 2024 17:57:36 -0700 Subject: [PATCH 028/462] [clang-format] Don't format comments in SkipMacroDefinitionBody (#94425) Fixes #94326. --- clang/lib/Format/UnwrappedLineParser.cpp | 6 +++--- clang/unittests/Format/FormatTest.cpp | 11 ++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp index d6061c2666c2..eb96b54ec4c9 100644 --- a/clang/lib/Format/UnwrappedLineParser.cpp +++ b/clang/lib/Format/UnwrappedLineParser.cpp @@ -1181,10 +1181,10 @@ void UnwrappedLineParser::parsePPDefine() { Line->InMacroBody = true; if (Style.SkipMacroDefinitionBody) { - do { + while (!eof()) { FormatTok->Finalized = true; - nextToken(); - } while (!eof()); + FormatTok = Tokens->getNextToken(); + } addUnwrappedLine(); return; } diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index 004ecb63f662..4e427268fb82 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -24879,7 +24879,7 @@ TEST_F(FormatTest, SkipMacroDefinitionBody) { Style); // With comments. - verifyFormat("/* */ #define A a // a a", "/* */ # define A a // a a", + verifyFormat("/* */ #define A a // a a", "/* */ # define A a // a a", Style); verifyNoChange("/* */ #define A a // a a", Style); @@ -24891,6 +24891,15 @@ TEST_F(FormatTest, SkipMacroDefinitionBody) { "int aaa; // a", Style); + verifyNoChange( + "#define MACRO_WITH_COMMENTS() \\\n" + " public: \\\n" + " /* Documentation parsed by Doxygen for the following method. */ \\\n" + " static MyType getClassTypeId(); \\\n" + " /** Normal comment for the following method. */ \\\n" + " virtual MyType getTypeId() const;", + Style); + // multiline macro definitions verifyNoChange("#define A a\\\n" " A a \\\n " -- GitLab From fb8ea85987470ad82b365fb1a78bb66754f927c6 Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Thu, 6 Jun 2024 02:32:27 +0100 Subject: [PATCH 029/462] [RuntimeDyld][ELF] Fix unwanted sign extension. (#94482) Casting the result of `Section.getAddressWithOffset()` goes wrong if we are on a 32-bit platform whose addresses are regarded as signed; in that case, just doing ``` (uint64_t)Section.getAddressWithOffset(...) ``` or ``` reinterpret_cast(Section.getAddressWithOffset(...)) ``` will result in sign-extension. We use these expressions when constructing branch stubs, which is before we know the final load address, so we can just switch to the `Section.getLoadAddressWithOffset(...)` method instead. Doing that is also more consistent, since when calculating relative offsets for relocations, we use the load address anyway, so the code currently only works because `Section.Address` is equal to `Section.LoadAddress` at this point. Fixes #94478. --- .../RuntimeDyld/RuntimeDyldELF.cpp | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index 004622061120..f44a6a472cb6 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -1183,8 +1183,7 @@ void RuntimeDyldELF::resolveAArch64Branch(unsigned SectionID, StubMap::const_iterator i = Stubs.find(Value); if (i != Stubs.end()) { resolveRelocation(Section, Offset, - (uint64_t)Section.getAddressWithOffset(i->second), - RelType, 0); + Section.getLoadAddressWithOffset(i->second), RelType, 0); LLVM_DEBUG(dbgs() << " Stub function found\n"); } else if (!resolveAArch64ShortBranch(SectionID, RelI, Value)) { // Create a new stub function. @@ -1217,8 +1216,7 @@ void RuntimeDyldELF::resolveAArch64Branch(unsigned SectionID, addRelocationForSection(REmovk_g0, Value.SectionID); } resolveRelocation(Section, Offset, - reinterpret_cast(Section.getAddressWithOffset( - Section.getStubOffset())), + Section.getLoadAddressWithOffset(Section.getStubOffset()), RelType, 0); Section.advanceStubOffset(getMaxStubSize()); } @@ -1349,10 +1347,9 @@ RuntimeDyldELF::processRelocationRef( // Look for an existing stub. StubMap::const_iterator i = Stubs.find(Value); if (i != Stubs.end()) { - resolveRelocation( - Section, Offset, - reinterpret_cast(Section.getAddressWithOffset(i->second)), - RelType, 0); + resolveRelocation(Section, Offset, + Section.getLoadAddressWithOffset(i->second), RelType, + 0); LLVM_DEBUG(dbgs() << " Stub function found\n"); } else { // Create a new stub function. @@ -1367,10 +1364,10 @@ RuntimeDyldELF::processRelocationRef( else addRelocationForSection(RE, Value.SectionID); - resolveRelocation(Section, Offset, reinterpret_cast( - Section.getAddressWithOffset( - Section.getStubOffset())), - RelType, 0); + resolveRelocation( + Section, Offset, + Section.getLoadAddressWithOffset(Section.getStubOffset()), RelType, + 0); Section.advanceStubOffset(getMaxStubSize()); } } else { @@ -1609,8 +1606,7 @@ RuntimeDyldELF::processRelocationRef( if (i != Stubs.end()) { // Symbol function stub already created, just relocate to it resolveRelocation(Section, Offset, - reinterpret_cast( - Section.getAddressWithOffset(i->second)), + Section.getLoadAddressWithOffset(i->second), RelType, 0); LLVM_DEBUG(dbgs() << " Stub function found\n"); } else { @@ -1652,10 +1648,10 @@ RuntimeDyldELF::processRelocationRef( addRelocationForSection(REl, Value.SectionID); } - resolveRelocation(Section, Offset, reinterpret_cast( - Section.getAddressWithOffset( - Section.getStubOffset())), - RelType, 0); + resolveRelocation( + Section, Offset, + Section.getLoadAddressWithOffset(Section.getStubOffset()), + RelType, 0); Section.advanceStubOffset(getMaxStubSize()); } if (IsExtern || (AbiVariant == 2 && Value.SectionID != SectionID)) { -- GitLab From 4b702946006cfa9be9ab646ce5fc5b25248edd81 Mon Sep 17 00:00:00 2001 From: hev Date: Thu, 6 Jun 2024 09:44:08 +0800 Subject: [PATCH 030/462] [LoongArch] Add a hook to sign extend i32 ConstantInt operands of phis on LA64 (#93813) Materializing constants on LoongArch is simpler if the constant is sign extended from i32. By default i32 constant operands of phis are zero extended. This patch adds a hook to allow LoongArch to override this for i32. We have an existing isSExtCheaperThanZExt, but it operates on EVT which we don't have at these places in the code. --- .../LoongArch/LoongArchISelLowering.cpp | 4 +++ .../Target/LoongArch/LoongArchISelLowering.h | 1 + llvm/test/CodeGen/LoongArch/sextw-removal.ll | 32 ++++++------------- 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index 51384f25d245..32e02e3ee8f5 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -5004,6 +5004,10 @@ bool LoongArchTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64; } +bool LoongArchTargetLowering::signExtendConstant(const ConstantInt *CI) const { + return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32); +} + bool LoongArchTargetLowering::hasAndNotCompare(SDValue Y) const { // TODO: Support vectors. if (Y.getValueType().isVector()) diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h index f274b1971fd2..9328831a17a3 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h @@ -229,6 +229,7 @@ public: bool isLegalAddImmediate(int64_t Imm) const override; bool isZExtFree(SDValue Val, EVT VT2) const override; bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override; + bool signExtendConstant(const ConstantInt *CI) const override; bool hasAndNotCompare(SDValue Y) const override; diff --git a/llvm/test/CodeGen/LoongArch/sextw-removal.ll b/llvm/test/CodeGen/LoongArch/sextw-removal.ll index 502b6cf88099..ba889fc8393b 100644 --- a/llvm/test/CodeGen/LoongArch/sextw-removal.ll +++ b/llvm/test/CodeGen/LoongArch/sextw-removal.ll @@ -762,7 +762,6 @@ define signext i32 @test14(i32 signext %0, i32 signext %1) { ; CHECK-NEXT: # %bb.1: # %.preheader ; CHECK-NEXT: ori $a3, $zero, 1 ; CHECK-NEXT: addi.w $a2, $zero, -1 -; CHECK-NEXT: lu32i.d $a2, 0 ; CHECK-NEXT: ori $a4, $zero, 1000 ; CHECK-NEXT: .p2align 4, , 16 ; CHECK-NEXT: .LBB13_2: # =>This Inner Loop Header: Depth=1 @@ -772,10 +771,9 @@ define signext i32 @test14(i32 signext %0, i32 signext %1) { ; CHECK-NEXT: addi.w $a3, $a3, 1 ; CHECK-NEXT: blt $a3, $a1, .LBB13_2 ; CHECK-NEXT: .LBB13_4: -; CHECK-NEXT: addi.w $a0, $a0, 0 ; CHECK-NEXT: ret ; CHECK-NEXT: .LBB13_5: -; CHECK-NEXT: addi.w $a0, $a2, 0 +; CHECK-NEXT: move $a0, $a2 ; CHECK-NEXT: ret ; ; NORMV-LABEL: test14: @@ -785,7 +783,6 @@ define signext i32 @test14(i32 signext %0, i32 signext %1) { ; NORMV-NEXT: # %bb.1: # %.preheader ; NORMV-NEXT: ori $a3, $zero, 1 ; NORMV-NEXT: addi.w $a2, $zero, -1 -; NORMV-NEXT: lu32i.d $a2, 0 ; NORMV-NEXT: ori $a4, $zero, 1000 ; NORMV-NEXT: .p2align 4, , 16 ; NORMV-NEXT: .LBB13_2: # =>This Inner Loop Header: Depth=1 @@ -795,13 +792,12 @@ define signext i32 @test14(i32 signext %0, i32 signext %1) { ; NORMV-NEXT: add.d $a0, $a3, $a0 ; NORMV-NEXT: addi.d $a3, $a3, 1 ; NORMV-NEXT: addi.w $a3, $a3, 0 -; NORMV-NEXT: addi.d $a0, $a0, 0 +; NORMV-NEXT: addi.w $a0, $a0, 0 ; NORMV-NEXT: blt $a3, $a1, .LBB13_2 ; NORMV-NEXT: .LBB13_4: -; NORMV-NEXT: addi.w $a0, $a0, 0 ; NORMV-NEXT: ret ; NORMV-NEXT: .LBB13_5: -; NORMV-NEXT: addi.w $a0, $a2, 0 +; NORMV-NEXT: move $a0, $a2 ; NORMV-NEXT: ret %3 = icmp sgt i32 %1, 1 br i1 %3, label %4, label %12 @@ -830,8 +826,7 @@ define signext i32 @test14b(i32 %0, i32 signext %1) { ; CHECK-NEXT: blt $a1, $a2, .LBB14_4 ; CHECK-NEXT: # %bb.1: # %.preheader ; CHECK-NEXT: ori $a3, $zero, 1 -; CHECK-NEXT: addi.w $a2, $zero, -1 -; CHECK-NEXT: lu32i.d $a2, 0 +; CHECK-NEXT: addi.d $a2, $zero, -1 ; CHECK-NEXT: ori $a4, $zero, 1000 ; CHECK-NEXT: .p2align 4, , 16 ; CHECK-NEXT: .LBB14_2: # =>This Inner Loop Header: Depth=1 @@ -854,8 +849,7 @@ define signext i32 @test14b(i32 %0, i32 signext %1) { ; NORMV-NEXT: blt $a1, $a2, .LBB14_4 ; NORMV-NEXT: # %bb.1: # %.preheader ; NORMV-NEXT: ori $a3, $zero, 1 -; NORMV-NEXT: addi.w $a2, $zero, -1 -; NORMV-NEXT: lu32i.d $a2, 0 +; NORMV-NEXT: addi.d $a2, $zero, -1 ; NORMV-NEXT: ori $a4, $zero, 1000 ; NORMV-NEXT: .p2align 4, , 16 ; NORMV-NEXT: .LBB14_2: # =>This Inner Loop Header: Depth=1 @@ -900,8 +894,7 @@ define signext i32 @test14c(i32 zeroext %0, i32 signext %1) { ; CHECK-NEXT: blt $a1, $a2, .LBB15_4 ; CHECK-NEXT: # %bb.1: # %.preheader ; CHECK-NEXT: ori $a3, $zero, 1 -; CHECK-NEXT: addi.w $a2, $zero, -1 -; CHECK-NEXT: lu32i.d $a2, 0 +; CHECK-NEXT: addi.d $a2, $zero, -1 ; CHECK-NEXT: ori $a4, $zero, 1000 ; CHECK-NEXT: .p2align 4, , 16 ; CHECK-NEXT: .LBB15_2: # =>This Inner Loop Header: Depth=1 @@ -924,8 +917,7 @@ define signext i32 @test14c(i32 zeroext %0, i32 signext %1) { ; NORMV-NEXT: blt $a1, $a2, .LBB15_4 ; NORMV-NEXT: # %bb.1: # %.preheader ; NORMV-NEXT: ori $a3, $zero, 1 -; NORMV-NEXT: addi.w $a2, $zero, -1 -; NORMV-NEXT: lu32i.d $a2, 0 +; NORMV-NEXT: addi.d $a2, $zero, -1 ; NORMV-NEXT: ori $a4, $zero, 1000 ; NORMV-NEXT: .p2align 4, , 16 ; NORMV-NEXT: .LBB15_2: # =>This Inner Loop Header: Depth=1 @@ -971,7 +963,6 @@ define signext i32 @test14d(i31 zeroext %0, i32 signext %1) { ; CHECK-NEXT: # %bb.1: # %.preheader ; CHECK-NEXT: ori $a3, $zero, 1 ; CHECK-NEXT: addi.w $a2, $zero, -1 -; CHECK-NEXT: lu32i.d $a2, 0 ; CHECK-NEXT: ori $a4, $zero, 1000 ; CHECK-NEXT: .p2align 4, , 16 ; CHECK-NEXT: .LBB16_2: # =>This Inner Loop Header: Depth=1 @@ -981,10 +972,9 @@ define signext i32 @test14d(i31 zeroext %0, i32 signext %1) { ; CHECK-NEXT: addi.w $a3, $a3, 1 ; CHECK-NEXT: blt $a3, $a1, .LBB16_2 ; CHECK-NEXT: .LBB16_4: -; CHECK-NEXT: addi.w $a0, $a0, 0 ; CHECK-NEXT: ret ; CHECK-NEXT: .LBB16_5: -; CHECK-NEXT: addi.w $a0, $a2, 0 +; CHECK-NEXT: move $a0, $a2 ; CHECK-NEXT: ret ; ; NORMV-LABEL: test14d: @@ -994,7 +984,6 @@ define signext i32 @test14d(i31 zeroext %0, i32 signext %1) { ; NORMV-NEXT: # %bb.1: # %.preheader ; NORMV-NEXT: ori $a3, $zero, 1 ; NORMV-NEXT: addi.w $a2, $zero, -1 -; NORMV-NEXT: lu32i.d $a2, 0 ; NORMV-NEXT: ori $a4, $zero, 1000 ; NORMV-NEXT: .p2align 4, , 16 ; NORMV-NEXT: .LBB16_2: # =>This Inner Loop Header: Depth=1 @@ -1004,13 +993,12 @@ define signext i32 @test14d(i31 zeroext %0, i32 signext %1) { ; NORMV-NEXT: add.d $a0, $a3, $a0 ; NORMV-NEXT: addi.d $a3, $a3, 1 ; NORMV-NEXT: addi.w $a3, $a3, 0 -; NORMV-NEXT: addi.d $a0, $a0, 0 +; NORMV-NEXT: addi.w $a0, $a0, 0 ; NORMV-NEXT: blt $a3, $a1, .LBB16_2 ; NORMV-NEXT: .LBB16_4: -; NORMV-NEXT: addi.w $a0, $a0, 0 ; NORMV-NEXT: ret ; NORMV-NEXT: .LBB16_5: -; NORMV-NEXT: addi.w $a0, $a2, 0 +; NORMV-NEXT: move $a0, $a2 ; NORMV-NEXT: ret %zext = zext i31 %0 to i32 %3 = icmp sgt i32 %1, 1 -- GitLab From d5ab38f69c1a5c1456bc1a8cc936489d31599f33 Mon Sep 17 00:00:00 2001 From: Jianjian Guan Date: Thu, 6 Jun 2024 10:33:54 +0800 Subject: [PATCH 031/462] [RISCV] Support select/merge like ops for bf16 vectors when have Zvfbfmin (#91936) --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 32 +- .../Target/RISCV/RISCVInstrInfoVPseudos.td | 15 +- .../Target/RISCV/RISCVInstrInfoVSDPatterns.td | 8 +- .../Target/RISCV/RISCVInstrInfoVVLPatterns.td | 5 +- .../RISCV/rvv/fixed-vectors-select-fp.ll | 128 +++++++- .../RISCV/rvv/fixed-vectors-vpmerge.ll | 144 +++++++- .../RISCV/rvv/fixed-vectors-vselect-vp.ll | 56 +++- llvm/test/CodeGen/RISCV/rvv/select-fp.ll | 188 ++++++++++- llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll | 308 +++++++++++++++++- llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll | 224 ++++++++++++- llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll | 80 ++++- 11 files changed, 1150 insertions(+), 38 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 8ace5d79af07..4051279fdbf8 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -1102,6 +1102,15 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::EXTRACT_SUBVECTOR}, VT, Custom); setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); + if (Subtarget.hasStdExtZfbfmin()) { + if (Subtarget.hasVInstructionsF16()) + setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); + else if (Subtarget.hasVInstructionsF16Minimal()) + setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); + } + setOperationAction({ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT, + Custom); + setOperationAction(ISD::SELECT_CC, VT, Expand); // TODO: Promote to fp32. } } @@ -1331,6 +1340,15 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::EXTRACT_SUBVECTOR}, VT, Custom); setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); + if (Subtarget.hasStdExtZfbfmin()) { + if (Subtarget.hasVInstructionsF16()) + setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); + else if (Subtarget.hasVInstructionsF16Minimal()) + setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); + } + setOperationAction( + {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT, + Custom); // TODO: Promote to fp32. continue; } @@ -6704,10 +6722,16 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, case ISD::BUILD_VECTOR: return lowerBUILD_VECTOR(Op, DAG, Subtarget); case ISD::SPLAT_VECTOR: - if (Op.getValueType().getScalarType() == MVT::f16 && - (Subtarget.hasVInstructionsF16Minimal() && - !Subtarget.hasVInstructionsF16())) { - if (Op.getValueType() == MVT::nxv32f16) + if ((Op.getValueType().getScalarType() == MVT::f16 && + (Subtarget.hasVInstructionsF16Minimal() && + Subtarget.hasStdExtZfhminOrZhinxmin() && + !Subtarget.hasVInstructionsF16())) || + (Op.getValueType().getScalarType() == MVT::bf16 && + (Subtarget.hasVInstructionsBF16() && Subtarget.hasStdExtZfbfmin() && + Subtarget.hasVInstructionsF16Minimal() && + !Subtarget.hasVInstructionsF16()))) { + if (Op.getValueType() == MVT::nxv32f16 || + Op.getValueType() == MVT::nxv32bf16) return SplitVectorOp(Op, DAG); SDLoc DL(Op); SDValue NewScalar = diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index b0949f5fc1d7..fe4d839e4fdc 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -382,7 +382,20 @@ class GetIntVTypeInfo { // Equivalent integer vector type. Eg. // VI8M1 → VI8M1 (identity) // VF64M4 → VI64M4 - VTypeInfo Vti = !cast(!subst("VF", "VI", !cast(vti))); + VTypeInfo Vti = !cast(!subst("VBF", "VI", + !subst("VF", "VI", + !cast(vti)))); +} + +// This functor is used to obtain the fp vector type that has the same SEW and +// multiplier as the input parameter type. +class GetFpVTypeInfo { + // Equivalent integer vector type. Eg. + // VF16M1 → VF16M1 (identity) + // VBF16M1 → VF16M1 + VTypeInfo Vti = !cast(!subst("VBF", "VF", + !subst("VI", "VF", + !cast(vti)))); } class MTypeInfo { diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td index 3163e4bafd4b..497c4aadf753 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td @@ -1394,7 +1394,7 @@ defm : VPatFPSetCCSDNode_VV_VF_FV; // Floating-point vselects: // 11.15. Vector Integer Merge Instructions // 13.15. Vector Floating-Point Merge Instruction -foreach fvti = AllFloatVectors in { +foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in { defvar ivti = GetIntVTypeInfo.Vti; let Predicates = GetVTypePredicates.Predicates in { def : Pat<(fvti.Vector (vselect (fvti.Mask V0), fvti.RegClass:$rs1, @@ -1412,7 +1412,9 @@ foreach fvti = AllFloatVectors in { fvti.RegClass:$rs2, 0, (fvti.Mask V0), fvti.AVL, fvti.Log2SEW)>; } - let Predicates = GetVTypePredicates.Predicates in + + let Predicates = !listconcat(GetVTypePredicates.Vti>.Predicates, + GetVTypeScalarPredicates.Predicates) in def : Pat<(fvti.Vector (vselect (fvti.Mask V0), (SplatFPOp fvti.ScalarRegClass:$rs1), fvti.RegClass:$rs2)), @@ -1475,7 +1477,7 @@ foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in { //===----------------------------------------------------------------------===// foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in { - let Predicates = !listconcat(GetVTypePredicates.Predicates, + let Predicates = !listconcat(GetVTypePredicates.Vti>.Predicates, GetVTypeScalarPredicates.Predicates) in def : Pat<(fvti.Vector (riscv_vfmv_v_f_vl undef, fvti.ScalarRegClass:$rs1, srcvalue)), (!cast("PseudoVFMV_V_"#fvti.ScalarSuffix#"_"#fvti.LMul.MX) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td index ce8133a5a297..70d8265e7be4 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td @@ -2604,7 +2604,7 @@ foreach vti = AllFloatVectors in { } } -foreach fvti = AllFloatVectors in { +foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in { // Floating-point vselects: // 11.15. Vector Integer Merge Instructions // 13.15. Vector Floating-Point Merge Instruction @@ -2639,7 +2639,8 @@ foreach fvti = AllFloatVectors in { GPR:$vl, fvti.Log2SEW)>; } - let Predicates = GetVTypePredicates.Predicates in { + let Predicates = !listconcat(GetVTypePredicates.Vti>.Predicates, + GetVTypeScalarPredicates.Predicates) in { def : Pat<(fvti.Vector (riscv_vmerge_vl (fvti.Mask V0), (SplatFPOp fvti.ScalarRegClass:$rs1), fvti.RegClass:$rs2, diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll index d945cf561698..7a96aad31f08 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d -riscv-v-vector-bits-min=128 \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d -riscv-v-vector-bits-min=128 \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d -riscv-v-vector-bits-min=128 \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d -riscv-v-vector-bits-min=128 \ ; RUN: -verify-machineinstrs < %s | FileCheck %s define <2 x half> @select_v2f16(i1 zeroext %c, <2 x half> %a, <2 x half> %b) { @@ -343,3 +343,123 @@ define <16 x double> @selectcc_v16f64(double %a, double %b, <16 x double> %c, <1 %v = select i1 %cmp, <16 x double> %c, <16 x double> %d ret <16 x double> %v } + +define <2 x bfloat> @select_v2bf16(i1 zeroext %c, <2 x bfloat> %a, <2 x bfloat> %b) { +; CHECK-LABEL: select_v2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, <2 x bfloat> %a, <2 x bfloat> %b + ret <2 x bfloat> %v +} + +define <2 x bfloat> @selectcc_v2bf16(bfloat %a, bfloat %b, <2 x bfloat> %c, <2 x bfloat> %d) { +; CHECK-LABEL: selectcc_v2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, <2 x bfloat> %c, <2 x bfloat> %d + ret <2 x bfloat> %v +} + +define <4 x bfloat> @select_v4bf16(i1 zeroext %c, <4 x bfloat> %a, <4 x bfloat> %b) { +; CHECK-LABEL: select_v4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, <4 x bfloat> %a, <4 x bfloat> %b + ret <4 x bfloat> %v +} + +define <4 x bfloat> @selectcc_v4bf16(bfloat %a, bfloat %b, <4 x bfloat> %c, <4 x bfloat> %d) { +; CHECK-LABEL: selectcc_v4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, <4 x bfloat> %c, <4 x bfloat> %d + ret <4 x bfloat> %v +} + +define <8 x bfloat> @select_v8bf16(i1 zeroext %c, <8 x bfloat> %a, <8 x bfloat> %b) { +; CHECK-LABEL: select_v8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, <8 x bfloat> %a, <8 x bfloat> %b + ret <8 x bfloat> %v +} + +define <8 x bfloat> @selectcc_v8bf16(bfloat %a, bfloat %b, <8 x bfloat> %c, <8 x bfloat> %d) { +; CHECK-LABEL: selectcc_v8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, <8 x bfloat> %c, <8 x bfloat> %d + ret <8 x bfloat> %v +} + +define <16 x bfloat> @select_v16bf16(i1 zeroext %c, <16 x bfloat> %a, <16 x bfloat> %b) { +; CHECK-LABEL: select_v16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vmsne.vi v0, v12, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, <16 x bfloat> %a, <16 x bfloat> %b + ret <16 x bfloat> %v +} + +define <16 x bfloat> @selectcc_v16bf16(bfloat %a, bfloat %b, <16 x bfloat> %c, <16 x bfloat> %d) { +; CHECK-LABEL: selectcc_v16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vmsne.vi v0, v12, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, <16 x bfloat> %c, <16 x bfloat> %d + ret <16 x bfloat> %v +} diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll index 9f0561b394b8..d360c3f635b5 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,RV32 -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,RV64 -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,RV32ZVFHMIN -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,RV64ZVFHMIN declare <4 x i1> @llvm.vp.merge.v4i1(<4 x i1>, <4 x i1>, <4 x i1>, i32) @@ -1240,3 +1240,139 @@ define <32 x double> @vpmerge_vf_v32f64(double %a, <32 x double> %vb, <32 x i1> %v = call <32 x double> @llvm.vp.merge.v32f64(<32 x i1> %m, <32 x double> %va, <32 x double> %vb, i32 %evl) ret <32 x double> %v } + +declare <2 x bfloat> @llvm.vp.merge.v2bf16(<2 x i1>, <2 x bfloat>, <2 x bfloat>, i32) + +define <2 x bfloat> @vpmerge_vv_v2bf16(<2 x bfloat> %va, <2 x bfloat> %vb, <2 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_v2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf4, tu, ma +; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call <2 x bfloat> @llvm.vp.merge.v2bf16(<2 x i1> %m, <2 x bfloat> %va, <2 x bfloat> %vb, i32 %evl) + ret <2 x bfloat> %v +} + +define <2 x bfloat> @vpmerge_vf_v2bf16(bfloat %a, <2 x bfloat> %vb, <2 x i1> %m, i32 zeroext %evl) { +; ZVFH-LABEL: vpmerge_vf_v2bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, mf4, tu, ma +; ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vpmerge_vf_v2bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a1, zero, e32, mf2, ta, ma +; ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, tu, mu +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFHMIN-NEXT: ret + %elt.head = insertelement <2 x bfloat> poison, bfloat %a, i32 0 + %va = shufflevector <2 x bfloat> %elt.head, <2 x bfloat> poison, <2 x i32> zeroinitializer + %v = call <2 x bfloat> @llvm.vp.merge.v2bf16(<2 x i1> %m, <2 x bfloat> %va, <2 x bfloat> %vb, i32 %evl) + ret <2 x bfloat> %v +} + +declare <4 x bfloat> @llvm.vp.merge.v4bf16(<4 x i1>, <4 x bfloat>, <4 x bfloat>, i32) + +define <4 x bfloat> @vpmerge_vv_v4bf16(<4 x bfloat> %va, <4 x bfloat> %vb, <4 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_v4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, tu, ma +; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call <4 x bfloat> @llvm.vp.merge.v4bf16(<4 x i1> %m, <4 x bfloat> %va, <4 x bfloat> %vb, i32 %evl) + ret <4 x bfloat> %v +} + +define <4 x bfloat> @vpmerge_vf_v4bf16(bfloat %a, <4 x bfloat> %vb, <4 x i1> %m, i32 zeroext %evl) { +; ZVFH-LABEL: vpmerge_vf_v4bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, mf2, tu, ma +; ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vpmerge_vf_v4bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a1, zero, e32, m1, ta, ma +; ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf2, tu, mu +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFHMIN-NEXT: ret + %elt.head = insertelement <4 x bfloat> poison, bfloat %a, i32 0 + %va = shufflevector <4 x bfloat> %elt.head, <4 x bfloat> poison, <4 x i32> zeroinitializer + %v = call <4 x bfloat> @llvm.vp.merge.v4bf16(<4 x i1> %m, <4 x bfloat> %va, <4 x bfloat> %vb, i32 %evl) + ret <4 x bfloat> %v +} + +declare <8 x bfloat> @llvm.vp.merge.v8bf16(<8 x i1>, <8 x bfloat>, <8 x bfloat>, i32) + +define <8 x bfloat> @vpmerge_vv_v8bf16(<8 x bfloat> %va, <8 x bfloat> %vb, <8 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_v8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m1, tu, ma +; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call <8 x bfloat> @llvm.vp.merge.v8bf16(<8 x i1> %m, <8 x bfloat> %va, <8 x bfloat> %vb, i32 %evl) + ret <8 x bfloat> %v +} + +define <8 x bfloat> @vpmerge_vf_v8bf16(bfloat %a, <8 x bfloat> %vb, <8 x i1> %m, i32 zeroext %evl) { +; ZVFH-LABEL: vpmerge_vf_v8bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, m1, tu, ma +; ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vpmerge_vf_v8bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a1, zero, e32, m2, ta, ma +; ZVFHMIN-NEXT: vfmv.v.f v10, fa5 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m1, tu, mu +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t +; ZVFHMIN-NEXT: ret + %elt.head = insertelement <8 x bfloat> poison, bfloat %a, i32 0 + %va = shufflevector <8 x bfloat> %elt.head, <8 x bfloat> poison, <8 x i32> zeroinitializer + %v = call <8 x bfloat> @llvm.vp.merge.v8bf16(<8 x i1> %m, <8 x bfloat> %va, <8 x bfloat> %vb, i32 %evl) + ret <8 x bfloat> %v +} + +declare <16 x bfloat> @llvm.vp.merge.v16bf16(<16 x i1>, <16 x bfloat>, <16 x bfloat>, i32) + +define <16 x bfloat> @vpmerge_vv_v16bf16(<16 x bfloat> %va, <16 x bfloat> %vb, <16 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_v16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m2, tu, ma +; CHECK-NEXT: vmerge.vvm v10, v10, v8, v0 +; CHECK-NEXT: vmv2r.v v8, v10 +; CHECK-NEXT: ret + %v = call <16 x bfloat> @llvm.vp.merge.v16bf16(<16 x i1> %m, <16 x bfloat> %va, <16 x bfloat> %vb, i32 %evl) + ret <16 x bfloat> %v +} + +define <16 x bfloat> @vpmerge_vf_v16bf16(bfloat %a, <16 x bfloat> %vb, <16 x i1> %m, i32 zeroext %evl) { +; ZVFH-LABEL: vpmerge_vf_v16bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, m2, tu, ma +; ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vpmerge_vf_v16bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a1, zero, e32, m4, ta, ma +; ZVFHMIN-NEXT: vfmv.v.f v12, fa5 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m2, tu, mu +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t +; ZVFHMIN-NEXT: ret + %elt.head = insertelement <16 x bfloat> poison, bfloat %a, i32 0 + %va = shufflevector <16 x bfloat> %elt.head, <16 x bfloat> poison, <16 x i32> zeroinitializer + %v = call <16 x bfloat> @llvm.vp.merge.v16bf16(<16 x i1> %m, <16 x bfloat> %va, <16 x bfloat> %vb, i32 %evl) + ret <16 x bfloat> %v +} diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll index 0a2ed3eb1ffb..c5d9cdacae74 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d -riscv-v-vector-bits-min=128 \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d -riscv-v-vector-bits-min=128 \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d -riscv-v-vector-bits-min=128 \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d -riscv-v-vector-bits-min=128 \ ; RUN: -verify-machineinstrs < %s | FileCheck %s declare <1 x i1> @llvm.vp.select.v1i1(<1 x i1>, <1 x i1>, <1 x i1>, i32) @@ -683,3 +683,51 @@ define <16 x double> @select_v16f64(<16 x i1> %a, <16 x double> %b, <16 x double %v = call <16 x double> @llvm.vp.select.v16f64(<16 x i1> %a, <16 x double> %b, <16 x double> %c, i32 %evl) ret <16 x double> %v } + +declare <2 x bfloat> @llvm.vp.select.v2bf16(<2 x i1>, <2 x bfloat>, <2 x bfloat>, i32) + +define <2 x bfloat> @select_v2bf16(<2 x i1> %a, <2 x bfloat> %b, <2 x bfloat> %c, i32 zeroext %evl) { +; CHECK-LABEL: select_v2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = call <2 x bfloat> @llvm.vp.select.v2bf16(<2 x i1> %a, <2 x bfloat> %b, <2 x bfloat> %c, i32 %evl) + ret <2 x bfloat> %v +} + +declare <4 x bfloat> @llvm.vp.select.v4bf16(<4 x i1>, <4 x bfloat>, <4 x bfloat>, i32) + +define <4 x bfloat> @select_v4bf16(<4 x i1> %a, <4 x bfloat> %b, <4 x bfloat> %c, i32 zeroext %evl) { +; CHECK-LABEL: select_v4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = call <4 x bfloat> @llvm.vp.select.v4bf16(<4 x i1> %a, <4 x bfloat> %b, <4 x bfloat> %c, i32 %evl) + ret <4 x bfloat> %v +} + +declare <8 x bfloat> @llvm.vp.select.v8bf16(<8 x i1>, <8 x bfloat>, <8 x bfloat>, i32) + +define <8 x bfloat> @select_v8bf16(<8 x i1> %a, <8 x bfloat> %b, <8 x bfloat> %c, i32 zeroext %evl) { +; CHECK-LABEL: select_v8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = call <8 x bfloat> @llvm.vp.select.v8bf16(<8 x i1> %a, <8 x bfloat> %b, <8 x bfloat> %c, i32 %evl) + ret <8 x bfloat> %v +} + +declare <16 x bfloat> @llvm.vp.select.v16bf16(<16 x i1>, <16 x bfloat>, <16 x bfloat>, i32) + +define <16 x bfloat> @select_v16bf16(<16 x i1> %a, <16 x bfloat> %b, <16 x bfloat> %c, i32 zeroext %evl) { +; CHECK-LABEL: select_v16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %v = call <16 x bfloat> @llvm.vp.select.v16bf16(<16 x i1> %a, <16 x bfloat> %b, <16 x bfloat> %c, i32 %evl) + ret <16 x bfloat> %v +} diff --git a/llvm/test/CodeGen/RISCV/rvv/select-fp.ll b/llvm/test/CodeGen/RISCV/rvv/select-fp.ll index f8581d8e21b3..2b9d847a9e87 100644 --- a/llvm/test/CodeGen/RISCV/rvv/select-fp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/select-fp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s define @select_nxv1f16(i1 zeroext %c, %a, %b) { @@ -427,3 +427,183 @@ define @selectcc_nxv8f64(double %a, double %b, %c, %d ret %v } + +define @select_nxv1bf16(i1 zeroext %c, %a, %b) { +; CHECK-LABEL: select_nxv1bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, %a, %b + ret %v +} + +define @selectcc_nxv1bf16(bfloat %a, bfloat %b, %c, %d) { +; CHECK-LABEL: selectcc_nxv1bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, %c, %d + ret %v +} + +define @select_nxv2bf16(i1 zeroext %c, %a, %b) { +; CHECK-LABEL: select_nxv2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, %a, %b + ret %v +} + +define @selectcc_nxv2bf16(bfloat %a, bfloat %b, %c, %d) { +; CHECK-LABEL: selectcc_nxv2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, %c, %d + ret %v +} + +define @select_nxv4bf16(i1 zeroext %c, %a, %b) { +; CHECK-LABEL: select_nxv4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, %a, %b + ret %v +} + +define @selectcc_nxv4bf16(bfloat %a, bfloat %b, %c, %d) { +; CHECK-LABEL: selectcc_nxv4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, %c, %d + ret %v +} + +define @select_nxv8bf16(i1 zeroext %c, %a, %b) { +; CHECK-LABEL: select_nxv8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vmsne.vi v0, v12, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, %a, %b + ret %v +} + +define @selectcc_nxv8bf16(bfloat %a, bfloat %b, %c, %d) { +; CHECK-LABEL: selectcc_nxv8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vmsne.vi v0, v12, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, %c, %d + ret %v +} + +define @select_nxv16bf16(i1 zeroext %c, %a, %b) { +; CHECK-LABEL: select_nxv16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m2, ta, ma +; CHECK-NEXT: vmv.v.x v16, a0 +; CHECK-NEXT: vmsne.vi v0, v16, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, %a, %b + ret %v +} + +define @selectcc_nxv16bf16(bfloat %a, bfloat %b, %c, %d) { +; CHECK-LABEL: selectcc_nxv16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetvli a1, zero, e8, m2, ta, ma +; CHECK-NEXT: vmv.v.x v16, a0 +; CHECK-NEXT: vmsne.vi v0, v16, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, %c, %d + ret %v +} + +define @select_nxv32bf16(i1 zeroext %c, %a, %b) { +; CHECK-LABEL: select_nxv32bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m4, ta, ma +; CHECK-NEXT: vmv.v.x v24, a0 +; CHECK-NEXT: vmsne.vi v0, v24, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m8, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v16, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, %a, %b + ret %v +} + +define @selectcc_nxv32bf16(bfloat %a, bfloat %b, %c, %d) { +; CHECK-LABEL: selectcc_nxv32bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetvli a1, zero, e8, m4, ta, ma +; CHECK-NEXT: vmv.v.x v24, a0 +; CHECK-NEXT: vmsne.vi v0, v24, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m8, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v16, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, %c, %d + ret %v +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll index 094e6c9cc754..e33c795169fa 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFH -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVFH -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFHMIN -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVFHMIN declare @llvm.vp.merge.nxv1i1(, , , i32) @@ -1547,3 +1547,303 @@ define @vpmerge_vf_nxv8f64(double %a, @llvm.vp.merge.nxv8f64( %m, %va, %vb, i32 %evl) ret %v } + +declare @llvm.vp.merge.nxv1bf16(, , , i32) + +define @vpmerge_vv_nxv1bf16( %va, %vb, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_nxv1bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf4, tu, ma +; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call @llvm.vp.merge.nxv1bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +define @vpmerge_vf_nxv1bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { +; RV32ZVFH-LABEL: vpmerge_vf_nxv1bf16: +; RV32ZVFH: # %bb.0: +; RV32ZVFH-NEXT: vsetvli zero, a0, e16, mf4, tu, ma +; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV32ZVFH-NEXT: ret +; +; RV64ZVFH-LABEL: vpmerge_vf_nxv1bf16: +; RV64ZVFH: # %bb.0: +; RV64ZVFH-NEXT: vsetvli zero, a0, e16, mf4, tu, ma +; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV64ZVFH-NEXT: ret +; +; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv1bf16: +; RV32ZVFHMIN: # %bb.0: +; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, mf2, ta, ma +; RV32ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, tu, mu +; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; RV32ZVFHMIN-NEXT: ret +; +; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv1bf16: +; RV64ZVFHMIN: # %bb.0: +; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, mf2, ta, ma +; RV64ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, tu, mu +; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; RV64ZVFHMIN-NEXT: ret + %elt.head = insertelement poison, bfloat %a, i32 0 + %va = shufflevector %elt.head, poison, zeroinitializer + %v = call @llvm.vp.merge.nxv1bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +declare @llvm.vp.merge.nxv2bf16(, , , i32) + +define @vpmerge_vv_nxv2bf16( %va, %vb, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_nxv2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, tu, ma +; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call @llvm.vp.merge.nxv2bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +define @vpmerge_vf_nxv2bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { +; RV32ZVFH-LABEL: vpmerge_vf_nxv2bf16: +; RV32ZVFH: # %bb.0: +; RV32ZVFH-NEXT: vsetvli zero, a0, e16, mf2, tu, ma +; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV32ZVFH-NEXT: ret +; +; RV64ZVFH-LABEL: vpmerge_vf_nxv2bf16: +; RV64ZVFH: # %bb.0: +; RV64ZVFH-NEXT: vsetvli zero, a0, e16, mf2, tu, ma +; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV64ZVFH-NEXT: ret +; +; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv2bf16: +; RV32ZVFHMIN: # %bb.0: +; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m1, ta, ma +; RV32ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf2, tu, mu +; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; RV32ZVFHMIN-NEXT: ret +; +; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv2bf16: +; RV64ZVFHMIN: # %bb.0: +; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m1, ta, ma +; RV64ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf2, tu, mu +; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; RV64ZVFHMIN-NEXT: ret + %elt.head = insertelement poison, bfloat %a, i32 0 + %va = shufflevector %elt.head, poison, zeroinitializer + %v = call @llvm.vp.merge.nxv2bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +declare @llvm.vp.merge.nxv4bf16(, , , i32) + +define @vpmerge_vv_nxv4bf16( %va, %vb, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_nxv4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m1, tu, ma +; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call @llvm.vp.merge.nxv4bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +define @vpmerge_vf_nxv4bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { +; RV32ZVFH-LABEL: vpmerge_vf_nxv4bf16: +; RV32ZVFH: # %bb.0: +; RV32ZVFH-NEXT: vsetvli zero, a0, e16, m1, tu, ma +; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV32ZVFH-NEXT: ret +; +; RV64ZVFH-LABEL: vpmerge_vf_nxv4bf16: +; RV64ZVFH: # %bb.0: +; RV64ZVFH-NEXT: vsetvli zero, a0, e16, m1, tu, ma +; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV64ZVFH-NEXT: ret +; +; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv4bf16: +; RV32ZVFHMIN: # %bb.0: +; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m2, ta, ma +; RV32ZVFHMIN-NEXT: vfmv.v.f v10, fa5 +; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, m1, tu, mu +; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t +; RV32ZVFHMIN-NEXT: ret +; +; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv4bf16: +; RV64ZVFHMIN: # %bb.0: +; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m2, ta, ma +; RV64ZVFHMIN-NEXT: vfmv.v.f v10, fa5 +; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, m1, tu, mu +; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t +; RV64ZVFHMIN-NEXT: ret + %elt.head = insertelement poison, bfloat %a, i32 0 + %va = shufflevector %elt.head, poison, zeroinitializer + %v = call @llvm.vp.merge.nxv4bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +declare @llvm.vp.merge.nxv8bf16(, , , i32) + +define @vpmerge_vv_nxv8bf16( %va, %vb, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_nxv8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m2, tu, ma +; CHECK-NEXT: vmerge.vvm v10, v10, v8, v0 +; CHECK-NEXT: vmv2r.v v8, v10 +; CHECK-NEXT: ret + %v = call @llvm.vp.merge.nxv8bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +define @vpmerge_vf_nxv8bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { +; RV32ZVFH-LABEL: vpmerge_vf_nxv8bf16: +; RV32ZVFH: # %bb.0: +; RV32ZVFH-NEXT: vsetvli zero, a0, e16, m2, tu, ma +; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV32ZVFH-NEXT: ret +; +; RV64ZVFH-LABEL: vpmerge_vf_nxv8bf16: +; RV64ZVFH: # %bb.0: +; RV64ZVFH-NEXT: vsetvli zero, a0, e16, m2, tu, ma +; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV64ZVFH-NEXT: ret +; +; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv8bf16: +; RV32ZVFHMIN: # %bb.0: +; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m4, ta, ma +; RV32ZVFHMIN-NEXT: vfmv.v.f v12, fa5 +; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, m2, tu, mu +; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t +; RV32ZVFHMIN-NEXT: ret +; +; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv8bf16: +; RV64ZVFHMIN: # %bb.0: +; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m4, ta, ma +; RV64ZVFHMIN-NEXT: vfmv.v.f v12, fa5 +; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, m2, tu, mu +; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t +; RV64ZVFHMIN-NEXT: ret + %elt.head = insertelement poison, bfloat %a, i32 0 + %va = shufflevector %elt.head, poison, zeroinitializer + %v = call @llvm.vp.merge.nxv8bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +declare @llvm.vp.merge.nxv16bf16(, , , i32) + +define @vpmerge_vv_nxv16bf16( %va, %vb, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_nxv16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m4, tu, ma +; CHECK-NEXT: vmerge.vvm v12, v12, v8, v0 +; CHECK-NEXT: vmv4r.v v8, v12 +; CHECK-NEXT: ret + %v = call @llvm.vp.merge.nxv16bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +define @vpmerge_vf_nxv16bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { +; RV32ZVFH-LABEL: vpmerge_vf_nxv16bf16: +; RV32ZVFH: # %bb.0: +; RV32ZVFH-NEXT: vsetvli zero, a0, e16, m4, tu, ma +; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV32ZVFH-NEXT: ret +; +; RV64ZVFH-LABEL: vpmerge_vf_nxv16bf16: +; RV64ZVFH: # %bb.0: +; RV64ZVFH-NEXT: vsetvli zero, a0, e16, m4, tu, ma +; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV64ZVFH-NEXT: ret +; +; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv16bf16: +; RV32ZVFHMIN: # %bb.0: +; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m8, ta, ma +; RV32ZVFHMIN-NEXT: vfmv.v.f v16, fa5 +; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, tu, mu +; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; RV32ZVFHMIN-NEXT: ret +; +; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv16bf16: +; RV64ZVFHMIN: # %bb.0: +; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m8, ta, ma +; RV64ZVFHMIN-NEXT: vfmv.v.f v16, fa5 +; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, tu, mu +; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; RV64ZVFHMIN-NEXT: ret + %elt.head = insertelement poison, bfloat %a, i32 0 + %va = shufflevector %elt.head, poison, zeroinitializer + %v = call @llvm.vp.merge.nxv16bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +declare @llvm.vp.merge.nxv32bf16(, , , i32) + +define @vpmerge_vv_nxv32bf16( %va, %vb, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_nxv32bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m8, tu, ma +; CHECK-NEXT: vmerge.vvm v16, v16, v8, v0 +; CHECK-NEXT: vmv8r.v v8, v16 +; CHECK-NEXT: ret + %v = call @llvm.vp.merge.nxv32bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +define @vpmerge_vf_nxv32bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { +; RV32ZVFH-LABEL: vpmerge_vf_nxv32bf16: +; RV32ZVFH: # %bb.0: +; RV32ZVFH-NEXT: vsetvli zero, a0, e16, m8, tu, ma +; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV32ZVFH-NEXT: ret +; +; RV64ZVFH-LABEL: vpmerge_vf_nxv32bf16: +; RV64ZVFH: # %bb.0: +; RV64ZVFH-NEXT: vsetvli zero, a0, e16, m8, tu, ma +; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV64ZVFH-NEXT: ret +; +; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv32bf16: +; RV32ZVFHMIN: # %bb.0: +; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m8, ta, ma +; RV32ZVFHMIN-NEXT: vfmv.v.f v24, fa5 +; RV32ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v16, v24 +; RV32ZVFHMIN-NEXT: vmv.v.v v20, v16 +; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, m8, tu, ma +; RV32ZVFHMIN-NEXT: vmerge.vvm v8, v8, v16, v0 +; RV32ZVFHMIN-NEXT: ret +; +; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv32bf16: +; RV64ZVFHMIN: # %bb.0: +; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m8, ta, ma +; RV64ZVFHMIN-NEXT: vfmv.v.f v24, fa5 +; RV64ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v16, v24 +; RV64ZVFHMIN-NEXT: vmv.v.v v20, v16 +; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, m8, tu, ma +; RV64ZVFHMIN-NEXT: vmerge.vvm v8, v8, v16, v0 +; RV64ZVFHMIN-NEXT: ret + %elt.head = insertelement poison, bfloat %a, i32 0 + %va = shufflevector %elt.head, poison, zeroinitializer + %v = call @llvm.vp.merge.nxv32bf16( %m, %va, %vb, i32 %evl) + ret %v +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll index 53b8e4a78b75..82c2fe3273bd 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFH -; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFH -; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFHMIN -; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfhmin,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFHMIN define @vfmerge_vv_nxv1f16( %va, %vb, %cond) { @@ -512,3 +512,219 @@ define void @vselect_legalize_regression( %a, %sel, ptr %out ret void } + +define @vfmerge_vv_nxv1bf16( %va, %vb, %cond) { +; CHECK-LABEL: vfmerge_vv_nxv1bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %vc = select %cond, %va, %vb + ret %vc +} + +define @vfmerge_fv_nxv1bf16( %va, bfloat %b, %cond) { +; CHECK-ZVFH-LABEL: vfmerge_fv_nxv1bf16: +; CHECK-ZVFH: # %bb.0: +; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; CHECK-ZVFH-NEXT: ret +; +; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv1bf16: +; CHECK-ZVFHMIN: # %bb.0: +; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, mu +; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; CHECK-ZVFHMIN-NEXT: ret + %head = insertelement poison, bfloat %b, i32 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = select %cond, %splat, %va + ret %vc +} + +define @vfmerge_vv_nxv2bf16( %va, %vb, %cond) { +; CHECK-LABEL: vfmerge_vv_nxv2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %vc = select %cond, %va, %vb + ret %vc +} + +define @vfmerge_fv_nxv2bf16( %va, bfloat %b, %cond) { +; CHECK-ZVFH-LABEL: vfmerge_fv_nxv2bf16: +; CHECK-ZVFH: # %bb.0: +; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; CHECK-ZVFH-NEXT: ret +; +; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv2bf16: +; CHECK-ZVFHMIN: # %bb.0: +; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, mu +; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; CHECK-ZVFHMIN-NEXT: ret + %head = insertelement poison, bfloat %b, i32 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = select %cond, %splat, %va + ret %vc +} + +define @vfmerge_vv_nxv4bf16( %va, %vb, %cond) { +; CHECK-LABEL: vfmerge_vv_nxv4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %vc = select %cond, %va, %vb + ret %vc +} + +define @vfmerge_fv_nxv4bf16( %va, bfloat %b, %cond) { +; CHECK-ZVFH-LABEL: vfmerge_fv_nxv4bf16: +; CHECK-ZVFH: # %bb.0: +; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; CHECK-ZVFH-NEXT: ret +; +; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv4bf16: +; CHECK-ZVFHMIN: # %bb.0: +; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-ZVFHMIN-NEXT: vfmv.v.f v10, fa5 +; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, mu +; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t +; CHECK-ZVFHMIN-NEXT: ret + %head = insertelement poison, bfloat %b, i32 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = select %cond, %splat, %va + ret %vc +} + +define @vfmerge_vv_nxv8bf16( %va, %vb, %cond) { +; CHECK-LABEL: vfmerge_vv_nxv8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %vc = select %cond, %va, %vb + ret %vc +} + +define @vfmerge_fv_nxv8bf16( %va, bfloat %b, %cond) { +; CHECK-ZVFH-LABEL: vfmerge_fv_nxv8bf16: +; CHECK-ZVFH: # %bb.0: +; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; CHECK-ZVFH-NEXT: ret +; +; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv8bf16: +; CHECK-ZVFHMIN: # %bb.0: +; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-ZVFHMIN-NEXT: vfmv.v.f v12, fa5 +; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, mu +; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t +; CHECK-ZVFHMIN-NEXT: ret + %head = insertelement poison, bfloat %b, i32 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = select %cond, %splat, %va + ret %vc +} + +define @vfmerge_zv_nxv8bf16( %va, %cond) { +; CHECK-LABEL: vfmerge_zv_nxv8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vim v8, v8, 0, v0 +; CHECK-NEXT: ret + %vc = select %cond, splat (bfloat zeroinitializer), %va + ret %vc +} + +define @vmerge_truelhs_nxv8bf16_0( %va, %vb) { +; CHECK-LABEL: vmerge_truelhs_nxv8bf16_0: +; CHECK: # %bb.0: +; CHECK-NEXT: ret + %vc = select splat (i1 1), %va, %vb + ret %vc +} + +define @vmerge_falselhs_nxv8bf16_0( %va, %vb) { +; CHECK-LABEL: vmerge_falselhs_nxv8bf16_0: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv2r.v v8, v10 +; CHECK-NEXT: ret + %vc = select zeroinitializer, %va, %vb + ret %vc +} + +define @vfmerge_vv_nxv16bf16( %va, %vb, %cond) { +; CHECK-LABEL: vfmerge_vv_nxv16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0 +; CHECK-NEXT: ret + %vc = select %cond, %va, %vb + ret %vc +} + +define @vfmerge_fv_nxv16bf16( %va, bfloat %b, %cond) { +; CHECK-ZVFH-LABEL: vfmerge_fv_nxv16bf16: +; CHECK-ZVFH: # %bb.0: +; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; CHECK-ZVFH-NEXT: ret +; +; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv16bf16: +; CHECK-ZVFHMIN: # %bb.0: +; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m8, ta, ma +; CHECK-ZVFHMIN-NEXT: vfmv.v.f v16, fa5 +; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, mu +; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; CHECK-ZVFHMIN-NEXT: ret + %head = insertelement poison, bfloat %b, i32 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = select %cond, %splat, %va + ret %vc +} + +define @vfmerge_vv_nxv32bf16( %va, %vb, %cond) { +; CHECK-LABEL: vfmerge_vv_nxv32bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m8, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v16, v8, v0 +; CHECK-NEXT: ret + %vc = select %cond, %va, %vb + ret %vc +} + +define @vfmerge_fv_nxv32bf16( %va, bfloat %b, %cond) { +; CHECK-ZVFH-LABEL: vfmerge_fv_nxv32bf16: +; CHECK-ZVFH: # %bb.0: +; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, m8, ta, ma +; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; CHECK-ZVFH-NEXT: ret +; +; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv32bf16: +; CHECK-ZVFHMIN: # %bb.0: +; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m8, ta, ma +; CHECK-ZVFHMIN-NEXT: vfmv.v.f v24, fa5 +; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v16, v24 +; CHECK-ZVFHMIN-NEXT: vmv.v.v v20, v16 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e16, m8, ta, ma +; CHECK-ZVFHMIN-NEXT: vmerge.vvm v8, v8, v16, v0 +; CHECK-ZVFHMIN-NEXT: ret + %head = insertelement poison, bfloat %b, i32 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = select %cond, %splat, %va + ret %vc +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll index ee0617c93148..d1049e14fa29 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfh,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfh,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfhmin,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfhmin,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s declare @llvm.vp.select.nxv1i1(, , , i32) @@ -922,3 +922,75 @@ define @select_unknown_T_T( %x, @llvm.vp.select.nxv2i1( %x, %y, %y, i32 %evl) ret %a } + +declare @llvm.vp.select.nxv1bf16(, , , i32) + +define @select_nxv1bf16( %a, %b, %c, i32 zeroext %evl) { +; CHECK-LABEL: select_nxv1bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = call @llvm.vp.select.nxv1bf16( %a, %b, %c, i32 %evl) + ret %v +} + +declare @llvm.vp.select.nxv2bf16(, , , i32) + +define @select_nxv2bf16( %a, %b, %c, i32 zeroext %evl) { +; CHECK-LABEL: select_nxv2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = call @llvm.vp.select.nxv2bf16( %a, %b, %c, i32 %evl) + ret %v +} + +declare @llvm.vp.select.nxv4bf16(, , , i32) + +define @select_nxv4bf16( %a, %b, %c, i32 zeroext %evl) { +; CHECK-LABEL: select_nxv4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = call @llvm.vp.select.nxv4bf16( %a, %b, %c, i32 %evl) + ret %v +} + +declare @llvm.vp.select.nxv8bf16(, , , i32) + +define @select_nxv8bf16( %a, %b, %c, i32 zeroext %evl) { +; CHECK-LABEL: select_nxv8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %v = call @llvm.vp.select.nxv8bf16( %a, %b, %c, i32 %evl) + ret %v +} + +declare @llvm.vp.select.nxv16bf16(, , , i32) + +define @select_nxv16bf16( %a, %b, %c, i32 zeroext %evl) { +; CHECK-LABEL: select_nxv16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0 +; CHECK-NEXT: ret + %v = call @llvm.vp.select.nxv16bf16( %a, %b, %c, i32 %evl) + ret %v +} + +declare @llvm.vp.select.nxv32bf16(, , , i32) + +define @select_nxv32bf16( %a, %b, %c, i32 zeroext %evl) { +; CHECK-LABEL: select_nxv32bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m8, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v16, v8, v0 +; CHECK-NEXT: ret + %v = call @llvm.vp.select.nxv32bf16( %a, %b, %c, i32 %evl) + ret %v +} -- GitLab From 8e674953264e1545397d9a19a2661e157e8d6525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hendrik=20H=C3=BCbner?= <117831077+HendrikHuebner@users.noreply.github.com> Date: Thu, 6 Jun 2024 04:44:44 +0200 Subject: [PATCH 032/462] [libc][math][c23] Implement fmaxf16 and fminf16 function (#94131) Implements fmaxf16 and fminf16, which are two missing functions listed here: #93566 --- libc/config/linux/aarch64/entrypoints.txt | 2 ++ libc/config/linux/x86_64/entrypoints.txt | 2 ++ libc/docs/math/index.rst | 4 ++-- libc/spec/stdc.td | 2 ++ libc/src/math/CMakeLists.txt | 2 ++ libc/src/math/fmaxf16.h | 20 +++++++++++++++++ libc/src/math/fminf16.h | 20 +++++++++++++++++ libc/src/math/generic/CMakeLists.txt | 27 +++++++++++++++++++++++ libc/src/math/generic/fmaxf16.cpp | 19 ++++++++++++++++ libc/src/math/generic/fminf16.cpp | 19 ++++++++++++++++ libc/test/src/math/smoke/CMakeLists.txt | 26 ++++++++++++++++++++++ libc/test/src/math/smoke/fmaxf16_test.cpp | 13 +++++++++++ libc/test/src/math/smoke/fminf16_test.cpp | 13 +++++++++++ 13 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 libc/src/math/fmaxf16.h create mode 100644 libc/src/math/fminf16.h create mode 100644 libc/src/math/generic/fmaxf16.cpp create mode 100644 libc/src/math/generic/fminf16.cpp create mode 100644 libc/test/src/math/smoke/fmaxf16_test.cpp create mode 100644 libc/test/src/math/smoke/fminf16_test.cpp diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index c990a5ba9e66..2a4f789925e8 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -505,6 +505,8 @@ if(LIBC_TYPES_HAS_FLOAT16) libc.src.math.fabsf16 libc.src.math.fdimf16 libc.src.math.floorf16 + libc.src.math.fmaxf16 + libc.src.math.fminf16 libc.src.math.fromfpf16 libc.src.math.fromfpxf16 libc.src.math.llrintf16 diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 780ffb6d972c..fe121820d6b6 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -538,6 +538,8 @@ if(LIBC_TYPES_HAS_FLOAT16) libc.src.math.fabsf16 libc.src.math.fdimf16 libc.src.math.floorf16 + libc.src.math.fmaxf16 + libc.src.math.fminf16 libc.src.math.fromfpf16 libc.src.math.fromfpxf16 libc.src.math.llrintf16 diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst index fd753742b522..afadf7124565 100644 --- a/libc/docs/math/index.rst +++ b/libc/docs/math/index.rst @@ -136,7 +136,7 @@ Basic Operations +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ | floor | |check| | |check| | |check| | |check| | |check| | 7.12.9.2 | F.10.6.2 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| fmax | |check| | |check| | |check| | | |check| | 7.12.12.2 | F.10.9.2 | +| fmax | |check| | |check| | |check| | |check| | |check| | 7.12.12.2 | F.10.9.2 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ | fmaximum | |check| | |check| | |check| | | |check| | 7.12.12.4 | F.10.9.4 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ @@ -146,7 +146,7 @@ Basic Operations +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ | fmaximum_num | |check| | |check| | |check| | | |check| | 7.12.12.8 | F.10.9.5 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| fmin | |check| | |check| | |check| | | |check| | 7.12.12.3 | F.10.9.3 | +| fmin | |check| | |check| | |check| | |check| | |check| | 7.12.12.3 | F.10.9.3 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ | fminimum | |check| | |check| | |check| | | |check| | 7.12.12.5 | F.10.9.4 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 0aadeb1a43a0..3098736963d3 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -416,11 +416,13 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"fminf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fminl", RetValSpec, [ArgSpec, ArgSpec]>, GuardedFunctionSpec<"fminf128", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, + GuardedFunctionSpec<"fminf16", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, FunctionSpec<"fmax", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fmaxf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fmaxl", RetValSpec, [ArgSpec, ArgSpec]>, GuardedFunctionSpec<"fmaxf128", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, + GuardedFunctionSpec<"fmaxf16", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, FunctionSpec<"fmaximum", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fmaximumf", RetValSpec, [ArgSpec, ArgSpec]>, diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt index 5ae03b1f46c3..83ce322e8273 100644 --- a/libc/src/math/CMakeLists.txt +++ b/libc/src/math/CMakeLists.txt @@ -124,11 +124,13 @@ add_math_entrypoint_object(fmax) add_math_entrypoint_object(fmaxf) add_math_entrypoint_object(fmaxl) add_math_entrypoint_object(fmaxf128) +add_math_entrypoint_object(fmaxf16) add_math_entrypoint_object(fmin) add_math_entrypoint_object(fminf) add_math_entrypoint_object(fminl) add_math_entrypoint_object(fminf128) +add_math_entrypoint_object(fminf16) add_math_entrypoint_object(fmaximum) add_math_entrypoint_object(fmaximumf) diff --git a/libc/src/math/fmaxf16.h b/libc/src/math/fmaxf16.h new file mode 100644 index 000000000000..bf608f82c420 --- /dev/null +++ b/libc/src/math/fmaxf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for fmaxf16 -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_FMAXF16_H +#define LLVM_LIBC_SRC_MATH_FMAXF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 fmaxf16(float16 x, float16 y); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_FMAXF16_H diff --git a/libc/src/math/fminf16.h b/libc/src/math/fminf16.h new file mode 100644 index 000000000000..22d4e6c80394 --- /dev/null +++ b/libc/src/math/fminf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for fminf16 -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_FMINF16_H +#define LLVM_LIBC_SRC_MATH_FMINF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 fminf16(float16 x, float16 y); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_FMINF16_H diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt index 95904bef93d2..00e2f0abc952 100644 --- a/libc/src/math/generic/CMakeLists.txt +++ b/libc/src/math/generic/CMakeLists.txt @@ -1782,6 +1782,20 @@ add_entrypoint_object( -O3 ) +add_entrypoint_object( + fminf16 + SRCS + fminf16.cpp + HDRS + ../fminf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.basic_operations + COMPILE_OPTIONS + -O3 +) + + add_entrypoint_object( fmax SRCS @@ -1831,6 +1845,19 @@ add_entrypoint_object( -O3 ) +add_entrypoint_object( + fmaxf16 + SRCS + fmaxf16.cpp + HDRS + ../fmaxf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.basic_operations + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( fmaximum SRCS diff --git a/libc/src/math/generic/fmaxf16.cpp b/libc/src/math/generic/fmaxf16.cpp new file mode 100644 index 000000000000..c317aef3cb3b --- /dev/null +++ b/libc/src/math/generic/fmaxf16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of fmaxf16 function --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmaxf16.h" +#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, fmaxf16, (float16 x, float16 y)) { + return fputil::fmax(x, y); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/fminf16.cpp b/libc/src/math/generic/fminf16.cpp new file mode 100644 index 000000000000..12547c33c636 --- /dev/null +++ b/libc/src/math/generic/fminf16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of fminf16 function --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fminf16.h" +#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, fminf16, (float16 x, float16 y)) { + return fputil::fmin(x, y); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt index 09e54349501c..a331c02a12ea 100644 --- a/libc/test/src/math/smoke/CMakeLists.txt +++ b/libc/test/src/math/smoke/CMakeLists.txt @@ -1747,6 +1747,19 @@ add_fp_unittest( libc.src.__support.FPUtil.fp_bits ) +add_fp_unittest( + fminf16_test + SUITE + libc-math-smoke-tests + SRCS + fminf16_test.cpp + HDRS + FMinTest.h + DEPENDS + libc.src.math.fminf16 + libc.src.__support.FPUtil.fp_bits +) + add_fp_unittest( fmaxf_test SUITE @@ -1799,6 +1812,19 @@ add_fp_unittest( libc.src.__support.FPUtil.fp_bits ) +add_fp_unittest( + fmaxf16_test + SUITE + libc-math-smoke-tests + SRCS + fmaxf16_test.cpp + HDRS + FMaxTest.h + DEPENDS + libc.src.math.fmaxf16 + libc.src.__support.FPUtil.fp_bits +) + add_fp_unittest( fmaximuml_test SUITE diff --git a/libc/test/src/math/smoke/fmaxf16_test.cpp b/libc/test/src/math/smoke/fmaxf16_test.cpp new file mode 100644 index 000000000000..79c03b7685b0 --- /dev/null +++ b/libc/test/src/math/smoke/fmaxf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for fmaxf16 --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "FMaxTest.h" + +#include "src/math/fmaxf16.h" + +LIST_FMAX_TESTS(float16, LIBC_NAMESPACE::fmaxf16) diff --git a/libc/test/src/math/smoke/fminf16_test.cpp b/libc/test/src/math/smoke/fminf16_test.cpp new file mode 100644 index 000000000000..4379a6e75065 --- /dev/null +++ b/libc/test/src/math/smoke/fminf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for fminf16 ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "FMinTest.h" + +#include "src/math/fminf16.h" + +LIST_FMIN_TESTS(float16, LIBC_NAMESPACE::fminf16) -- GitLab From 6ca0f44cd89aa802c306c303764eabf83a7f5029 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Wed, 5 Jun 2024 19:47:49 -0700 Subject: [PATCH 033/462] [lldb] Fix inconsistencies in DWARFExpression errors (#94554) This patch make all errors start with a lowercase letter and removes trailing periods and newlines. This fixes inconsistencies between error messages and facilitate concatenating them. --- lldb/source/Expression/DWARFExpression.cpp | 169 ++++++++++----------- 1 file changed, 80 insertions(+), 89 deletions(-) diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp index 80d03c84fadb..4681dbafb6f9 100644 --- a/lldb/source/Expression/DWARFExpression.cpp +++ b/lldb/source/Expression/DWARFExpression.cpp @@ -973,7 +973,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_deref: { if (stack.empty()) return llvm::createStringError( - "Expression stack empty for DW_OP_deref."); + "expression stack empty for DW_OP_deref"); Value::ValueType value_type = stack.back().GetValueType(); switch (value_type) { case Value::ValueType::HostAddress: { @@ -1023,16 +1023,16 @@ llvm::Expected DWARFExpression::Evaluate( pointer_addr, error.AsCString()); } } else { - return llvm::createStringError("NULL process for DW_OP_deref.\n"); + return llvm::createStringError("NULL process for DW_OP_deref"); } } else { return llvm::createStringError( - "NULL execution context for DW_OP_deref.\n"); + "NULL execution context for DW_OP_deref"); } break; case Value::ValueType::Invalid: - return llvm::createStringError("Invalid value type for DW_OP_deref.\n"); + return llvm::createStringError("invalid value type for DW_OP_deref"); } } break; @@ -1052,7 +1052,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_deref_size: { if (stack.empty()) { return llvm::createStringError( - "Expression stack empty for DW_OP_deref_size."); + "expression stack empty for DW_OP_deref_size"); } uint8_t size = opcodes.GetU8(&offset); if (size > 8) { @@ -1164,18 +1164,17 @@ llvm::Expected DWARFExpression::Evaluate( } } else { - return llvm::createStringError( - "NULL process for DW_OP_deref_size.\n"); + return llvm::createStringError("NULL process for DW_OP_deref_size"); } } else { return llvm::createStringError( - "NULL execution context for DW_OP_deref_size.\n"); + "NULL execution context for DW_OP_deref_size"); } break; case Value::ValueType::Invalid: - return llvm::createStringError("Invalid value for DW_OP_deref_size.\n"); + return llvm::createStringError("invalid value for DW_OP_deref_size"); } } break; @@ -1196,9 +1195,7 @@ llvm::Expected DWARFExpression::Evaluate( // extended to the size of an address on the target machine before being // pushed on the expression stack. case DW_OP_xderef_size: - - return llvm::createStringError( - "Unimplemented opcode: DW_OP_xderef_size."); + return llvm::createStringError("unimplemented opcode: DW_OP_xderef_size"); // OPCODE: DW_OP_xderef // OPERANDS: none // DESCRIPTION: Provides an extended dereference mechanism. The entry at @@ -1210,7 +1207,7 @@ llvm::Expected DWARFExpression::Evaluate( // retrieved from the dereferenced address is the size of an address on the // target machine. case DW_OP_xderef: - return llvm::createStringError("Unimplemented opcode: DW_OP_xderef."); + return llvm::createStringError("unimplemented opcode: DW_OP_xderef"); // All DW_OP_constXXX opcodes have a single operand as noted below: // @@ -1263,7 +1260,7 @@ llvm::Expected DWARFExpression::Evaluate( // DESCRIPTION: duplicates the value at the top of the stack case DW_OP_dup: if (stack.empty()) { - return llvm::createStringError("Expression stack empty for DW_OP_dup."); + return llvm::createStringError("expression stack empty for DW_OP_dup"); } else stack.push_back(stack.back()); break; @@ -1273,8 +1270,7 @@ llvm::Expected DWARFExpression::Evaluate( // DESCRIPTION: pops the value at the top of the stack case DW_OP_drop: if (stack.empty()) { - return llvm::createStringError( - "Expression stack empty for DW_OP_drop."); + return llvm::createStringError("expression stack empty for DW_OP_drop"); } else stack.pop_back(); break; @@ -1286,7 +1282,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_over: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_over."); + "expression stack needs at least 2 items for DW_OP_over"); } else stack.push_back(stack[stack.size() - 2]); break; @@ -1313,7 +1309,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_swap: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_swap."); + "expression stack needs at least 2 items for DW_OP_swap"); } else { tmp = stack.back(); stack.back() = stack[stack.size() - 2]; @@ -1330,7 +1326,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_rot: if (stack.size() < 3) { return llvm::createStringError( - "Expression stack needs at least 3 items for DW_OP_rot."); + "expression stack needs at least 3 items for DW_OP_rot"); } else { size_t last_idx = stack.size() - 1; Value old_top = stack[last_idx]; @@ -1348,10 +1344,10 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_abs: if (stack.empty()) { return llvm::createStringError( - "Expression stack needs at least 1 item for DW_OP_abs."); + "expression stack needs at least 1 item for DW_OP_abs"); } else if (!stack.back().ResolveValue(exe_ctx).AbsoluteValue()) { return llvm::createStringError( - "Failed to take the absolute value of the first stack item."); + "failed to take the absolute value of the first stack item"); } break; @@ -1362,7 +1358,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_and: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_and."); + "expression stack needs at least 2 items for DW_OP_and"); } else { tmp = stack.back(); stack.pop_back(); @@ -1379,23 +1375,22 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_div: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_div."); + "expression stack needs at least 2 items for DW_OP_div"); } else { tmp = stack.back(); - if (tmp.ResolveValue(exe_ctx).IsZero()) { - return llvm::createStringError("Divide by zero."); - } else { - stack.pop_back(); - Scalar divisor, dividend; - divisor = tmp.ResolveValue(exe_ctx); - dividend = stack.back().ResolveValue(exe_ctx); - divisor.MakeSigned(); - dividend.MakeSigned(); - stack.back() = dividend / divisor; - if (!stack.back().ResolveValue(exe_ctx).IsValid()) { - return llvm::createStringError("Divide failed."); - } - } + if (tmp.ResolveValue(exe_ctx).IsZero()) + return llvm::createStringError("divide by zero"); + + stack.pop_back(); + Scalar divisor, dividend; + divisor = tmp.ResolveValue(exe_ctx); + dividend = stack.back().ResolveValue(exe_ctx); + divisor.MakeSigned(); + dividend.MakeSigned(); + stack.back() = dividend / divisor; + + if (!stack.back().ResolveValue(exe_ctx).IsValid()) + return llvm::createStringError("divide failed"); } break; @@ -1406,7 +1401,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_minus: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_minus."); + "expression stack needs at least 2 items for DW_OP_minus"); } else { tmp = stack.back(); stack.pop_back(); @@ -1423,7 +1418,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_mod: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_mod."); + "expression stack needs at least 2 items for DW_OP_mod"); } else { tmp = stack.back(); stack.pop_back(); @@ -1439,7 +1434,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_mul: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_mul."); + "expression stack needs at least 2 items for DW_OP_mul"); } else { tmp = stack.back(); stack.pop_back(); @@ -1454,10 +1449,10 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_neg: if (stack.empty()) { return llvm::createStringError( - "Expression stack needs at least 1 item for DW_OP_neg."); + "expression stack needs at least 1 item for DW_OP_neg"); } else { if (!stack.back().ResolveValue(exe_ctx).UnaryNegate()) - return llvm::createStringError("Unary negate failed."); + return llvm::createStringError("unary negate failed"); } break; @@ -1468,10 +1463,10 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_not: if (stack.empty()) { return llvm::createStringError( - "Expression stack needs at least 1 item for DW_OP_not."); + "expression stack needs at least 1 item for DW_OP_not"); } else { if (!stack.back().ResolveValue(exe_ctx).OnesComplement()) { - return llvm::createStringError("Logical NOT failed."); + return llvm::createStringError("logical NOT failed"); } } break; @@ -1483,7 +1478,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_or: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_or."); + "expression stack needs at least 2 items for DW_OP_or"); } else { tmp = stack.back(); stack.pop_back(); @@ -1499,7 +1494,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_plus: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_plus."); + "expression stack needs at least 2 items for DW_OP_plus"); } else { tmp = stack.back(); stack.pop_back(); @@ -1514,15 +1509,13 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_plus_uconst: if (stack.empty()) { return llvm::createStringError( - "Expression stack needs at least 1 item for DW_OP_plus_uconst."); + "expression stack needs at least 1 item for DW_OP_plus_uconst"); } else { const uint64_t uconst_value = opcodes.GetULEB128(&offset); // Implicit conversion from a UINT to a Scalar... stack.back().GetScalar() += uconst_value; - if (!stack.back().GetScalar().IsValid()) { - - return llvm::createStringError("DW_OP_plus_uconst failed."); - } + if (!stack.back().GetScalar().IsValid()) + return llvm::createStringError("DW_OP_plus_uconst failed"); } break; @@ -1534,7 +1527,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_shl: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_shl."); + "expression stack needs at least 2 items for DW_OP_shl"); } else { tmp = stack.back(); stack.pop_back(); @@ -1550,13 +1543,13 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_shr: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_shr."); + "expression stack needs at least 2 items for DW_OP_shr"); } else { tmp = stack.back(); stack.pop_back(); if (!stack.back().ResolveValue(exe_ctx).ShiftRightLogical( tmp.ResolveValue(exe_ctx))) { - return llvm::createStringError("DW_OP_shr failed."); + return llvm::createStringError("DW_OP_shr failed"); } } break; @@ -1570,7 +1563,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_shra: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_shra."); + "expression stack needs at least 2 items for DW_OP_shra"); } else { tmp = stack.back(); stack.pop_back(); @@ -1585,7 +1578,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_xor: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_xor."); + "expression stack needs at least 2 items for DW_OP_xor"); } else { tmp = stack.back(); stack.pop_back(); @@ -1625,7 +1618,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_bra: if (stack.empty()) { return llvm::createStringError( - "Expression stack needs at least 1 item for DW_OP_bra."); + "expression stack needs at least 1 item for DW_OP_bra"); } else { tmp = stack.back(); stack.pop_back(); @@ -1657,7 +1650,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_eq: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_eq."); + "expression stack needs at least 2 items for DW_OP_eq"); } else { tmp = stack.back(); stack.pop_back(); @@ -1676,7 +1669,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_ge: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_ge."); + "expression stack needs at least 2 items for DW_OP_ge"); } else { tmp = stack.back(); stack.pop_back(); @@ -1695,7 +1688,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_gt: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_gt."); + "expression stack needs at least 2 items for DW_OP_gt"); } else { tmp = stack.back(); stack.pop_back(); @@ -1714,7 +1707,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_le: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_le."); + "expression stack needs at least 2 items for DW_OP_le"); } else { tmp = stack.back(); stack.pop_back(); @@ -1733,7 +1726,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_lt: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_lt."); + "expression stack needs at least 2 items for DW_OP_lt"); } else { tmp = stack.back(); stack.pop_back(); @@ -1752,7 +1745,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_ne: if (stack.size() < 2) { return llvm::createStringError( - "Expression stack needs at least 2 items for DW_OP_ne."); + "expression stack needs at least 2 items for DW_OP_ne"); } else { tmp = stack.back(); stack.pop_back(); @@ -1944,11 +1937,11 @@ llvm::Expected DWARFExpression::Evaluate( return fb_err.ToError(); } else { return llvm::createStringError( - "Invalid stack frame in context for DW_OP_fbreg opcode."); + "invalid stack frame in context for DW_OP_fbreg opcode"); } } else { return llvm::createStringError( - "NULL execution context for DW_OP_fbreg.\n"); + "NULL execution context for DW_OP_fbreg"); } break; @@ -2098,7 +2091,7 @@ llvm::Expected DWARFExpression::Evaluate( // Reset for the next piece. dwarf4_location_description_kind = Memory; return llvm::createStringError( - "Expression stack needs at least 1 item for DW_OP_bit_piece."); + "expression stack needs at least 1 item for DW_OP_bit_piece"); } else { UpdateValueTypeFromLocationDescription( log, dwarf_cu, dwarf4_location_description_kind, &stack.back()); @@ -2109,7 +2102,7 @@ llvm::Expected DWARFExpression::Evaluate( switch (stack.back().GetValueType()) { case Value::ValueType::Invalid: return llvm::createStringError( - "unable to extract bit value from invalid value."); + "unable to extract bit value from invalid value"); case Value::ValueType::Scalar: { if (!stack.back().GetScalar().ExtractBitfield(piece_bit_size, piece_bit_offset)) { @@ -2147,7 +2140,7 @@ llvm::Expected DWARFExpression::Evaluate( if (!data) { LLDB_LOG(log, "Evaluate_DW_OP_implicit_value: could not be read data"); - return llvm::createStringError("Could not evaluate %s.", + return llvm::createStringError("could not evaluate %s", DW_OP_value_to_name(op)); } @@ -2198,7 +2191,7 @@ llvm::Expected DWARFExpression::Evaluate( // the stack by the called expression may be used as return values by prior // agreement between the calling and called expressions. case DW_OP_call2: - return llvm::createStringError("Unimplemented opcode DW_OP_call2."); + return llvm::createStringError("unimplemented opcode DW_OP_call2"); // OPCODE: DW_OP_call4 // OPERANDS: 1 // uint32_t compile unit relative offset of a DIE @@ -2219,8 +2212,7 @@ llvm::Expected DWARFExpression::Evaluate( // the stack by the called expression may be used as return values by prior // agreement between the calling and called expressions. case DW_OP_call4: - - return llvm::createStringError("Unimplemented opcode DW_OP_call4."); + return llvm::createStringError("unimplemented opcode DW_OP_call4"); // OPCODE: DW_OP_stack_value // OPERANDS: None @@ -2229,10 +2221,9 @@ llvm::Expected DWARFExpression::Evaluate( // value to be used. This is the actual object value and not the location. case DW_OP_stack_value: dwarf4_location_description_kind = Implicit; - if (stack.empty()) { + if (stack.empty()) return llvm::createStringError( - "Expression stack needs at least 1 item for DW_OP_stack_value."); - } + "expression stack needs at least 1 item for DW_OP_stack_value"); stack.back().SetValueType(Value::ValueType::Scalar); break; @@ -2246,7 +2237,7 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_convert: { if (stack.size() < 1) { return llvm::createStringError( - "Expression stack needs at least 1 item for DW_OP_convert."); + "expression stack needs at least 1 item for DW_OP_convert"); } const uint64_t die_offset = opcodes.GetULEB128(&offset); uint64_t bit_size; @@ -2256,7 +2247,7 @@ llvm::Expected DWARFExpression::Evaluate( // machine and an unspecified signedness. Scalar has no // "unspecified signedness", so we use unsigned types. if (!module_sp) - return llvm::createStringError("No module"); + return llvm::createStringError("no module"); sign = false; bit_size = module_sp->GetArchitecture().GetAddressByteSize() * 8; if (!bit_size) @@ -2269,7 +2260,7 @@ llvm::Expected DWARFExpression::Evaluate( DWARFDIE die = const_cast(dwarf_cu)->GetDIE(abs_die_offset); if (!die) return llvm::createStringError( - "Cannot resolve DW_OP_convert type DIE"); + "cannot resolve DW_OP_convert type DIE"); uint64_t encoding = die.GetAttributeValueAsUnsigned(DW_AT_encoding, DW_ATE_hi_user); bit_size = die.GetAttributeValueAsUnsigned(DW_AT_byte_size, 0) * 8; @@ -2277,7 +2268,7 @@ llvm::Expected DWARFExpression::Evaluate( bit_size = die.GetAttributeValueAsUnsigned(DW_AT_bit_size, 0); if (!bit_size) return llvm::createStringError( - "Unsupported type size in DW_OP_convert"); + "unsupported type size in DW_OP_convert"); switch (encoding) { case DW_ATE_signed: case DW_ATE_signed_char: @@ -2289,7 +2280,7 @@ llvm::Expected DWARFExpression::Evaluate( break; default: return llvm::createStringError( - "Unsupported encoding in DW_OP_convert"); + "unsupported encoding in DW_OP_convert"); } } Scalar &top = stack.back().ResolveValue(exe_ctx); @@ -2313,13 +2304,13 @@ llvm::Expected DWARFExpression::Evaluate( stack.back().SetValueType(Value::ValueType::LoadAddress); } else { return llvm::createStringError( - "Stack frame does not include a canonical " + "stack frame does not include a canonical " "frame address for DW_OP_call_frame_cfa " - "opcode."); + "opcode"); } } else { - return llvm::createStringError("Invalid stack frame in context for " - "DW_OP_call_frame_cfa opcode."); + return llvm::createStringError("unvalid stack frame in context for " + "DW_OP_call_frame_cfa opcode"); } break; @@ -2334,18 +2325,18 @@ llvm::Expected DWARFExpression::Evaluate( if (stack.size() < 1) { if (op == DW_OP_form_tls_address) return llvm::createStringError( - "DW_OP_form_tls_address needs an argument."); + "DW_OP_form_tls_address needs an argument"); else return llvm::createStringError( - "DW_OP_GNU_push_tls_address needs an argument."); + "DW_OP_GNU_push_tls_address needs an argument"); } if (!exe_ctx || !module_sp) - return llvm::createStringError("No context to evaluate TLS within."); + return llvm::createStringError("no context to evaluate TLS within"); Thread *thread = exe_ctx->GetThreadPtr(); if (!thread) - return llvm::createStringError("No thread to evaluate TLS within."); + return llvm::createStringError("no thread to evaluate TLS within"); // Lookup the TLS block address for this thread and module. const addr_t tls_file_addr = @@ -2355,7 +2346,7 @@ llvm::Expected DWARFExpression::Evaluate( if (tls_load_addr == LLDB_INVALID_ADDRESS) return llvm::createStringError( - "No TLS data currently exists for this thread."); + "no TLS data currently exists for this thread"); stack.back().GetScalar() = tls_load_addr; stack.back().SetValueType(Value::ValueType::LoadAddress); @@ -2430,7 +2421,7 @@ llvm::Expected DWARFExpression::Evaluate( if (pieces.GetBuffer().GetByteSize()) return pieces; - return llvm::createStringError("Stack empty after evaluation."); + return llvm::createStringError("stack empty after evaluation"); } UpdateValueTypeFromLocationDescription( -- GitLab From 63cda2d19c310826722e8724649ceae7307389d7 Mon Sep 17 00:00:00 2001 From: OverMighty Date: Thu, 6 Jun 2024 05:06:48 +0200 Subject: [PATCH 034/462] [libc][math][c23] Add {nextafter,nexttoward,nextup,nextdown}f16 C23 math functions (#94535) #93566 --- libc/config/linux/aarch64/entrypoints.txt | 4 + libc/config/linux/x86_64/entrypoints.txt | 4 + libc/docs/c23.rst | 4 +- libc/docs/math/index.rst | 8 +- libc/spec/stdc.td | 4 + libc/src/math/CMakeLists.txt | 4 + libc/src/math/generic/CMakeLists.txt | 52 +++++++++++ libc/src/math/generic/nextafterf16.cpp | 19 ++++ libc/src/math/generic/nextdownf16.cpp | 19 ++++ libc/src/math/generic/nexttowardf16.cpp | 21 +++++ libc/src/math/generic/nextupf16.cpp | 19 ++++ libc/src/math/nextafterf16.h | 20 ++++ libc/src/math/nextdownf16.h | 20 ++++ libc/src/math/nexttowardf16.h | 20 ++++ libc/src/math/nextupf16.h | 20 ++++ libc/test/src/math/smoke/CMakeLists.txt | 92 ++++++++++++++++--- libc/test/src/math/smoke/NextAfterTest.h | 10 +- libc/test/src/math/smoke/NextTowardTest.h | 11 +-- .../test/src/math/smoke/nextafterf16_test.cpp | 13 +++ libc/test/src/math/smoke/nextdownf16_test.cpp | 13 +++ .../src/math/smoke/nexttowardf16_test.cpp | 13 +++ libc/test/src/math/smoke/nextupf16_test.cpp | 13 +++ 22 files changed, 371 insertions(+), 32 deletions(-) create mode 100644 libc/src/math/generic/nextafterf16.cpp create mode 100644 libc/src/math/generic/nextdownf16.cpp create mode 100644 libc/src/math/generic/nexttowardf16.cpp create mode 100644 libc/src/math/generic/nextupf16.cpp create mode 100644 libc/src/math/nextafterf16.h create mode 100644 libc/src/math/nextdownf16.h create mode 100644 libc/src/math/nexttowardf16.h create mode 100644 libc/src/math/nextupf16.h create mode 100644 libc/test/src/math/smoke/nextafterf16_test.cpp create mode 100644 libc/test/src/math/smoke/nextdownf16_test.cpp create mode 100644 libc/test/src/math/smoke/nexttowardf16_test.cpp create mode 100644 libc/test/src/math/smoke/nextupf16_test.cpp diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index 2a4f789925e8..a8cb468b3cba 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -514,6 +514,10 @@ if(LIBC_TYPES_HAS_FLOAT16) libc.src.math.lrintf16 libc.src.math.lroundf16 libc.src.math.nearbyintf16 + libc.src.math.nextafterf16 + libc.src.math.nextdownf16 + libc.src.math.nexttowardf16 + libc.src.math.nextupf16 libc.src.math.rintf16 libc.src.math.roundf16 libc.src.math.roundevenf16 diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index fe121820d6b6..9db044e953f3 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -547,6 +547,10 @@ if(LIBC_TYPES_HAS_FLOAT16) libc.src.math.lrintf16 libc.src.math.lroundf16 libc.src.math.nearbyintf16 + libc.src.math.nextafterf16 + libc.src.math.nextdownf16 + libc.src.math.nexttowardf16 + libc.src.math.nextupf16 libc.src.math.rintf16 libc.src.math.roundf16 libc.src.math.roundevenf16 diff --git a/libc/docs/c23.rst b/libc/docs/c23.rst index 5bbb056ec5c7..fa7a59893f89 100644 --- a/libc/docs/c23.rst +++ b/libc/docs/c23.rst @@ -59,8 +59,8 @@ Additions: * ufromfp* |check| * fromfpx* |check| * ufromfpx* |check| - * nextup* - * nextdown* + * nextup* |check| + * nextdown* |check| * canonicalize* |check| * fmaximum* * fminimum* diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst index afadf7124565..8c3c07a35b93 100644 --- a/libc/docs/math/index.rst +++ b/libc/docs/math/index.rst @@ -190,13 +190,13 @@ Basic Operations +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ | nearbyint | |check| | |check| | |check| | |check| | |check| | 7.12.9.3 | F.10.6.3 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| nextafter | |check| | |check| | |check| | | |check| | 7.12.11.3 | F.10.8.3 | +| nextafter | |check| | |check| | |check| | |check| | |check| | 7.12.11.3 | F.10.8.3 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| nextdown | |check| | |check| | |check| | | |check| | 7.12.11.6 | F.10.8.6 | +| nextdown | |check| | |check| | |check| | |check| | |check| | 7.12.11.6 | F.10.8.6 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| nexttoward | |check| | |check| | |check| | | N/A | 7.12.11.4 | F.10.8.4 | +| nexttoward | |check| | |check| | |check| | |check| | N/A | 7.12.11.4 | F.10.8.4 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| nextup | |check| | |check| | |check| | | |check| | 7.12.11.5 | F.10.8.5 | +| nextup | |check| | |check| | |check| | |check| | |check| | 7.12.11.5 | F.10.8.5 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ | remainder | |check| | |check| | |check| | | | 7.12.10.2 | F.10.7.2 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 3098736963d3..f867a3d27403 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -634,20 +634,24 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"nextafterf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"nextafter", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"nextafterl", RetValSpec, [ArgSpec, ArgSpec]>, + GuardedFunctionSpec<"nextafterf16", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, GuardedFunctionSpec<"nextafterf128", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, FunctionSpec<"nexttowardf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"nexttoward", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"nexttowardl", RetValSpec, [ArgSpec, ArgSpec]>, + GuardedFunctionSpec<"nexttowardf16", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, FunctionSpec<"nextdown", RetValSpec, [ArgSpec]>, FunctionSpec<"nextdownf", RetValSpec, [ArgSpec]>, FunctionSpec<"nextdownl", RetValSpec, [ArgSpec]>, + GuardedFunctionSpec<"nextdownf16", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, GuardedFunctionSpec<"nextdownf128", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, FunctionSpec<"nextup", RetValSpec, [ArgSpec]>, FunctionSpec<"nextupf", RetValSpec, [ArgSpec]>, FunctionSpec<"nextupl", RetValSpec, [ArgSpec]>, + GuardedFunctionSpec<"nextupf16", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, GuardedFunctionSpec<"nextupf128", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, FunctionSpec<"powf", RetValSpec, [ArgSpec, ArgSpec]>, diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt index 83ce322e8273..498466692fcc 100644 --- a/libc/src/math/CMakeLists.txt +++ b/libc/src/math/CMakeLists.txt @@ -272,20 +272,24 @@ add_math_entrypoint_object(nearbyintf128) add_math_entrypoint_object(nextafter) add_math_entrypoint_object(nextafterf) add_math_entrypoint_object(nextafterl) +add_math_entrypoint_object(nextafterf16) add_math_entrypoint_object(nextafterf128) add_math_entrypoint_object(nexttoward) add_math_entrypoint_object(nexttowardf) add_math_entrypoint_object(nexttowardl) +add_math_entrypoint_object(nexttowardf16) add_math_entrypoint_object(nextdown) add_math_entrypoint_object(nextdownf) add_math_entrypoint_object(nextdownl) +add_math_entrypoint_object(nextdownf16) add_math_entrypoint_object(nextdownf128) add_math_entrypoint_object(nextup) add_math_entrypoint_object(nextupf) add_math_entrypoint_object(nextupl) +add_math_entrypoint_object(nextupf16) add_math_entrypoint_object(nextupf128) add_math_entrypoint_object(pow) diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt index 00e2f0abc952..c183f09fa5e3 100644 --- a/libc/src/math/generic/CMakeLists.txt +++ b/libc/src/math/generic/CMakeLists.txt @@ -2550,6 +2550,19 @@ add_entrypoint_object( -O3 ) +add_entrypoint_object( + nextafterf16 + SRCS + nextafterf16.cpp + HDRS + ../nextafterf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.manipulation_functions + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( nextafterf128 SRCS @@ -2599,6 +2612,19 @@ add_entrypoint_object( -O3 ) +add_entrypoint_object( + nexttowardf16 + SRCS + nexttowardf16.cpp + HDRS + ../nexttowardf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.manipulation_functions + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( nextdown SRCS @@ -2635,6 +2661,19 @@ add_entrypoint_object( -O3 ) +add_entrypoint_object( + nextdownf16 + SRCS + nextdownf16.cpp + HDRS + ../nextdownf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.manipulation_functions + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( nextdownf128 SRCS @@ -2684,6 +2723,19 @@ add_entrypoint_object( -O3 ) +add_entrypoint_object( + nextupf16 + SRCS + nextupf16.cpp + HDRS + ../nextupf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.manipulation_functions + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( nextupf128 SRCS diff --git a/libc/src/math/generic/nextafterf16.cpp b/libc/src/math/generic/nextafterf16.cpp new file mode 100644 index 000000000000..144b3fc61461 --- /dev/null +++ b/libc/src/math/generic/nextafterf16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of nextafterf16 function ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/nextafterf16.h" +#include "src/__support/FPUtil/ManipulationFunctions.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, nextafterf16, (float16 x, float16 y)) { + return fputil::nextafter(x, y); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/nextdownf16.cpp b/libc/src/math/generic/nextdownf16.cpp new file mode 100644 index 000000000000..9fdaa9dafdd8 --- /dev/null +++ b/libc/src/math/generic/nextdownf16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of nextdownf16 function ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/nextdownf16.h" +#include "src/__support/FPUtil/ManipulationFunctions.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, nextdownf16, (float16 x)) { + return fputil::nextupdown(x); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/nexttowardf16.cpp b/libc/src/math/generic/nexttowardf16.cpp new file mode 100644 index 000000000000..d1d78e8f22d3 --- /dev/null +++ b/libc/src/math/generic/nexttowardf16.cpp @@ -0,0 +1,21 @@ +//===-- Implementation of nexttowardf16 function --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/nexttowardf16.h" +#include "src/__support/FPUtil/ManipulationFunctions.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, nexttowardf16, (float16 x, long double y)) { + // We can reuse the nextafter implementation because the internal nextafter is + // templated on the types of the arguments. + return fputil::nextafter(x, y); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/nextupf16.cpp b/libc/src/math/generic/nextupf16.cpp new file mode 100644 index 000000000000..5d3d52c94068 --- /dev/null +++ b/libc/src/math/generic/nextupf16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of nextupf16 function ------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/nextupf16.h" +#include "src/__support/FPUtil/ManipulationFunctions.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, nextupf16, (float16 x)) { + return fputil::nextupdown(x); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/nextafterf16.h b/libc/src/math/nextafterf16.h new file mode 100644 index 000000000000..293569ef40c5 --- /dev/null +++ b/libc/src/math/nextafterf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for nextafterf16 ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_NEXTAFTERF16_H +#define LLVM_LIBC_SRC_MATH_NEXTAFTERF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 nextafterf16(float16 x, float16 y); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_NEXTAFTERF16_H diff --git a/libc/src/math/nextdownf16.h b/libc/src/math/nextdownf16.h new file mode 100644 index 000000000000..19137574ac92 --- /dev/null +++ b/libc/src/math/nextdownf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for nextdownf16 -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_NEXTDOWNF16_H +#define LLVM_LIBC_SRC_MATH_NEXTDOWNF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 nextdownf16(float16 x); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_NEXTDOWNF16_H diff --git a/libc/src/math/nexttowardf16.h b/libc/src/math/nexttowardf16.h new file mode 100644 index 000000000000..604eb32c2577 --- /dev/null +++ b/libc/src/math/nexttowardf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for nexttowardf16 -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_NEXTTOWARDF16_H +#define LLVM_LIBC_SRC_MATH_NEXTTOWARDF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 nexttowardf16(float16 x, long double y); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_NEXTTOWARDF16_H diff --git a/libc/src/math/nextupf16.h b/libc/src/math/nextupf16.h new file mode 100644 index 000000000000..b2973e4afc25 --- /dev/null +++ b/libc/src/math/nextupf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for nextupf16 ---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_NEXTUPF16_H +#define LLVM_LIBC_SRC_MATH_NEXTUPF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 nextupf16(float16 x); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_NEXTUPF16_H diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt index a331c02a12ea..8919b54262b0 100644 --- a/libc/test/src/math/smoke/CMakeLists.txt +++ b/libc/test/src/math/smoke/CMakeLists.txt @@ -2541,8 +2541,10 @@ add_fp_unittest( HDRS NextAfterTest.h DEPENDS + libc.hdr.fenv_macros libc.src.math.nextafter - libc.src.__support.FPUtil.basic_operations + libc.src.__support.CPP.bit + libc.src.__support.FPUtil.fenv_impl libc.src.__support.FPUtil.fp_bits ) @@ -2555,8 +2557,10 @@ add_fp_unittest( HDRS NextAfterTest.h DEPENDS + libc.hdr.fenv_macros libc.src.math.nextafterf - libc.src.__support.FPUtil.basic_operations + libc.src.__support.CPP.bit + libc.src.__support.FPUtil.fenv_impl libc.src.__support.FPUtil.fp_bits ) @@ -2569,8 +2573,26 @@ add_fp_unittest( HDRS NextAfterTest.h DEPENDS + libc.hdr.fenv_macros libc.src.math.nextafterl - libc.src.__support.FPUtil.basic_operations + libc.src.__support.CPP.bit + libc.src.__support.FPUtil.fenv_impl + libc.src.__support.FPUtil.fp_bits +) + +add_fp_unittest( + nextafterf16_test + SUITE + libc-math-smoke-tests + SRCS + nextafterf16_test.cpp + HDRS + NextAfterTest.h + DEPENDS + libc.hdr.fenv_macros + libc.src.math.nextafterf16 + libc.src.__support.CPP.bit + libc.src.__support.FPUtil.fenv_impl libc.src.__support.FPUtil.fp_bits ) @@ -2583,8 +2605,10 @@ add_fp_unittest( HDRS NextAfterTest.h DEPENDS + libc.hdr.fenv_macros libc.src.math.nextafterf128 - libc.src.__support.FPUtil.basic_operations + libc.src.__support.CPP.bit + libc.src.__support.FPUtil.fenv_impl libc.src.__support.FPUtil.fp_bits ) @@ -2599,8 +2623,10 @@ if(NOT LIBC_TARGET_OS_IS_GPU) HDRS NextTowardTest.h DEPENDS + libc.hdr.fenv_macros libc.src.math.nexttoward - libc.src.__support.FPUtil.basic_operations + libc.src.__support.CPP.bit + libc.src.__support.FPUtil.fenv_impl libc.src.__support.FPUtil.fp_bits ) @@ -2613,8 +2639,10 @@ if(NOT LIBC_TARGET_OS_IS_GPU) HDRS NextTowardTest.h DEPENDS + libc.hdr.fenv_macros libc.src.math.nexttowardf - libc.src.__support.FPUtil.basic_operations + libc.src.__support.CPP.bit + libc.src.__support.FPUtil.fenv_impl libc.src.__support.FPUtil.fp_bits ) endif() @@ -2628,8 +2656,26 @@ add_fp_unittest( HDRS NextTowardTest.h DEPENDS + libc.hdr.fenv_macros libc.src.math.nexttowardl - libc.src.__support.FPUtil.basic_operations + libc.src.__support.CPP.bit + libc.src.__support.FPUtil.fenv_impl + libc.src.__support.FPUtil.fp_bits +) + +add_fp_unittest( + nexttowardf16_test + SUITE + libc-math-smoke-tests + SRCS + nexttowardf16_test.cpp + HDRS + NextTowardTest.h + DEPENDS + libc.hdr.fenv_macros + libc.src.math.nexttowardf16 + libc.src.__support.CPP.bit + libc.src.__support.FPUtil.fenv_impl libc.src.__support.FPUtil.fp_bits ) @@ -2643,7 +2689,6 @@ add_fp_unittest( NextDownTest.h DEPENDS libc.src.math.nextdown - libc.src.__support.FPUtil.manipulation_functions ) add_fp_unittest( @@ -2656,7 +2701,6 @@ add_fp_unittest( NextDownTest.h DEPENDS libc.src.math.nextdownf - libc.src.__support.FPUtil.manipulation_functions ) add_fp_unittest( @@ -2669,7 +2713,18 @@ add_fp_unittest( NextDownTest.h DEPENDS libc.src.math.nextdownl - libc.src.__support.FPUtil.manipulation_functions +) + +add_fp_unittest( + nextdownf16_test + SUITE + libc-math-smoke-tests + SRCS + nextdownf16_test.cpp + HDRS + NextDownTest.h + DEPENDS + libc.src.math.nextdownf16 ) add_fp_unittest( @@ -2682,7 +2737,6 @@ add_fp_unittest( NextDownTest.h DEPENDS libc.src.math.nextdownf128 - libc.src.__support.FPUtil.manipulation_functions ) add_fp_unittest( @@ -2695,7 +2749,6 @@ add_fp_unittest( NextUpTest.h DEPENDS libc.src.math.nextup - libc.src.__support.FPUtil.manipulation_functions ) add_fp_unittest( @@ -2708,7 +2761,6 @@ add_fp_unittest( NextUpTest.h DEPENDS libc.src.math.nextupf - libc.src.__support.FPUtil.manipulation_functions ) add_fp_unittest( @@ -2721,7 +2773,18 @@ add_fp_unittest( NextUpTest.h DEPENDS libc.src.math.nextupl - libc.src.__support.FPUtil.manipulation_functions +) + +add_fp_unittest( + nextupf16_test + SUITE + libc-math-smoke-tests + SRCS + nextupf16_test.cpp + HDRS + NextUpTest.h + DEPENDS + libc.src.math.nextupf16 ) add_fp_unittest( @@ -2734,7 +2797,6 @@ add_fp_unittest( NextUpTest.h DEPENDS libc.src.math.nextupf128 - libc.src.__support.FPUtil.manipulation_functions ) # TODO(lntue): The current implementation of fputil::general::fma is only diff --git a/libc/test/src/math/smoke/NextAfterTest.h b/libc/test/src/math/smoke/NextAfterTest.h index d65ccdf8e70c..6278f899d8a8 100644 --- a/libc/test/src/math/smoke/NextAfterTest.h +++ b/libc/test/src/math/smoke/NextAfterTest.h @@ -9,15 +9,15 @@ #ifndef LLVM_LIBC_TEST_SRC_MATH_NEXTAFTERTEST_H #define LLVM_LIBC_TEST_SRC_MATH_NEXTAFTERTEST_H -#include "hdr/math_macros.h" #include "src/__support/CPP/bit.h" -#include "src/__support/CPP/type_traits.h" -#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/FPUtil/FEnvImpl.h" #include "src/__support/FPUtil/FPBits.h" #include "test/UnitTest/FEnvSafeTest.h" #include "test/UnitTest/FPMatcher.h" #include "test/UnitTest/Test.h" +#include "hdr/fenv_macros.h" + // TODO: Strengthen errno,exception checks and remove these assert macros // after new matchers/test fixtures are added #define ASSERT_FP_EQ_WITH_EXCEPTION(result, expected, expected_exception) \ @@ -181,7 +181,7 @@ public: result_bits = FPBits(result); ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), - x_bits.get_mantissa() + StorageType(1)); + static_cast(x_bits.get_mantissa() + StorageType(1))); x = -x; @@ -195,7 +195,7 @@ public: result_bits = FPBits(result); ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), - x_bits.get_mantissa() + StorageType(1)); + static_cast(x_bits.get_mantissa() + StorageType(1))); } }; diff --git a/libc/test/src/math/smoke/NextTowardTest.h b/libc/test/src/math/smoke/NextTowardTest.h index a24ec9ff6bd8..5992273d9190 100644 --- a/libc/test/src/math/smoke/NextTowardTest.h +++ b/libc/test/src/math/smoke/NextTowardTest.h @@ -9,16 +9,15 @@ #ifndef LLVM_LIBC_TEST_SRC_MATH_NEXTTOWARDTEST_H #define LLVM_LIBC_TEST_SRC_MATH_NEXTTOWARDTEST_H -#include "hdr/fenv_macros.h" -#include "hdr/math_macros.h" #include "src/__support/CPP/bit.h" -#include "src/__support/CPP/type_traits.h" -#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/FPUtil/FEnvImpl.h" #include "src/__support/FPUtil/FPBits.h" #include "test/UnitTest/FEnvSafeTest.h" #include "test/UnitTest/FPMatcher.h" #include "test/UnitTest/Test.h" +#include "hdr/fenv_macros.h" + // TODO: Strengthen errno,exception checks and remove these assert macros // after new matchers/test fixtures are added #define ASSERT_FP_EQ_WITH_EXCEPTION(result, expected, expected_exception) \ @@ -194,7 +193,7 @@ public: result_bits = FPBits(result); ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), - x_bits.get_mantissa() + StorageType(1)); + static_cast(x_bits.get_mantissa() + StorageType(1))); x = -x; @@ -208,7 +207,7 @@ public: result_bits = FPBits(result); ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), - x_bits.get_mantissa() + StorageType(1)); + static_cast(x_bits.get_mantissa() + StorageType(1))); } }; diff --git a/libc/test/src/math/smoke/nextafterf16_test.cpp b/libc/test/src/math/smoke/nextafterf16_test.cpp new file mode 100644 index 000000000000..860a0c74acbc --- /dev/null +++ b/libc/test/src/math/smoke/nextafterf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for nextafterf16 ----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "NextAfterTest.h" + +#include "src/math/nextafterf16.h" + +LIST_NEXTAFTER_TESTS(float16, LIBC_NAMESPACE::nextafterf16) diff --git a/libc/test/src/math/smoke/nextdownf16_test.cpp b/libc/test/src/math/smoke/nextdownf16_test.cpp new file mode 100644 index 000000000000..353f08586177 --- /dev/null +++ b/libc/test/src/math/smoke/nextdownf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for nextdownf16 -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "NextDownTest.h" + +#include "src/math/nextdownf16.h" + +LIST_NEXTDOWN_TESTS(float16, LIBC_NAMESPACE::nextdownf16) diff --git a/libc/test/src/math/smoke/nexttowardf16_test.cpp b/libc/test/src/math/smoke/nexttowardf16_test.cpp new file mode 100644 index 000000000000..8490e8de94ce --- /dev/null +++ b/libc/test/src/math/smoke/nexttowardf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for nexttowardf16 ---------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "NextTowardTest.h" + +#include "src/math/nexttowardf16.h" + +LIST_NEXTTOWARD_TESTS(float16, LIBC_NAMESPACE::nexttowardf16) diff --git a/libc/test/src/math/smoke/nextupf16_test.cpp b/libc/test/src/math/smoke/nextupf16_test.cpp new file mode 100644 index 000000000000..a146d279f3a7 --- /dev/null +++ b/libc/test/src/math/smoke/nextupf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for nextupf16 -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "NextUpTest.h" + +#include "src/math/nextupf16.h" + +LIST_NEXTUP_TESTS(float16, LIBC_NAMESPACE::nextupf16) -- GitLab From 8c9bb9c506a08b06607c7bf931a384095f0553ee Mon Sep 17 00:00:00 2001 From: Jianjian Guan Date: Thu, 6 Jun 2024 11:19:00 +0800 Subject: [PATCH 035/462] [NFC] Remove unused value (#94439) --- llvm/lib/Transforms/Utils/InlineFunction.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index eb471b259c7d..cfe63496a100 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -1221,7 +1221,6 @@ static void AddAliasScopeMetadata(CallBase &CB, ValueToValueMapTy &VMap, SmallPtrSet ObjSet; SmallVector Scopes, NoAliases; - SmallSetVector NAPtrArgs; for (const Value *V : PtrArgs) { SmallVector Objects; getUnderlyingObjects(V, Objects, /* LI = */ nullptr); -- GitLab From 86dddbe3b54eae22db6e208e6bc1c3cda9b7e149 Mon Sep 17 00:00:00 2001 From: Med Ismail Bennani Date: Wed, 5 Jun 2024 20:22:36 -0700 Subject: [PATCH 036/462] [lldb/crashlog] Always load Application Specific Backtrace Thread images (#94259) This patch changes the crashlog image loading default behaviour to not only load images from the crashed thread but also for the application specific backtrace thread. This patch also move the Application Specific Backtrace / Last Exception Backtrace tag from the thread queue field to the thread name. rdar://128276576 Signed-off-by: Med Ismail Bennani --- lldb/examples/python/crashlog.py | 10 ++++++---- lldb/examples/python/crashlog_scripted_process.py | 5 +---- .../Crashlog/app_specific_backtrace_crashlog.test | 2 +- .../Crashlog/last_exception_backtrace_crashlog.test | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/lldb/examples/python/crashlog.py b/lldb/examples/python/crashlog.py index 7c6c60e518d7..5281d6d949ba 100755 --- a/lldb/examples/python/crashlog.py +++ b/lldb/examples/python/crashlog.py @@ -547,9 +547,9 @@ class CrashLog(symbolication.Symbolicator): for image in self.images: image.resolve = True elif options.crashed_only: + images_to_load = [] for thread in self.threads: - if thread.did_crash(): - images_to_load = [] + if thread.did_crash() or thread.app_specific_backtrace: for ident in thread.idents: for image in self.find_images_with_identifier(ident): image.resolve = True @@ -864,7 +864,7 @@ class JSONCrashLogParser(CrashLogParser): thread = self.crashlog.Thread( len(self.crashlog.threads), True, self.crashlog.process_arch ) - thread.queue = "Application Specific Backtrace" + thread.name = "Application Specific Backtrace" if self.parse_asi_backtrace(thread, json_app_specific_bts[0]): self.crashlog.threads.append(thread) else: @@ -874,7 +874,7 @@ class JSONCrashLogParser(CrashLogParser): thread = self.crashlog.Thread( len(self.crashlog.threads), True, self.crashlog.process_arch ) - thread.queue = "Last Exception Backtrace" + thread.name = "Last Exception Backtrace" self.parse_frames(thread, json_last_exc_bts) self.crashlog.threads.append(thread) @@ -1174,11 +1174,13 @@ class TextCrashLogParser(CrashLogParser): self.thread = self.crashlog.Thread( idx, True, self.crashlog.process_arch ) + self.thread.name = "Application Specific Backtrace" elif line.startswith("Last Exception Backtrace:"): # iOS self.parse_mode = self.CrashLogParseMode.THREAD self.app_specific_backtrace = True idx = 1 self.thread = self.crashlog.Thread(idx, True, self.crashlog.process_arch) + self.thread.name = "Last Exception Backtrace" self.crashlog.info_lines.append(line.strip()) def parse_thread(self, line): diff --git a/lldb/examples/python/crashlog_scripted_process.py b/lldb/examples/python/crashlog_scripted_process.py index 2ee030239ee3..be0ed49d3590 100644 --- a/lldb/examples/python/crashlog_scripted_process.py +++ b/lldb/examples/python/crashlog_scripted_process.py @@ -173,10 +173,7 @@ class CrashLogScriptedThread(ScriptedThread): self.backing_thread = crashlog_thread self.idx = self.backing_thread.index self.tid = self.backing_thread.id - if self.backing_thread.app_specific_backtrace: - self.name = "Application Specific Backtrace" - else: - self.name = self.backing_thread.name + self.name = self.backing_thread.name self.queue = self.backing_thread.queue self.has_crashed = self.originating_process.crashed_thread_idx == self.idx self.create_stackframes() diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/app_specific_backtrace_crashlog.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/app_specific_backtrace_crashlog.test index c57cefdaf32d..430febb09625 100644 --- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/app_specific_backtrace_crashlog.test +++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/app_specific_backtrace_crashlog.test @@ -3,7 +3,7 @@ # RUN: mkdir -p %t.dir # RUN: yaml2obj %S/Inputs/application_specific_info/asi.yaml > %t.dir/asi # RUN: %lldb -o 'command script import lldb.macosx.crashlog' \ -# RUN: -o 'crashlog -a -i -t %t.dir/asi %S/Inputs/application_specific_info/asi.txt' \ +# RUN: -o 'crashlog -i -t %t.dir/asi %S/Inputs/application_specific_info/asi.txt' \ # RUN: -o "thread list" -o "bt all" 2>&1 | FileCheck %s # CHECK: "crashlog" {{.*}} commands have been installed, use the "--help" options on these commands diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/last_exception_backtrace_crashlog.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/last_exception_backtrace_crashlog.test index c2f61963ed0c..fa857ac0e84f 100644 --- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/last_exception_backtrace_crashlog.test +++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/last_exception_backtrace_crashlog.test @@ -3,7 +3,7 @@ # RUN: mkdir -p %t.dir # RUN: yaml2obj %S/Inputs/application_specific_info/asi.yaml > %t.dir/asi # RUN: %lldb -o 'command script import lldb.macosx.crashlog' \ -# RUN: -o 'crashlog -a -i -t %t.dir/asi %S/Inputs/application_specific_info/leb.txt' \ +# RUN: -o 'crashlog -i -t %t.dir/asi %S/Inputs/application_specific_info/leb.txt' \ # RUN: -o "thread list" -o "bt all" 2>&1 | FileCheck %s # CHECK: "crashlog" {{.*}} commands have been installed, use the "--help" options on these commands -- GitLab From c2244f8284f45471aba911f08b33cb72a6e8de9c Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Wed, 5 Jun 2024 20:28:51 -0700 Subject: [PATCH 037/462] [WebAssembly] Set IS_64 flag correctly on __indirect_function_table in object files (#94487) Follow up to #92042 --- llvm/include/llvm/MC/MCSymbolWasm.h | 11 +++++++---- llvm/lib/MC/WasmObjectWriter.cpp | 2 +- .../WebAssembly/AsmParser/WebAssemblyAsmParser.cpp | 12 ++++++++---- llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp | 3 ++- llvm/test/MC/WebAssembly/reloc-pic64.s | 1 + 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/llvm/include/llvm/MC/MCSymbolWasm.h b/llvm/include/llvm/MC/MCSymbolWasm.h index 0ce95c72a73f..0c2b97a59423 100644 --- a/llvm/include/llvm/MC/MCSymbolWasm.h +++ b/llvm/include/llvm/MC/MCSymbolWasm.h @@ -114,9 +114,11 @@ public: return isTable() && hasTableType() && getTableType().ElemType == wasm::ValType::FUNCREF; } - void setFunctionTable() { + void setFunctionTable(bool is64) { setType(wasm::WASM_SYMBOL_TYPE_TABLE); - setTableType(wasm::ValType::FUNCREF); + uint8_t flags = + is64 ? wasm::WASM_LIMITS_FLAG_IS_64 : wasm::WASM_LIMITS_FLAG_NONE; + setTableType(wasm::ValType::FUNCREF, flags); } void setUsedInGOT() const { IsUsedInGOT = true; } @@ -140,10 +142,11 @@ public: return *TableType; } void setTableType(wasm::WasmTableType TT) { TableType = TT; } - void setTableType(wasm::ValType VT) { + void setTableType(wasm::ValType VT, + uint8_t flags = wasm::WASM_LIMITS_FLAG_NONE) { // Declare a table with element type VT and no limits (min size 0, no max // size). - wasm::WasmLimits Limits = {wasm::WASM_LIMITS_FLAG_NONE, 0, 0}; + wasm::WasmLimits Limits = {flags, 0, 0}; setTableType({VT, Limits}); } }; diff --git a/llvm/lib/MC/WasmObjectWriter.cpp b/llvm/lib/MC/WasmObjectWriter.cpp index 985f9351f4a3..5207af803823 100644 --- a/llvm/lib/MC/WasmObjectWriter.cpp +++ b/llvm/lib/MC/WasmObjectWriter.cpp @@ -877,7 +877,7 @@ void WasmObjectWriter::writeImportSection(ArrayRef Imports, break; case wasm::WASM_EXTERNAL_TABLE: W->OS << char(Import.Table.ElemType); - encodeULEB128(0, W->OS); // flags + encodeULEB128(Import.Table.Limits.Flags, W->OS); encodeULEB128(NumElements, W->OS); // initial break; case wasm::WASM_EXTERNAL_TAG: diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp index 8e2063121e00..f5bc584ac4e1 100644 --- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp +++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp @@ -178,14 +178,15 @@ static wasm::WasmLimits DefaultLimits() { } static MCSymbolWasm *GetOrCreateFunctionTableSymbol(MCContext &Ctx, - const StringRef &Name) { + const StringRef &Name, + bool is64) { MCSymbolWasm *Sym = cast_or_null(Ctx.lookupSymbol(Name)); if (Sym) { if (!Sym->isFunctionTable()) Ctx.reportError(SMLoc(), "symbol is not a wasm funcref table"); } else { Sym = cast(Ctx.getOrCreateSymbol(Name)); - Sym->setFunctionTable(); + Sym->setFunctionTable(is64); // The default function table is synthesized by the linker. Sym->setUndefined(); } @@ -258,7 +259,7 @@ public: MCAsmParserExtension::Initialize(Parser); DefaultFunctionTable = GetOrCreateFunctionTableSymbol( - getContext(), "__indirect_function_table"); + getContext(), "__indirect_function_table", is64); if (!STI->checkFeatures("+reference-types")) DefaultFunctionTable->setOmitFromLinkingSection(); } @@ -508,7 +509,7 @@ public: auto &Tok = Lexer.getTok(); if (Tok.is(AsmToken::Identifier)) { auto *Sym = - GetOrCreateFunctionTableSymbol(getContext(), Tok.getString()); + GetOrCreateFunctionTableSymbol(getContext(), Tok.getString(), is64); const auto *Val = MCSymbolRefExpr::create(Sym, getContext()); *Op = std::make_unique( WebAssemblyOperand::Symbol, Tok.getLoc(), Tok.getEndLoc(), @@ -836,6 +837,9 @@ public: // symbol auto WasmSym = cast(Ctx.getOrCreateSymbol(SymName)); WasmSym->setType(wasm::WASM_SYMBOL_TYPE_TABLE); + if (is64) { + Limits.Flags |= wasm::WASM_LIMITS_FLAG_IS_64; + } wasm::WasmTableType Type = {*ElemType, Limits}; WasmSym->setTableType(Type); TOut.emitTableType(WasmSym); diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp index 5e7279808cce..c5a047ee47d7 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp @@ -108,8 +108,9 @@ MCSymbolWasm *WebAssembly::getOrCreateFunctionTableSymbol( if (!Sym->isFunctionTable()) Ctx.reportError(SMLoc(), "symbol is not a wasm funcref table"); } else { + bool is64 = Subtarget && Subtarget->getTargetTriple().isArch64Bit(); Sym = cast(Ctx.getOrCreateSymbol(Name)); - Sym->setFunctionTable(); + Sym->setFunctionTable(is64); // The default function table is synthesized by the linker. Sym->setUndefined(); } diff --git a/llvm/test/MC/WebAssembly/reloc-pic64.s b/llvm/test/MC/WebAssembly/reloc-pic64.s index 0f2ebba2a2f3..981d8f0e094c 100644 --- a/llvm/test/MC/WebAssembly/reloc-pic64.s +++ b/llvm/test/MC/WebAssembly/reloc-pic64.s @@ -93,6 +93,7 @@ hidden_func: # CHECK-NEXT: Index: 0 # CHECK-NEXT: ElemType: FUNCREF # CHECK-NEXT: Limits: +# CHECK-NEXT: Flags: [ IS_64 ] # CHECK-NEXT: Minimum: 0x1 # CHECK-NEXT: - Module: GOT.mem # CHECK-NEXT: Field: default_data -- GitLab From 97c866f6c86456b3316006e6beff47e68a81c00a Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Thu, 6 Jun 2024 11:51:05 +0800 Subject: [PATCH 038/462] [serialization] no transitive decl change (#92083) Following of https://github.com/llvm/llvm-project/pull/86912 The motivation of the patch series is that, for a module interface unit `X`, when the dependent modules of `X` changes, if the changes is not relevant with `X`, we hope the BMI of `X` won't change. For the specific patch, we hope if the changes was about irrelevant declaration changes, we hope the BMI of `X` won't change. **However**, I found the patch itself is not very useful in practice, since the adding or removing declarations, will change the state of identifiers and types in most cases. That said, for the most simple example, ``` // partA.cppm export module m:partA; // partA.v1.cppm export module m:partA; export void a() {} // partB.cppm export module m:partB; export void b() {} // m.cppm export module m; export import :partA; export import :partB; // onlyUseB; export module onlyUseB; import m; export inline void onluUseB() { b(); } ``` the BMI of `onlyUseB` will change after we change the implementation of `partA.cppm` to `partA.v1.cppm`. Since `partA.v1.cppm` introduces new identifiers and types (the function prototype). So in this patch, we have to write the tests as: ``` // partA.cppm export module m:partA; export int getA() { ... } export int getA2(int) { ... } // partA.v1.cppm export module m:partA; export int getA() { ... } export int getA(int) { ... } export int getA2(int) { ... } // partB.cppm export module m:partB; export void b() {} // m.cppm export module m; export import :partA; export import :partB; // onlyUseB; export module onlyUseB; import m; export inline void onluUseB() { b(); } ``` so that the new introduced declaration `int getA(int)` doesn't introduce new identifiers and types, then the BMI of `onlyUseB` can keep unchanged. While it looks not so great, the patch should be the base of the patch to erase the transitive change for identifiers and types since I don't know how can we introduce new types and identifiers without introducing new declarations. Given how tightly the relationship between declarations, types and identifiers, I think we can only reach the ideal state after we made the series for all of the three entties. The design of the patch is similar to https://github.com/llvm/llvm-project/pull/86912, which extends the 32-bit DeclID to 64-bit and use the higher bits to store the module file index and the lower bits to store the Local Decl ID. A slight difference is that we only use 48 bits to store the new DeclID since we try to use the higher 16 bits to store the module ID in the prefix of Decl class. Previously, we use 32 bits to store the module ID and 32 bits to store the DeclID. I don't want to allocate additional space so I tried to make the additional space the same as 64 bits. An potential interesting thing here is about the relationship between the module ID and the module file index. I feel we can get the module file index by the module ID. But I didn't prove it or implement it. Since I want to make the patch itself as small as possible. We can make it in the future if we want. Another change in the patch is the new concept Decl Index, which means the index of the very big array `DeclsLoaded` in ASTReader. Previously, the index of a loaded declaration is simply the Decl ID minus PREDEFINED_DECL_NUMs. So there are some places they got used ambiguously. But this patch tried to split these two concepts. As https://github.com/llvm/llvm-project/pull/86912 did, the change will increase the on-disk PCM file sizes. As the declaration ID may be the most IDs in the PCM file, this can have the biggest impact on the size. In my experiments, this change will bring 6.6% increase of the on-disk PCM size. No compile-time performance regression observed. Given the benefits in the motivation example, I think the cost is worthwhile. --- clang/include/clang/AST/DeclBase.h | 17 +- clang/include/clang/AST/DeclID.h | 18 +- .../include/clang/Serialization/ASTBitCodes.h | 31 +++- clang/include/clang/Serialization/ASTReader.h | 36 ++-- .../include/clang/Serialization/ModuleFile.h | 18 +- .../clang/Serialization/ModuleManager.h | 2 +- clang/lib/AST/DeclBase.cpp | 40 ++++- clang/lib/Serialization/ASTReader.cpp | 161 ++++++++++-------- clang/lib/Serialization/ASTReaderDecl.cpp | 16 +- clang/lib/Serialization/ASTWriter.cpp | 7 +- clang/lib/Serialization/ModuleFile.cpp | 3 +- .../Modules/no-transitive-decls-change.cppm | 112 ++++++++++++ 12 files changed, 305 insertions(+), 156 deletions(-) create mode 100644 clang/test/Modules/no-transitive-decls-change.cppm diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h index 600ce73c7f01..5f19af1891b7 100644 --- a/clang/include/clang/AST/DeclBase.h +++ b/clang/include/clang/AST/DeclBase.h @@ -708,10 +708,7 @@ public: /// Set the owning module ID. This may only be called for /// deserialized Decls. - void setOwningModuleID(unsigned ID) { - assert(isFromASTFile() && "Only works on a deserialized declaration"); - *((unsigned*)this - 2) = ID; - } + void setOwningModuleID(unsigned ID); public: /// Determine the availability of the given declaration. @@ -784,19 +781,11 @@ public: /// Retrieve the global declaration ID associated with this /// declaration, which specifies where this Decl was loaded from. - GlobalDeclID getGlobalID() const { - if (isFromASTFile()) - return (*((const GlobalDeclID *)this - 1)); - return GlobalDeclID(); - } + GlobalDeclID getGlobalID() const; /// Retrieve the global ID of the module that owns this particular /// declaration. - unsigned getOwningModuleID() const { - if (isFromASTFile()) - return *((const unsigned*)this - 2); - return 0; - } + unsigned getOwningModuleID() const; private: Module *getOwningModuleSlow() const; diff --git a/clang/include/clang/AST/DeclID.h b/clang/include/clang/AST/DeclID.h index 614ba06b6386..32d2ed41a374 100644 --- a/clang/include/clang/AST/DeclID.h +++ b/clang/include/clang/AST/DeclID.h @@ -19,6 +19,8 @@ #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/iterator.h" +#include + namespace clang { /// Predefined declaration IDs. @@ -107,12 +109,16 @@ public: /// /// DeclID should only be used directly in serialization. All other users /// should use LocalDeclID or GlobalDeclID. - using DeclID = uint32_t; + using DeclID = uint64_t; protected: DeclIDBase() : ID(PREDEF_DECL_NULL_ID) {} explicit DeclIDBase(DeclID ID) : ID(ID) {} + explicit DeclIDBase(unsigned LocalID, unsigned ModuleFileIndex) { + ID = (DeclID)LocalID | ((DeclID)ModuleFileIndex << 32); + } + public: DeclID get() const { return ID; } @@ -124,6 +130,10 @@ public: bool isInvalid() const { return ID == PREDEF_DECL_NULL_ID; } + unsigned getModuleFileIndex() const { return ID >> 32; } + + unsigned getLocalDeclIndex() const; + friend bool operator==(const DeclIDBase &LHS, const DeclIDBase &RHS) { return LHS.ID == RHS.ID; } @@ -156,6 +166,9 @@ public: LocalDeclID(PredefinedDeclIDs ID) : Base(ID) {} explicit LocalDeclID(DeclID ID) : Base(ID) {} + explicit LocalDeclID(unsigned LocalID, unsigned ModuleFileIndex) + : Base(LocalID, ModuleFileIndex) {} + LocalDeclID &operator++() { ++ID; return *this; @@ -175,6 +188,9 @@ public: GlobalDeclID() : Base() {} explicit GlobalDeclID(DeclID ID) : Base(ID) {} + explicit GlobalDeclID(unsigned LocalID, unsigned ModuleFileIndex) + : Base(LocalID, ModuleFileIndex) {} + // For DeclIDIterator to be able to convert a GlobalDeclID // to a LocalDeclID. explicit operator LocalDeclID() const { return LocalDeclID(this->ID); } diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h index f59ff6af4c76..3f3fb8869f5f 100644 --- a/clang/include/clang/Serialization/ASTBitCodes.h +++ b/clang/include/clang/Serialization/ASTBitCodes.h @@ -255,6 +255,12 @@ public: } }; +// The unaligned decl ID used in the Blobs of bistreams. +using unaligned_decl_id_t = + llvm::support::detail::packed_endian_specific_integral< + serialization::DeclID, llvm::endianness::native, + llvm::support::unaligned>; + /// The number of predefined preprocessed entity IDs. const unsigned int NUM_PREDEF_PP_ENTITY_IDS = 1; @@ -1980,33 +1986,46 @@ enum CleanupObjectKind { COK_Block, COK_CompoundLiteral }; /// Describes the categories of an Objective-C class. struct ObjCCategoriesInfo { - // The ID of the definition - LocalDeclID DefinitionID; + // The ID of the definition. Use unaligned_decl_id_t to keep + // ObjCCategoriesInfo 32-bit aligned. + unaligned_decl_id_t DefinitionID; // Offset into the array of category lists. unsigned Offset; + ObjCCategoriesInfo() = default; + ObjCCategoriesInfo(LocalDeclID ID, unsigned Offset) + : DefinitionID(ID.get()), Offset(Offset) {} + + LocalDeclID getDefinitionID() const { + return LocalDeclID(DefinitionID); + } + friend bool operator<(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.DefinitionID < Y.DefinitionID; + return X.getDefinitionID() < Y.getDefinitionID(); } friend bool operator>(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.DefinitionID > Y.DefinitionID; + return X.getDefinitionID() > Y.getDefinitionID(); } friend bool operator<=(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.DefinitionID <= Y.DefinitionID; + return X.getDefinitionID() <= Y.getDefinitionID(); } friend bool operator>=(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.DefinitionID >= Y.DefinitionID; + return X.getDefinitionID() >= Y.getDefinitionID(); } }; +static_assert(alignof(ObjCCategoriesInfo) <= 4); +static_assert(std::is_standard_layout_v && + std::is_trivial_v); + /// A key used when looking up entities by \ref DeclarationName. /// /// Different \ref DeclarationNames are mapped to different keys, but the diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index bea58d60d36c..0a9006223dcb 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -504,12 +504,6 @@ private: /// = I + 1 has already been loaded. llvm::PagedVector DeclsLoaded; - using GlobalDeclMapType = ContinuousRangeMap; - - /// Mapping from global declaration IDs to the module in which the - /// declaration resides. - GlobalDeclMapType GlobalDeclMap; - using FileOffset = std::pair; using FileOffsetsTy = SmallVector; using DeclUpdateOffsetsMap = llvm::DenseMap; @@ -592,10 +586,11 @@ private: struct FileDeclsInfo { ModuleFile *Mod = nullptr; - ArrayRef Decls; + ArrayRef Decls; FileDeclsInfo() = default; - FileDeclsInfo(ModuleFile *Mod, ArrayRef Decls) + FileDeclsInfo(ModuleFile *Mod, + ArrayRef Decls) : Mod(Mod), Decls(Decls) {} }; @@ -604,11 +599,7 @@ private: /// An array of lexical contents of a declaration context, as a sequence of /// Decl::Kind, DeclID pairs. - using unaligned_decl_id_t = - llvm::support::detail::packed_endian_specific_integral< - serialization::DeclID, llvm::endianness::native, - llvm::support::unaligned>; - using LexicalContents = ArrayRef; + using LexicalContents = ArrayRef; /// Map from a DeclContext to its lexical contents. llvm::DenseMap> @@ -1489,10 +1480,11 @@ private: unsigned ClientLoadCapabilities); public: - class ModuleDeclIterator : public llvm::iterator_adaptor_base< - ModuleDeclIterator, const LocalDeclID *, - std::random_access_iterator_tag, const Decl *, - ptrdiff_t, const Decl *, const Decl *> { + class ModuleDeclIterator + : public llvm::iterator_adaptor_base< + ModuleDeclIterator, const serialization::unaligned_decl_id_t *, + std::random_access_iterator_tag, const Decl *, ptrdiff_t, + const Decl *, const Decl *> { ASTReader *Reader = nullptr; ModuleFile *Mod = nullptr; @@ -1500,11 +1492,11 @@ public: ModuleDeclIterator() : iterator_adaptor_base(nullptr) {} ModuleDeclIterator(ASTReader *Reader, ModuleFile *Mod, - const LocalDeclID *Pos) + const serialization::unaligned_decl_id_t *Pos) : iterator_adaptor_base(Pos), Reader(Reader), Mod(Mod) {} value_type operator*() const { - return Reader->GetDecl(Reader->getGlobalDeclID(*Mod, *I)); + return Reader->GetDecl(Reader->getGlobalDeclID(*Mod, (LocalDeclID)*I)); } value_type operator->() const { return **this; } @@ -1544,6 +1536,9 @@ private: StringRef Arg2 = StringRef(), StringRef Arg3 = StringRef()) const; void Error(llvm::Error &&Err) const; + /// Translate a \param GlobalDeclID to the index of DeclsLoaded array. + unsigned translateGlobalDeclIDToIndex(GlobalDeclID ID) const; + public: /// Load the AST file and validate its contents against the given /// Preprocessor. @@ -1915,7 +1910,8 @@ public: /// Retrieve the module file that owns the given declaration, or NULL /// if the declaration is not from a module file. - ModuleFile *getOwningModuleFile(const Decl *D); + ModuleFile *getOwningModuleFile(const Decl *D) const; + ModuleFile *getOwningModuleFile(GlobalDeclID ID) const; /// Returns the source location for the decl \p ID. SourceLocation getSourceLocationForDeclID(GlobalDeclID ID); diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 992d26a8b88c..56193d44dd6f 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -454,23 +454,11 @@ public: /// by the declaration ID (-1). const DeclOffset *DeclOffsets = nullptr; - /// Base declaration ID for declarations local to this module. - serialization::DeclID BaseDeclID = 0; - - /// Remapping table for declaration IDs in this module. - ContinuousRangeMap DeclRemap; - - /// Mapping from the module files that this module file depends on - /// to the base declaration ID for that module as it is understood within this - /// module. - /// - /// This is effectively a reverse global-to-local mapping for declaration - /// IDs, so that we can interpret a true global ID (for this translation unit) - /// as a local ID (for this module file). - llvm::DenseMap GlobalToLocalDeclIDs; + /// Base declaration index in ASTReader for declarations local to this module. + unsigned BaseDeclIndex = 0; /// Array of file-level DeclIDs sorted by file. - const LocalDeclID *FileSortedDecls = nullptr; + const serialization::unaligned_decl_id_t *FileSortedDecls = nullptr; unsigned NumFileSortedDecls = 0; /// Array of category list location information within this diff --git a/clang/include/clang/Serialization/ModuleManager.h b/clang/include/clang/Serialization/ModuleManager.h index d770bc52eaf4..f898dab39f06 100644 --- a/clang/include/clang/Serialization/ModuleManager.h +++ b/clang/include/clang/Serialization/ModuleManager.h @@ -45,7 +45,7 @@ namespace serialization { /// Manages the set of modules loaded by an AST reader. class ModuleManager { /// The chain of AST files, in the order in which we started to load - /// them (this order isn't really useful for anything). + /// them. SmallVector, 2> Chain; /// The chain of non-module PCH files. The first entry is the one named diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp index 1e9c879e371b..7f1ed9c691e9 100644 --- a/clang/lib/AST/DeclBase.cpp +++ b/clang/lib/AST/DeclBase.cpp @@ -74,18 +74,17 @@ void *Decl::operator new(std::size_t Size, const ASTContext &Context, GlobalDeclID ID, std::size_t Extra) { // Allocate an extra 8 bytes worth of storage, which ensures that the // resulting pointer will still be 8-byte aligned. - static_assert(sizeof(unsigned) * 2 >= alignof(Decl), - "Decl won't be misaligned"); + static_assert(sizeof(uint64_t) >= alignof(Decl), "Decl won't be misaligned"); void *Start = Context.Allocate(Size + Extra + 8); void *Result = (char*)Start + 8; - unsigned *PrefixPtr = (unsigned *)Result - 2; + uint64_t *PrefixPtr = (uint64_t *)Result - 1; - // Zero out the first 4 bytes; this is used to store the owning module ID. - PrefixPtr[0] = 0; + *PrefixPtr = ID.get(); - // Store the global declaration ID in the second 4 bytes. - PrefixPtr[1] = ID.get(); + // We leave the upper 16 bits to store the module IDs. 48 bits should be + // sufficient to store a declaration ID. + assert(*PrefixPtr < llvm::maskTrailingOnes(48)); return Result; } @@ -111,6 +110,29 @@ void *Decl::operator new(std::size_t Size, const ASTContext &Ctx, return ::operator new(Size + Extra, Ctx); } +GlobalDeclID Decl::getGlobalID() const { + if (!isFromASTFile()) + return GlobalDeclID(); + // See the comments in `Decl::operator new` for details. + uint64_t ID = *((const uint64_t *)this - 1); + return GlobalDeclID(ID & llvm::maskTrailingOnes(48)); +} + +unsigned Decl::getOwningModuleID() const { + if (!isFromASTFile()) + return 0; + + uint64_t ID = *((const uint64_t *)this - 1); + return ID >> 48; +} + +void Decl::setOwningModuleID(unsigned ID) { + assert(isFromASTFile() && "Only works on a deserialized declaration"); + uint64_t *IDAddress = (uint64_t *)this - 1; + *IDAddress &= llvm::maskTrailingOnes(48); + *IDAddress |= (uint64_t)ID << 48; +} + Module *Decl::getOwningModuleSlow() const { assert(isFromASTFile() && "Not from AST file?"); return getASTContext().getExternalSource()->getModule(getOwningModuleID()); @@ -2163,3 +2185,7 @@ DependentDiagnostic *DependentDiagnostic::Create(ASTContext &C, return DD; } + +unsigned DeclIDBase::getLocalDeclIndex() const { + return ID & llvm::maskTrailingOnes(32); +} diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 33cea32c3be6..1ff770b05584 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -1658,7 +1658,7 @@ bool ASTReader::ReadSLocEntry(int ID) { unsigned NumFileDecls = Record[7]; if (NumFileDecls && ContextObj) { - const LocalDeclID *FirstDecl = F->FileSortedDecls + Record[6]; + const unaligned_decl_id_t *FirstDecl = F->FileSortedDecls + Record[6]; assert(F->FileSortedDecls && "FILE_SORTED_DECLS not encountered yet ?"); FileDeclIDs[FID] = FileDeclsInfo(F, llvm::ArrayRef(FirstDecl, NumFileDecls)); @@ -3377,26 +3377,11 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F, "duplicate DECL_OFFSET record in AST file"); F.DeclOffsets = (const DeclOffset *)Blob.data(); F.LocalNumDecls = Record[0]; - unsigned LocalBaseDeclID = Record[1]; - F.BaseDeclID = getTotalNumDecls(); - - if (F.LocalNumDecls > 0) { - // Introduce the global -> local mapping for declarations within this - // module. - GlobalDeclMap.insert(std::make_pair( - GlobalDeclID(getTotalNumDecls() + NUM_PREDEF_DECL_IDS), &F)); - - // Introduce the local -> global mapping for declarations within this - // module. - F.DeclRemap.insertOrReplace( - std::make_pair(LocalBaseDeclID, F.BaseDeclID - LocalBaseDeclID)); - - // Introduce the global -> local mapping for declarations within this - // module. - F.GlobalToLocalDeclIDs[&F] = LocalBaseDeclID; + F.BaseDeclIndex = getTotalNumDecls(); + if (F.LocalNumDecls > 0) DeclsLoaded.resize(DeclsLoaded.size() + F.LocalNumDecls); - } + break; } @@ -3631,7 +3616,7 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F, break; case FILE_SORTED_DECLS: - F.FileSortedDecls = (const LocalDeclID *)Blob.data(); + F.FileSortedDecls = (const unaligned_decl_id_t *)Blob.data(); F.NumFileSortedDecls = Record[0]; break; @@ -4058,7 +4043,6 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { RemapBuilder PreprocessedEntityRemap(F.PreprocessedEntityRemap); RemapBuilder SubmoduleRemap(F.SubmoduleRemap); RemapBuilder SelectorRemap(F.SelectorRemap); - RemapBuilder DeclRemap(F.DeclRemap); RemapBuilder TypeRemap(F.TypeRemap); auto &ImportedModuleVector = F.TransitiveImports; @@ -4097,8 +4081,6 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { endian::readNext(Data); uint32_t SelectorIDOffset = endian::readNext(Data); - uint32_t DeclIDOffset = - endian::readNext(Data); uint32_t TypeIndexOffset = endian::readNext(Data); @@ -4116,11 +4098,7 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { PreprocessedEntityRemap); mapOffset(SubmoduleIDOffset, OM->BaseSubmoduleID, SubmoduleRemap); mapOffset(SelectorIDOffset, OM->BaseSelectorID, SelectorRemap); - mapOffset(DeclIDOffset, OM->BaseDeclID, DeclRemap); mapOffset(TypeIndexOffset, OM->BaseTypeIndex, TypeRemap); - - // Global -> local mappings. - F.GlobalToLocalDeclIDs[OM] = DeclIDOffset; } } @@ -4645,7 +4623,7 @@ ASTReader::ASTReadResult ASTReader::ReadAST(StringRef FileName, ModuleKind Type, // that we load any additional categories. if (ContextObj) { for (unsigned I = 0, N = ObjCClassesLoaded.size(); I != N; ++I) { - loadObjCCategories(GlobalDeclID(ObjCClassesLoaded[I]->getGlobalID()), + loadObjCCategories(ObjCClassesLoaded[I]->getGlobalID(), ObjCClassesLoaded[I], PreviousGeneration); } } @@ -7644,18 +7622,25 @@ CXXBaseSpecifier *ASTReader::GetExternalCXXBaseSpecifiers(uint64_t Offset) { GlobalDeclID ASTReader::getGlobalDeclID(ModuleFile &F, LocalDeclID LocalID) const { - DeclID ID = LocalID.get(); - if (ID < NUM_PREDEF_DECL_IDS) - return GlobalDeclID(ID); + if (LocalID.get() < NUM_PREDEF_DECL_IDS) + return GlobalDeclID(LocalID.get()); + + unsigned OwningModuleFileIndex = LocalID.getModuleFileIndex(); + DeclID ID = LocalID.getLocalDeclIndex(); if (!F.ModuleOffsetMap.empty()) ReadModuleOffsetMap(F); - ContinuousRangeMap::iterator I = - F.DeclRemap.find(ID - NUM_PREDEF_DECL_IDS); - assert(I != F.DeclRemap.end() && "Invalid index into decl index remap"); + ModuleFile *OwningModuleFile = + OwningModuleFileIndex == 0 + ? &F + : F.TransitiveImports[OwningModuleFileIndex - 1]; + + if (OwningModuleFileIndex == 0) + ID -= NUM_PREDEF_DECL_IDS; - return GlobalDeclID(ID + I->second); + uint64_t NewModuleFileIndex = OwningModuleFile->Index + 1; + return GlobalDeclID(ID, NewModuleFileIndex); } bool ASTReader::isDeclIDFromModule(GlobalDeclID ID, ModuleFile &M) const { @@ -7663,31 +7648,33 @@ bool ASTReader::isDeclIDFromModule(GlobalDeclID ID, ModuleFile &M) const { if (ID.get() < NUM_PREDEF_DECL_IDS) return false; - return ID.get() - NUM_PREDEF_DECL_IDS >= M.BaseDeclID && - ID.get() - NUM_PREDEF_DECL_IDS < M.BaseDeclID + M.LocalNumDecls; + unsigned ModuleFileIndex = ID.getModuleFileIndex(); + return M.Index == ModuleFileIndex - 1; +} + +ModuleFile *ASTReader::getOwningModuleFile(GlobalDeclID ID) const { + // Predefined decls aren't from any module. + if (ID.get() < NUM_PREDEF_DECL_IDS) + return nullptr; + + uint64_t ModuleFileIndex = ID.getModuleFileIndex(); + assert(ModuleFileIndex && "Untranslated Local Decl?"); + + return &getModuleManager()[ModuleFileIndex - 1]; } -ModuleFile *ASTReader::getOwningModuleFile(const Decl *D) { +ModuleFile *ASTReader::getOwningModuleFile(const Decl *D) const { if (!D->isFromASTFile()) return nullptr; - GlobalDeclMapType::const_iterator I = - GlobalDeclMap.find(GlobalDeclID(D->getGlobalID())); - assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); - return I->second; + + return getOwningModuleFile(D->getGlobalID()); } SourceLocation ASTReader::getSourceLocationForDeclID(GlobalDeclID ID) { if (ID.get() < NUM_PREDEF_DECL_IDS) return SourceLocation(); - unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; - - if (Index > DeclsLoaded.size()) { - Error("declaration ID out-of-range for AST file"); - return SourceLocation(); - } - - if (Decl *D = DeclsLoaded[Index]) + if (Decl *D = GetExistingDecl(ID)) return D->getLocation(); SourceLocation Loc; @@ -7754,8 +7741,19 @@ static Decl *getPredefinedDecl(ASTContext &Context, PredefinedDeclIDs ID) { llvm_unreachable("PredefinedDeclIDs unknown enum value"); } +unsigned ASTReader::translateGlobalDeclIDToIndex(GlobalDeclID GlobalID) const { + ModuleFile *OwningModuleFile = getOwningModuleFile(GlobalID); + if (!OwningModuleFile) { + assert(GlobalID.get() < NUM_PREDEF_DECL_IDS && "Untransalted Global ID?"); + return GlobalID.get(); + } + + return OwningModuleFile->BaseDeclIndex + GlobalID.getLocalDeclIndex(); +} + Decl *ASTReader::GetExistingDecl(GlobalDeclID ID) { assert(ContextObj && "reading decl with no AST context"); + if (ID.get() < NUM_PREDEF_DECL_IDS) { Decl *D = getPredefinedDecl(*ContextObj, (PredefinedDeclIDs)ID); if (D) { @@ -7768,7 +7766,7 @@ Decl *ASTReader::GetExistingDecl(GlobalDeclID ID) { return D; } - unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; + unsigned Index = translateGlobalDeclIDToIndex(ID); if (Index >= DeclsLoaded.size()) { assert(0 && "declaration ID out-of-range for AST file"); @@ -7783,7 +7781,7 @@ Decl *ASTReader::GetDecl(GlobalDeclID ID) { if (ID.get() < NUM_PREDEF_DECL_IDS) return GetExistingDecl(ID); - unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; + unsigned Index = translateGlobalDeclIDToIndex(ID); if (Index >= DeclsLoaded.size()) { assert(0 && "declaration ID out-of-range for AST file"); @@ -7802,20 +7800,31 @@ Decl *ASTReader::GetDecl(GlobalDeclID ID) { LocalDeclID ASTReader::mapGlobalIDToModuleFileGlobalID(ModuleFile &M, GlobalDeclID GlobalID) { - DeclID ID = GlobalID.get(); - if (ID < NUM_PREDEF_DECL_IDS) + if (GlobalID.get() < NUM_PREDEF_DECL_IDS) + return LocalDeclID(GlobalID.get()); + + if (!M.ModuleOffsetMap.empty()) + ReadModuleOffsetMap(M); + + ModuleFile *Owner = getOwningModuleFile(GlobalID); + DeclID ID = GlobalID.getLocalDeclIndex(); + + if (Owner == &M) { + ID += NUM_PREDEF_DECL_IDS; return LocalDeclID(ID); + } - GlobalDeclMapType::const_iterator I = GlobalDeclMap.find(GlobalID); - assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); - ModuleFile *Owner = I->second; + uint64_t OrignalModuleFileIndex = 0; + for (unsigned I = 0; I < M.TransitiveImports.size(); I++) + if (M.TransitiveImports[I] == Owner) { + OrignalModuleFileIndex = I + 1; + break; + } - llvm::DenseMap::iterator Pos = - M.GlobalToLocalDeclIDs.find(Owner); - if (Pos == M.GlobalToLocalDeclIDs.end()) + if (!OrignalModuleFileIndex) return LocalDeclID(); - return LocalDeclID(ID - Owner->BaseDeclID + Pos->second); + return LocalDeclID(ID, OrignalModuleFileIndex); } GlobalDeclID ASTReader::ReadDeclID(ModuleFile &F, const RecordData &Record, @@ -7894,32 +7903,34 @@ void ASTReader::FindExternalLexicalDecls( namespace { -class DeclIDComp { +class UnalignedDeclIDComp { ASTReader &Reader; ModuleFile &Mod; public: - DeclIDComp(ASTReader &Reader, ModuleFile &M) : Reader(Reader), Mod(M) {} + UnalignedDeclIDComp(ASTReader &Reader, ModuleFile &M) + : Reader(Reader), Mod(M) {} - bool operator()(LocalDeclID L, LocalDeclID R) const { + bool operator()(unaligned_decl_id_t L, unaligned_decl_id_t R) const { SourceLocation LHS = getLocation(L); SourceLocation RHS = getLocation(R); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - bool operator()(SourceLocation LHS, LocalDeclID R) const { + bool operator()(SourceLocation LHS, unaligned_decl_id_t R) const { SourceLocation RHS = getLocation(R); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - bool operator()(LocalDeclID L, SourceLocation RHS) const { + bool operator()(unaligned_decl_id_t L, SourceLocation RHS) const { SourceLocation LHS = getLocation(L); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - SourceLocation getLocation(LocalDeclID ID) const { + SourceLocation getLocation(unaligned_decl_id_t ID) const { return Reader.getSourceManager().getFileLoc( - Reader.getSourceLocationForDeclID(Reader.getGlobalDeclID(Mod, ID))); + Reader.getSourceLocationForDeclID( + Reader.getGlobalDeclID(Mod, (LocalDeclID)ID))); } }; @@ -7942,8 +7953,8 @@ void ASTReader::FindFileRegionDecls(FileID File, BeginLoc = SM.getLocForStartOfFile(File).getLocWithOffset(Offset); SourceLocation EndLoc = BeginLoc.getLocWithOffset(Length); - DeclIDComp DIDComp(*this, *DInfo.Mod); - ArrayRef::iterator BeginIt = + UnalignedDeclIDComp DIDComp(*this, *DInfo.Mod); + ArrayRef::iterator BeginIt = llvm::lower_bound(DInfo.Decls, BeginLoc, DIDComp); if (BeginIt != DInfo.Decls.begin()) --BeginIt; @@ -7952,17 +7963,18 @@ void ASTReader::FindFileRegionDecls(FileID File, // to backtrack until we find it otherwise we will fail to report that the // region overlaps with an objc container. while (BeginIt != DInfo.Decls.begin() && - GetDecl(getGlobalDeclID(*DInfo.Mod, *BeginIt)) + GetDecl(getGlobalDeclID(*DInfo.Mod, (LocalDeclID)(*BeginIt))) ->isTopLevelDeclInObjCContainer()) --BeginIt; - ArrayRef::iterator EndIt = + ArrayRef::iterator EndIt = llvm::upper_bound(DInfo.Decls, EndLoc, DIDComp); if (EndIt != DInfo.Decls.end()) ++EndIt; - for (ArrayRef::iterator DIt = BeginIt; DIt != EndIt; ++DIt) - Decls.push_back(GetDecl(getGlobalDeclID(*DInfo.Mod, *DIt))); + for (ArrayRef::iterator DIt = BeginIt; DIt != EndIt; + ++DIt) + Decls.push_back(GetDecl(getGlobalDeclID(*DInfo.Mod, (LocalDeclID)(*DIt)))); } bool @@ -8169,7 +8181,6 @@ LLVM_DUMP_METHOD void ASTReader::dump() { dumpModuleIDMap("Global bit offset map", GlobalBitOffsetsMap); dumpModuleIDMap("Global source location entry map", GlobalSLocEntryMap); dumpModuleIDMap("Global type map", GlobalTypeMap); - dumpModuleIDMap("Global declaration map", GlobalDeclMap); dumpModuleIDMap("Global identifier map", GlobalIdentifierMap); dumpModuleIDMap("Global macro map", GlobalMacroMap); dumpModuleIDMap("Global submodule map", GlobalSubmoduleMap); diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp index 765c083ce8df..cf2dc32e30b9 100644 --- a/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/clang/lib/Serialization/ASTReaderDecl.cpp @@ -2924,7 +2924,7 @@ void ASTDeclReader::mergeTemplatePattern(RedeclarableTemplateDecl *D, auto *ExistingPattern = Existing->getTemplatedDecl(); RedeclarableResult Result( /*MergeWith*/ ExistingPattern, - GlobalDeclID(DPattern->getCanonicalDecl()->getGlobalID()), IsKeyDecl); + DPattern->getCanonicalDecl()->getGlobalID(), IsKeyDecl); if (auto *DClass = dyn_cast(DPattern)) { // Merge with any existing definition. @@ -3245,11 +3245,10 @@ bool ASTReader::isConsumerInterestedIn(Decl *D) { /// Get the correct cursor and offset for loading a declaration. ASTReader::RecordLocation ASTReader::DeclCursorForID(GlobalDeclID ID, SourceLocation &Loc) { - GlobalDeclMapType::iterator I = GlobalDeclMap.find(ID); - assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); - ModuleFile *M = I->second; - const DeclOffset &DOffs = - M->DeclOffsets[ID.get() - M->BaseDeclID - NUM_PREDEF_DECL_IDS]; + ModuleFile *M = getOwningModuleFile(ID); + assert(M); + unsigned LocalDeclIndex = ID.getLocalDeclIndex(); + const DeclOffset &DOffs = M->DeclOffsets[LocalDeclIndex]; Loc = ReadSourceLocation(*M, DOffs.getRawLoc()); return RecordLocation(M, DOffs.getBitOffset(M->DeclsBlockStartOffset)); } @@ -3792,7 +3791,6 @@ void ASTReader::markIncompleteDeclChain(Decl *D) { /// Read the declaration at the given offset from the AST file. Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) { - unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; SourceLocation DeclLoc; RecordLocation Loc = DeclCursorForID(ID, DeclLoc); llvm::BitstreamCursor &DeclsCursor = Loc.F->DeclsCursor; @@ -4123,7 +4121,7 @@ Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) { } assert(D && "Unknown declaration reading AST file"); - LoadedDecl(Index, D); + LoadedDecl(translateGlobalDeclIDToIndex(ID), D); // Set the DeclContext before doing any deserialization, to make sure internal // calls to Decl::getASTContext() by Decl's methods will find the // TranslationUnitDecl without crashing. @@ -4449,7 +4447,7 @@ namespace { M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap, Compare); if (Result == M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap || - Result->DefinitionID != LocalID) { + Result->getDefinitionID() != LocalID) { // We didn't find anything. If the class definition is in this module // file, then the module files it depends on cannot have any categories, // so suppress further lookup. diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 953990a4aca6..ee3e687636e6 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -3357,12 +3357,10 @@ void ASTWriter::WriteTypeDeclOffsets() { Abbrev = std::make_shared(); Abbrev->Add(BitCodeAbbrevOp(DECL_OFFSET)); Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of declarations - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // base decl ID Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // declarations block unsigned DeclOffsetAbbrev = Stream.EmitAbbrev(std::move(Abbrev)); { - RecordData::value_type Record[] = {DECL_OFFSET, DeclOffsets.size(), - FirstDeclID.get() - NUM_PREDEF_DECL_IDS}; + RecordData::value_type Record[] = {DECL_OFFSET, DeclOffsets.size()}; Stream.EmitRecordWithBlob(DeclOffsetAbbrev, Record, bytes(DeclOffsets)); } } @@ -5423,7 +5421,6 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, M.NumPreprocessedEntities); writeBaseIDOrNone(M.BaseSubmoduleID, M.LocalNumSubmodules); writeBaseIDOrNone(M.BaseSelectorID, M.LocalNumSelectors); - writeBaseIDOrNone(M.BaseDeclID, M.LocalNumDecls); writeBaseIDOrNone(M.BaseTypeIndex, M.LocalNumTypes); } } @@ -6618,13 +6615,11 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) { // Note, this will get called multiple times, once one the reader starts up // and again each time it's done reading a PCH or module. - FirstDeclID = LocalDeclID(NUM_PREDEF_DECL_IDS + Chain->getTotalNumDecls()); FirstTypeID = NUM_PREDEF_TYPE_IDS + Chain->getTotalNumTypes(); FirstIdentID = NUM_PREDEF_IDENT_IDS + Chain->getTotalNumIdentifiers(); FirstMacroID = NUM_PREDEF_MACRO_IDS + Chain->getTotalNumMacros(); FirstSubmoduleID = NUM_PREDEF_SUBMODULE_IDS + Chain->getTotalNumSubmodules(); FirstSelectorID = NUM_PREDEF_SELECTOR_IDS + Chain->getTotalNumSelectors(); - NextDeclID = FirstDeclID; NextTypeID = FirstTypeID; NextIdentID = FirstIdentID; NextMacroID = FirstMacroID; diff --git a/clang/lib/Serialization/ModuleFile.cpp b/clang/lib/Serialization/ModuleFile.cpp index 2c42d33a8f5d..f64a59bd9489 100644 --- a/clang/lib/Serialization/ModuleFile.cpp +++ b/clang/lib/Serialization/ModuleFile.cpp @@ -87,7 +87,6 @@ LLVM_DUMP_METHOD void ModuleFile::dump() { << " Number of types: " << LocalNumTypes << '\n'; dumpLocalRemap("Type index local -> global map", TypeRemap); - llvm::errs() << " Base decl ID: " << BaseDeclID << '\n' + llvm::errs() << " Base decl index: " << BaseDeclIndex << '\n' << " Number of decls: " << LocalNumDecls << '\n'; - dumpLocalRemap("Decl ID local -> global map", DeclRemap); } diff --git a/clang/test/Modules/no-transitive-decls-change.cppm b/clang/test/Modules/no-transitive-decls-change.cppm new file mode 100644 index 000000000000..42ac061bc90b --- /dev/null +++ b/clang/test/Modules/no-transitive-decls-change.cppm @@ -0,0 +1,112 @@ +// Testing that changing a declaration in an unused module file won't change +// the BMI of the current module file. +// +// RUN: rm -rf %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/m-partA.cppm -emit-reduced-module-interface -o %t/m-partA.pcm +// RUN: %clang_cc1 -std=c++20 %t/m-partA.v1.cppm -emit-reduced-module-interface -o \ +// RUN: %t/m-partA.v1.pcm +// RUN: %clang_cc1 -std=c++20 %t/m-partB.cppm -emit-reduced-module-interface -o %t/m-partB.pcm +// RUN: %clang_cc1 -std=c++20 %t/m.cppm -emit-reduced-module-interface -o %t/m.pcm \ +// RUN: -fmodule-file=m:partA=%t/m-partA.pcm -fmodule-file=m:partB=%t/m-partB.pcm +// RUN: %clang_cc1 -std=c++20 %t/m.cppm -emit-reduced-module-interface -o %t/m.v1.pcm \ +// RUN: -fmodule-file=m:partA=%t/m-partA.v1.pcm -fmodule-file=m:partB=%t/m-partB.pcm +// +// RUN: %clang_cc1 -std=c++20 %t/useBOnly.cppm -emit-reduced-module-interface -o %t/useBOnly.pcm \ +// RUN: -fmodule-file=m=%t/m.pcm -fmodule-file=m:partA=%t/m-partA.pcm \ +// RUN: -fmodule-file=m:partB=%t/m-partB.pcm +// RUN: %clang_cc1 -std=c++20 %t/useBOnly.cppm -emit-reduced-module-interface -o %t/useBOnly.v1.pcm \ +// RUN: -fmodule-file=m=%t/m.v1.pcm -fmodule-file=m:partA=%t/m-partA.v1.pcm \ +// RUN: -fmodule-file=m:partB=%t/m-partB.pcm +// Since useBOnly only uses partB from module M, the change in partA shouldn't affect +// useBOnly. +// RUN: diff %t/useBOnly.pcm %t/useBOnly.v1.pcm &> /dev/null + +//--- m-partA.cppm +export module m:partA; + +namespace A_Impl { + inline int getAImpl() { + return 43; + } + + inline int getA2Impl() { + return 43; + } +} + +namespace A { + using A_Impl::getAImpl; +} + +export inline int getA() { + return 43; +} + +export inline int getA2(int) { + return 88; +} + +//--- m-partA.v1.cppm +export module m:partA; + +namespace A_Impl { + inline int getAImpl() { + return 43; + } + + inline int getA2Impl() { + return 43; + } +} + +namespace A { + using A_Impl::getAImpl; + // Adding a new declaration without introducing a new declaration name. + using A_Impl::getA2Impl; +} + +inline int getA() { + return 43; +} + +inline int getA2(int) { + return 88; +} + +// Now we add a new declaration without introducing new identifier and new types. +// The consuming module which didn't use m:partA completely is expected to be +// not changed. +inline int getA(int) { + return 88; +} + +//--- m-partB.cppm +export module m:partB; + +export inline int getB() { + return 430; +} + +//--- m.cppm +export module m; +export import :partA; +export import :partB; + +//--- useBOnly.cppm +export module useBOnly; +import m; + +export inline int get() { + return getB(); +} + +//--- useAOnly.cppm +export module useAOnly; +import m; + +export inline int get() { + A a; + return a.getValue(); +} -- GitLab From 8c452d0cc5004d32bcce7a33e6652c19debf7f6d Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Wed, 5 Jun 2024 21:13:52 -0700 Subject: [PATCH 039/462] Revert "[RISCV] Support select/merge like ops for bf16 vectors when have Zvfbfmin" (#94565) Reverts llvm/llvm-project#91936 Premerge bots are broken. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 32 +- .../Target/RISCV/RISCVInstrInfoVPseudos.td | 15 +- .../Target/RISCV/RISCVInstrInfoVSDPatterns.td | 8 +- .../Target/RISCV/RISCVInstrInfoVVLPatterns.td | 5 +- .../RISCV/rvv/fixed-vectors-select-fp.ll | 128 +------- .../RISCV/rvv/fixed-vectors-vpmerge.ll | 144 +------- .../RISCV/rvv/fixed-vectors-vselect-vp.ll | 56 +--- llvm/test/CodeGen/RISCV/rvv/select-fp.ll | 188 +---------- llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll | 308 +----------------- llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll | 224 +------------ llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll | 80 +---- 11 files changed, 38 insertions(+), 1150 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 4051279fdbf8..8ace5d79af07 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -1102,15 +1102,6 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::EXTRACT_SUBVECTOR}, VT, Custom); setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); - if (Subtarget.hasStdExtZfbfmin()) { - if (Subtarget.hasVInstructionsF16()) - setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); - else if (Subtarget.hasVInstructionsF16Minimal()) - setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); - } - setOperationAction({ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT, - Custom); - setOperationAction(ISD::SELECT_CC, VT, Expand); // TODO: Promote to fp32. } } @@ -1340,15 +1331,6 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::EXTRACT_SUBVECTOR}, VT, Custom); setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); - if (Subtarget.hasStdExtZfbfmin()) { - if (Subtarget.hasVInstructionsF16()) - setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); - else if (Subtarget.hasVInstructionsF16Minimal()) - setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); - } - setOperationAction( - {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT, - Custom); // TODO: Promote to fp32. continue; } @@ -6722,16 +6704,10 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, case ISD::BUILD_VECTOR: return lowerBUILD_VECTOR(Op, DAG, Subtarget); case ISD::SPLAT_VECTOR: - if ((Op.getValueType().getScalarType() == MVT::f16 && - (Subtarget.hasVInstructionsF16Minimal() && - Subtarget.hasStdExtZfhminOrZhinxmin() && - !Subtarget.hasVInstructionsF16())) || - (Op.getValueType().getScalarType() == MVT::bf16 && - (Subtarget.hasVInstructionsBF16() && Subtarget.hasStdExtZfbfmin() && - Subtarget.hasVInstructionsF16Minimal() && - !Subtarget.hasVInstructionsF16()))) { - if (Op.getValueType() == MVT::nxv32f16 || - Op.getValueType() == MVT::nxv32bf16) + if (Op.getValueType().getScalarType() == MVT::f16 && + (Subtarget.hasVInstructionsF16Minimal() && + !Subtarget.hasVInstructionsF16())) { + if (Op.getValueType() == MVT::nxv32f16) return SplitVectorOp(Op, DAG); SDLoc DL(Op); SDValue NewScalar = diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index fe4d839e4fdc..b0949f5fc1d7 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -382,20 +382,7 @@ class GetIntVTypeInfo { // Equivalent integer vector type. Eg. // VI8M1 → VI8M1 (identity) // VF64M4 → VI64M4 - VTypeInfo Vti = !cast(!subst("VBF", "VI", - !subst("VF", "VI", - !cast(vti)))); -} - -// This functor is used to obtain the fp vector type that has the same SEW and -// multiplier as the input parameter type. -class GetFpVTypeInfo { - // Equivalent integer vector type. Eg. - // VF16M1 → VF16M1 (identity) - // VBF16M1 → VF16M1 - VTypeInfo Vti = !cast(!subst("VBF", "VF", - !subst("VI", "VF", - !cast(vti)))); + VTypeInfo Vti = !cast(!subst("VF", "VI", !cast(vti))); } class MTypeInfo { diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td index 497c4aadf753..3163e4bafd4b 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td @@ -1394,7 +1394,7 @@ defm : VPatFPSetCCSDNode_VV_VF_FV; // Floating-point vselects: // 11.15. Vector Integer Merge Instructions // 13.15. Vector Floating-Point Merge Instruction -foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in { +foreach fvti = AllFloatVectors in { defvar ivti = GetIntVTypeInfo.Vti; let Predicates = GetVTypePredicates.Predicates in { def : Pat<(fvti.Vector (vselect (fvti.Mask V0), fvti.RegClass:$rs1, @@ -1412,9 +1412,7 @@ foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in { fvti.RegClass:$rs2, 0, (fvti.Mask V0), fvti.AVL, fvti.Log2SEW)>; } - - let Predicates = !listconcat(GetVTypePredicates.Vti>.Predicates, - GetVTypeScalarPredicates.Predicates) in + let Predicates = GetVTypePredicates.Predicates in def : Pat<(fvti.Vector (vselect (fvti.Mask V0), (SplatFPOp fvti.ScalarRegClass:$rs1), fvti.RegClass:$rs2)), @@ -1477,7 +1475,7 @@ foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in { //===----------------------------------------------------------------------===// foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in { - let Predicates = !listconcat(GetVTypePredicates.Vti>.Predicates, + let Predicates = !listconcat(GetVTypePredicates.Predicates, GetVTypeScalarPredicates.Predicates) in def : Pat<(fvti.Vector (riscv_vfmv_v_f_vl undef, fvti.ScalarRegClass:$rs1, srcvalue)), (!cast("PseudoVFMV_V_"#fvti.ScalarSuffix#"_"#fvti.LMul.MX) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td index 70d8265e7be4..ce8133a5a297 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td @@ -2604,7 +2604,7 @@ foreach vti = AllFloatVectors in { } } -foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in { +foreach fvti = AllFloatVectors in { // Floating-point vselects: // 11.15. Vector Integer Merge Instructions // 13.15. Vector Floating-Point Merge Instruction @@ -2639,8 +2639,7 @@ foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in { GPR:$vl, fvti.Log2SEW)>; } - let Predicates = !listconcat(GetVTypePredicates.Vti>.Predicates, - GetVTypeScalarPredicates.Predicates) in { + let Predicates = GetVTypePredicates.Predicates in { def : Pat<(fvti.Vector (riscv_vmerge_vl (fvti.Mask V0), (SplatFPOp fvti.ScalarRegClass:$rs1), fvti.RegClass:$rs2, diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll index 7a96aad31f08..d945cf561698 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d -riscv-v-vector-bits-min=128 \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d -riscv-v-vector-bits-min=128 \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d -riscv-v-vector-bits-min=128 \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d -riscv-v-vector-bits-min=128 \ ; RUN: -verify-machineinstrs < %s | FileCheck %s define <2 x half> @select_v2f16(i1 zeroext %c, <2 x half> %a, <2 x half> %b) { @@ -343,123 +343,3 @@ define <16 x double> @selectcc_v16f64(double %a, double %b, <16 x double> %c, <1 %v = select i1 %cmp, <16 x double> %c, <16 x double> %d ret <16 x double> %v } - -define <2 x bfloat> @select_v2bf16(i1 zeroext %c, <2 x bfloat> %a, <2 x bfloat> %b) { -; CHECK-LABEL: select_v2bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %v = select i1 %c, <2 x bfloat> %a, <2 x bfloat> %b - ret <2 x bfloat> %v -} - -define <2 x bfloat> @selectcc_v2bf16(bfloat %a, bfloat %b, <2 x bfloat> %c, <2 x bfloat> %d) { -; CHECK-LABEL: selectcc_v2bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 -; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 -; CHECK-NEXT: feq.s a0, fa4, fa5 -; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %cmp = fcmp oeq bfloat %a, %b - %v = select i1 %cmp, <2 x bfloat> %c, <2 x bfloat> %d - ret <2 x bfloat> %v -} - -define <4 x bfloat> @select_v4bf16(i1 zeroext %c, <4 x bfloat> %a, <4 x bfloat> %b) { -; CHECK-LABEL: select_v4bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %v = select i1 %c, <4 x bfloat> %a, <4 x bfloat> %b - ret <4 x bfloat> %v -} - -define <4 x bfloat> @selectcc_v4bf16(bfloat %a, bfloat %b, <4 x bfloat> %c, <4 x bfloat> %d) { -; CHECK-LABEL: selectcc_v4bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 -; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 -; CHECK-NEXT: feq.s a0, fa4, fa5 -; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %cmp = fcmp oeq bfloat %a, %b - %v = select i1 %cmp, <4 x bfloat> %c, <4 x bfloat> %d - ret <4 x bfloat> %v -} - -define <8 x bfloat> @select_v8bf16(i1 zeroext %c, <8 x bfloat> %a, <8 x bfloat> %b) { -; CHECK-LABEL: select_v8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %v = select i1 %c, <8 x bfloat> %a, <8 x bfloat> %b - ret <8 x bfloat> %v -} - -define <8 x bfloat> @selectcc_v8bf16(bfloat %a, bfloat %b, <8 x bfloat> %c, <8 x bfloat> %d) { -; CHECK-LABEL: selectcc_v8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 -; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 -; CHECK-NEXT: feq.s a0, fa4, fa5 -; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %cmp = fcmp oeq bfloat %a, %b - %v = select i1 %cmp, <8 x bfloat> %c, <8 x bfloat> %d - ret <8 x bfloat> %v -} - -define <16 x bfloat> @select_v16bf16(i1 zeroext %c, <16 x bfloat> %a, <16 x bfloat> %b) { -; CHECK-LABEL: select_v16bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma -; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vmsne.vi v0, v12, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 -; CHECK-NEXT: ret - %v = select i1 %c, <16 x bfloat> %a, <16 x bfloat> %b - ret <16 x bfloat> %v -} - -define <16 x bfloat> @selectcc_v16bf16(bfloat %a, bfloat %b, <16 x bfloat> %c, <16 x bfloat> %d) { -; CHECK-LABEL: selectcc_v16bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 -; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 -; CHECK-NEXT: feq.s a0, fa4, fa5 -; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma -; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vmsne.vi v0, v12, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 -; CHECK-NEXT: ret - %cmp = fcmp oeq bfloat %a, %b - %v = select i1 %cmp, <16 x bfloat> %c, <16 x bfloat> %d - ret <16 x bfloat> %v -} diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll index d360c3f635b5..9f0561b394b8 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,RV32 -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,RV64 -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,RV32ZVFHMIN -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,RV64ZVFHMIN declare <4 x i1> @llvm.vp.merge.v4i1(<4 x i1>, <4 x i1>, <4 x i1>, i32) @@ -1240,139 +1240,3 @@ define <32 x double> @vpmerge_vf_v32f64(double %a, <32 x double> %vb, <32 x i1> %v = call <32 x double> @llvm.vp.merge.v32f64(<32 x i1> %m, <32 x double> %va, <32 x double> %vb, i32 %evl) ret <32 x double> %v } - -declare <2 x bfloat> @llvm.vp.merge.v2bf16(<2 x i1>, <2 x bfloat>, <2 x bfloat>, i32) - -define <2 x bfloat> @vpmerge_vv_v2bf16(<2 x bfloat> %va, <2 x bfloat> %vb, <2 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vpmerge_vv_v2bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, mf4, tu, ma -; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 -; CHECK-NEXT: vmv1r.v v8, v9 -; CHECK-NEXT: ret - %v = call <2 x bfloat> @llvm.vp.merge.v2bf16(<2 x i1> %m, <2 x bfloat> %va, <2 x bfloat> %vb, i32 %evl) - ret <2 x bfloat> %v -} - -define <2 x bfloat> @vpmerge_vf_v2bf16(bfloat %a, <2 x bfloat> %vb, <2 x i1> %m, i32 zeroext %evl) { -; ZVFH-LABEL: vpmerge_vf_v2bf16: -; ZVFH: # %bb.0: -; ZVFH-NEXT: vsetvli zero, a0, e16, mf4, tu, ma -; ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; ZVFH-NEXT: ret -; -; ZVFHMIN-LABEL: vpmerge_vf_v2bf16: -; ZVFHMIN: # %bb.0: -; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; ZVFHMIN-NEXT: vsetvli a1, zero, e32, mf2, ta, ma -; ZVFHMIN-NEXT: vfmv.v.f v9, fa5 -; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, tu, mu -; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t -; ZVFHMIN-NEXT: ret - %elt.head = insertelement <2 x bfloat> poison, bfloat %a, i32 0 - %va = shufflevector <2 x bfloat> %elt.head, <2 x bfloat> poison, <2 x i32> zeroinitializer - %v = call <2 x bfloat> @llvm.vp.merge.v2bf16(<2 x i1> %m, <2 x bfloat> %va, <2 x bfloat> %vb, i32 %evl) - ret <2 x bfloat> %v -} - -declare <4 x bfloat> @llvm.vp.merge.v4bf16(<4 x i1>, <4 x bfloat>, <4 x bfloat>, i32) - -define <4 x bfloat> @vpmerge_vv_v4bf16(<4 x bfloat> %va, <4 x bfloat> %vb, <4 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vpmerge_vv_v4bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, mf2, tu, ma -; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 -; CHECK-NEXT: vmv1r.v v8, v9 -; CHECK-NEXT: ret - %v = call <4 x bfloat> @llvm.vp.merge.v4bf16(<4 x i1> %m, <4 x bfloat> %va, <4 x bfloat> %vb, i32 %evl) - ret <4 x bfloat> %v -} - -define <4 x bfloat> @vpmerge_vf_v4bf16(bfloat %a, <4 x bfloat> %vb, <4 x i1> %m, i32 zeroext %evl) { -; ZVFH-LABEL: vpmerge_vf_v4bf16: -; ZVFH: # %bb.0: -; ZVFH-NEXT: vsetvli zero, a0, e16, mf2, tu, ma -; ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; ZVFH-NEXT: ret -; -; ZVFHMIN-LABEL: vpmerge_vf_v4bf16: -; ZVFHMIN: # %bb.0: -; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; ZVFHMIN-NEXT: vsetvli a1, zero, e32, m1, ta, ma -; ZVFHMIN-NEXT: vfmv.v.f v9, fa5 -; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf2, tu, mu -; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t -; ZVFHMIN-NEXT: ret - %elt.head = insertelement <4 x bfloat> poison, bfloat %a, i32 0 - %va = shufflevector <4 x bfloat> %elt.head, <4 x bfloat> poison, <4 x i32> zeroinitializer - %v = call <4 x bfloat> @llvm.vp.merge.v4bf16(<4 x i1> %m, <4 x bfloat> %va, <4 x bfloat> %vb, i32 %evl) - ret <4 x bfloat> %v -} - -declare <8 x bfloat> @llvm.vp.merge.v8bf16(<8 x i1>, <8 x bfloat>, <8 x bfloat>, i32) - -define <8 x bfloat> @vpmerge_vv_v8bf16(<8 x bfloat> %va, <8 x bfloat> %vb, <8 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vpmerge_vv_v8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m1, tu, ma -; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 -; CHECK-NEXT: vmv1r.v v8, v9 -; CHECK-NEXT: ret - %v = call <8 x bfloat> @llvm.vp.merge.v8bf16(<8 x i1> %m, <8 x bfloat> %va, <8 x bfloat> %vb, i32 %evl) - ret <8 x bfloat> %v -} - -define <8 x bfloat> @vpmerge_vf_v8bf16(bfloat %a, <8 x bfloat> %vb, <8 x i1> %m, i32 zeroext %evl) { -; ZVFH-LABEL: vpmerge_vf_v8bf16: -; ZVFH: # %bb.0: -; ZVFH-NEXT: vsetvli zero, a0, e16, m1, tu, ma -; ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; ZVFH-NEXT: ret -; -; ZVFHMIN-LABEL: vpmerge_vf_v8bf16: -; ZVFHMIN: # %bb.0: -; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; ZVFHMIN-NEXT: vsetvli a1, zero, e32, m2, ta, ma -; ZVFHMIN-NEXT: vfmv.v.f v10, fa5 -; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m1, tu, mu -; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t -; ZVFHMIN-NEXT: ret - %elt.head = insertelement <8 x bfloat> poison, bfloat %a, i32 0 - %va = shufflevector <8 x bfloat> %elt.head, <8 x bfloat> poison, <8 x i32> zeroinitializer - %v = call <8 x bfloat> @llvm.vp.merge.v8bf16(<8 x i1> %m, <8 x bfloat> %va, <8 x bfloat> %vb, i32 %evl) - ret <8 x bfloat> %v -} - -declare <16 x bfloat> @llvm.vp.merge.v16bf16(<16 x i1>, <16 x bfloat>, <16 x bfloat>, i32) - -define <16 x bfloat> @vpmerge_vv_v16bf16(<16 x bfloat> %va, <16 x bfloat> %vb, <16 x i1> %m, i32 zeroext %evl) { -; CHECK-LABEL: vpmerge_vv_v16bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m2, tu, ma -; CHECK-NEXT: vmerge.vvm v10, v10, v8, v0 -; CHECK-NEXT: vmv2r.v v8, v10 -; CHECK-NEXT: ret - %v = call <16 x bfloat> @llvm.vp.merge.v16bf16(<16 x i1> %m, <16 x bfloat> %va, <16 x bfloat> %vb, i32 %evl) - ret <16 x bfloat> %v -} - -define <16 x bfloat> @vpmerge_vf_v16bf16(bfloat %a, <16 x bfloat> %vb, <16 x i1> %m, i32 zeroext %evl) { -; ZVFH-LABEL: vpmerge_vf_v16bf16: -; ZVFH: # %bb.0: -; ZVFH-NEXT: vsetvli zero, a0, e16, m2, tu, ma -; ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; ZVFH-NEXT: ret -; -; ZVFHMIN-LABEL: vpmerge_vf_v16bf16: -; ZVFHMIN: # %bb.0: -; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; ZVFHMIN-NEXT: vsetvli a1, zero, e32, m4, ta, ma -; ZVFHMIN-NEXT: vfmv.v.f v12, fa5 -; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m2, tu, mu -; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t -; ZVFHMIN-NEXT: ret - %elt.head = insertelement <16 x bfloat> poison, bfloat %a, i32 0 - %va = shufflevector <16 x bfloat> %elt.head, <16 x bfloat> poison, <16 x i32> zeroinitializer - %v = call <16 x bfloat> @llvm.vp.merge.v16bf16(<16 x i1> %m, <16 x bfloat> %va, <16 x bfloat> %vb, i32 %evl) - ret <16 x bfloat> %v -} diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll index c5d9cdacae74..0a2ed3eb1ffb 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d -riscv-v-vector-bits-min=128 \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d -riscv-v-vector-bits-min=128 \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d -riscv-v-vector-bits-min=128 \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d -riscv-v-vector-bits-min=128 \ ; RUN: -verify-machineinstrs < %s | FileCheck %s declare <1 x i1> @llvm.vp.select.v1i1(<1 x i1>, <1 x i1>, <1 x i1>, i32) @@ -683,51 +683,3 @@ define <16 x double> @select_v16f64(<16 x i1> %a, <16 x double> %b, <16 x double %v = call <16 x double> @llvm.vp.select.v16f64(<16 x i1> %a, <16 x double> %b, <16 x double> %c, i32 %evl) ret <16 x double> %v } - -declare <2 x bfloat> @llvm.vp.select.v2bf16(<2 x i1>, <2 x bfloat>, <2 x bfloat>, i32) - -define <2 x bfloat> @select_v2bf16(<2 x i1> %a, <2 x bfloat> %b, <2 x bfloat> %c, i32 zeroext %evl) { -; CHECK-LABEL: select_v2bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, mf4, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %v = call <2 x bfloat> @llvm.vp.select.v2bf16(<2 x i1> %a, <2 x bfloat> %b, <2 x bfloat> %c, i32 %evl) - ret <2 x bfloat> %v -} - -declare <4 x bfloat> @llvm.vp.select.v4bf16(<4 x i1>, <4 x bfloat>, <4 x bfloat>, i32) - -define <4 x bfloat> @select_v4bf16(<4 x i1> %a, <4 x bfloat> %b, <4 x bfloat> %c, i32 zeroext %evl) { -; CHECK-LABEL: select_v4bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %v = call <4 x bfloat> @llvm.vp.select.v4bf16(<4 x i1> %a, <4 x bfloat> %b, <4 x bfloat> %c, i32 %evl) - ret <4 x bfloat> %v -} - -declare <8 x bfloat> @llvm.vp.select.v8bf16(<8 x i1>, <8 x bfloat>, <8 x bfloat>, i32) - -define <8 x bfloat> @select_v8bf16(<8 x i1> %a, <8 x bfloat> %b, <8 x bfloat> %c, i32 zeroext %evl) { -; CHECK-LABEL: select_v8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %v = call <8 x bfloat> @llvm.vp.select.v8bf16(<8 x i1> %a, <8 x bfloat> %b, <8 x bfloat> %c, i32 %evl) - ret <8 x bfloat> %v -} - -declare <16 x bfloat> @llvm.vp.select.v16bf16(<16 x i1>, <16 x bfloat>, <16 x bfloat>, i32) - -define <16 x bfloat> @select_v16bf16(<16 x i1> %a, <16 x bfloat> %b, <16 x bfloat> %c, i32 zeroext %evl) { -; CHECK-LABEL: select_v16bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 -; CHECK-NEXT: ret - %v = call <16 x bfloat> @llvm.vp.select.v16bf16(<16 x i1> %a, <16 x bfloat> %b, <16 x bfloat> %c, i32 %evl) - ret <16 x bfloat> %v -} diff --git a/llvm/test/CodeGen/RISCV/rvv/select-fp.ll b/llvm/test/CodeGen/RISCV/rvv/select-fp.ll index 2b9d847a9e87..f8581d8e21b3 100644 --- a/llvm/test/CodeGen/RISCV/rvv/select-fp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/select-fp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s define @select_nxv1f16(i1 zeroext %c, %a, %b) { @@ -427,183 +427,3 @@ define @selectcc_nxv8f64(double %a, double %b, %c, %d ret %v } - -define @select_nxv1bf16(i1 zeroext %c, %a, %b) { -; CHECK-LABEL: select_nxv1bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %v = select i1 %c, %a, %b - ret %v -} - -define @selectcc_nxv1bf16(bfloat %a, bfloat %b, %c, %d) { -; CHECK-LABEL: selectcc_nxv1bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 -; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 -; CHECK-NEXT: feq.s a0, fa4, fa5 -; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %cmp = fcmp oeq bfloat %a, %b - %v = select i1 %cmp, %c, %d - ret %v -} - -define @select_nxv2bf16(i1 zeroext %c, %a, %b) { -; CHECK-LABEL: select_nxv2bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %v = select i1 %c, %a, %b - ret %v -} - -define @selectcc_nxv2bf16(bfloat %a, bfloat %b, %c, %d) { -; CHECK-LABEL: selectcc_nxv2bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 -; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 -; CHECK-NEXT: feq.s a0, fa4, fa5 -; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %cmp = fcmp oeq bfloat %a, %b - %v = select i1 %cmp, %c, %d - ret %v -} - -define @select_nxv4bf16(i1 zeroext %c, %a, %b) { -; CHECK-LABEL: select_nxv4bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %v = select i1 %c, %a, %b - ret %v -} - -define @selectcc_nxv4bf16(bfloat %a, bfloat %b, %c, %d) { -; CHECK-LABEL: selectcc_nxv4bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 -; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 -; CHECK-NEXT: feq.s a0, fa4, fa5 -; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %cmp = fcmp oeq bfloat %a, %b - %v = select i1 %cmp, %c, %d - ret %v -} - -define @select_nxv8bf16(i1 zeroext %c, %a, %b) { -; CHECK-LABEL: select_nxv8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma -; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vmsne.vi v0, v12, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 -; CHECK-NEXT: ret - %v = select i1 %c, %a, %b - ret %v -} - -define @selectcc_nxv8bf16(bfloat %a, bfloat %b, %c, %d) { -; CHECK-LABEL: selectcc_nxv8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 -; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 -; CHECK-NEXT: feq.s a0, fa4, fa5 -; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma -; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vmsne.vi v0, v12, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 -; CHECK-NEXT: ret - %cmp = fcmp oeq bfloat %a, %b - %v = select i1 %cmp, %c, %d - ret %v -} - -define @select_nxv16bf16(i1 zeroext %c, %a, %b) { -; CHECK-LABEL: select_nxv16bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e8, m2, ta, ma -; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vmsne.vi v0, v16, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0 -; CHECK-NEXT: ret - %v = select i1 %c, %a, %b - ret %v -} - -define @selectcc_nxv16bf16(bfloat %a, bfloat %b, %c, %d) { -; CHECK-LABEL: selectcc_nxv16bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 -; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 -; CHECK-NEXT: feq.s a0, fa4, fa5 -; CHECK-NEXT: vsetvli a1, zero, e8, m2, ta, ma -; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vmsne.vi v0, v16, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0 -; CHECK-NEXT: ret - %cmp = fcmp oeq bfloat %a, %b - %v = select i1 %cmp, %c, %d - ret %v -} - -define @select_nxv32bf16(i1 zeroext %c, %a, %b) { -; CHECK-LABEL: select_nxv32bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e8, m4, ta, ma -; CHECK-NEXT: vmv.v.x v24, a0 -; CHECK-NEXT: vmsne.vi v0, v24, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, m8, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v16, v8, v0 -; CHECK-NEXT: ret - %v = select i1 %c, %a, %b - ret %v -} - -define @selectcc_nxv32bf16(bfloat %a, bfloat %b, %c, %d) { -; CHECK-LABEL: selectcc_nxv32bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 -; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 -; CHECK-NEXT: feq.s a0, fa4, fa5 -; CHECK-NEXT: vsetvli a1, zero, e8, m4, ta, ma -; CHECK-NEXT: vmv.v.x v24, a0 -; CHECK-NEXT: vmsne.vi v0, v24, 0 -; CHECK-NEXT: vsetvli zero, zero, e16, m8, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v16, v8, v0 -; CHECK-NEXT: ret - %cmp = fcmp oeq bfloat %a, %b - %v = select i1 %cmp, %c, %d - ret %v -} diff --git a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll index e33c795169fa..094e6c9cc754 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFH -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVFH -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFHMIN -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVFHMIN declare @llvm.vp.merge.nxv1i1(, , , i32) @@ -1547,303 +1547,3 @@ define @vpmerge_vf_nxv8f64(double %a, @llvm.vp.merge.nxv8f64( %m, %va, %vb, i32 %evl) ret %v } - -declare @llvm.vp.merge.nxv1bf16(, , , i32) - -define @vpmerge_vv_nxv1bf16( %va, %vb, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpmerge_vv_nxv1bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, mf4, tu, ma -; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 -; CHECK-NEXT: vmv1r.v v8, v9 -; CHECK-NEXT: ret - %v = call @llvm.vp.merge.nxv1bf16( %m, %va, %vb, i32 %evl) - ret %v -} - -define @vpmerge_vf_nxv1bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { -; RV32ZVFH-LABEL: vpmerge_vf_nxv1bf16: -; RV32ZVFH: # %bb.0: -; RV32ZVFH-NEXT: vsetvli zero, a0, e16, mf4, tu, ma -; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; RV32ZVFH-NEXT: ret -; -; RV64ZVFH-LABEL: vpmerge_vf_nxv1bf16: -; RV64ZVFH: # %bb.0: -; RV64ZVFH-NEXT: vsetvli zero, a0, e16, mf4, tu, ma -; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; RV64ZVFH-NEXT: ret -; -; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv1bf16: -; RV32ZVFHMIN: # %bb.0: -; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, mf2, ta, ma -; RV32ZVFHMIN-NEXT: vfmv.v.f v9, fa5 -; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, tu, mu -; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t -; RV32ZVFHMIN-NEXT: ret -; -; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv1bf16: -; RV64ZVFHMIN: # %bb.0: -; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, mf2, ta, ma -; RV64ZVFHMIN-NEXT: vfmv.v.f v9, fa5 -; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, tu, mu -; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t -; RV64ZVFHMIN-NEXT: ret - %elt.head = insertelement poison, bfloat %a, i32 0 - %va = shufflevector %elt.head, poison, zeroinitializer - %v = call @llvm.vp.merge.nxv1bf16( %m, %va, %vb, i32 %evl) - ret %v -} - -declare @llvm.vp.merge.nxv2bf16(, , , i32) - -define @vpmerge_vv_nxv2bf16( %va, %vb, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpmerge_vv_nxv2bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, mf2, tu, ma -; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 -; CHECK-NEXT: vmv1r.v v8, v9 -; CHECK-NEXT: ret - %v = call @llvm.vp.merge.nxv2bf16( %m, %va, %vb, i32 %evl) - ret %v -} - -define @vpmerge_vf_nxv2bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { -; RV32ZVFH-LABEL: vpmerge_vf_nxv2bf16: -; RV32ZVFH: # %bb.0: -; RV32ZVFH-NEXT: vsetvli zero, a0, e16, mf2, tu, ma -; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; RV32ZVFH-NEXT: ret -; -; RV64ZVFH-LABEL: vpmerge_vf_nxv2bf16: -; RV64ZVFH: # %bb.0: -; RV64ZVFH-NEXT: vsetvli zero, a0, e16, mf2, tu, ma -; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; RV64ZVFH-NEXT: ret -; -; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv2bf16: -; RV32ZVFHMIN: # %bb.0: -; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m1, ta, ma -; RV32ZVFHMIN-NEXT: vfmv.v.f v9, fa5 -; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf2, tu, mu -; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t -; RV32ZVFHMIN-NEXT: ret -; -; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv2bf16: -; RV64ZVFHMIN: # %bb.0: -; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m1, ta, ma -; RV64ZVFHMIN-NEXT: vfmv.v.f v9, fa5 -; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf2, tu, mu -; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t -; RV64ZVFHMIN-NEXT: ret - %elt.head = insertelement poison, bfloat %a, i32 0 - %va = shufflevector %elt.head, poison, zeroinitializer - %v = call @llvm.vp.merge.nxv2bf16( %m, %va, %vb, i32 %evl) - ret %v -} - -declare @llvm.vp.merge.nxv4bf16(, , , i32) - -define @vpmerge_vv_nxv4bf16( %va, %vb, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpmerge_vv_nxv4bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m1, tu, ma -; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 -; CHECK-NEXT: vmv1r.v v8, v9 -; CHECK-NEXT: ret - %v = call @llvm.vp.merge.nxv4bf16( %m, %va, %vb, i32 %evl) - ret %v -} - -define @vpmerge_vf_nxv4bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { -; RV32ZVFH-LABEL: vpmerge_vf_nxv4bf16: -; RV32ZVFH: # %bb.0: -; RV32ZVFH-NEXT: vsetvli zero, a0, e16, m1, tu, ma -; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; RV32ZVFH-NEXT: ret -; -; RV64ZVFH-LABEL: vpmerge_vf_nxv4bf16: -; RV64ZVFH: # %bb.0: -; RV64ZVFH-NEXT: vsetvli zero, a0, e16, m1, tu, ma -; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; RV64ZVFH-NEXT: ret -; -; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv4bf16: -; RV32ZVFHMIN: # %bb.0: -; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m2, ta, ma -; RV32ZVFHMIN-NEXT: vfmv.v.f v10, fa5 -; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, m1, tu, mu -; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t -; RV32ZVFHMIN-NEXT: ret -; -; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv4bf16: -; RV64ZVFHMIN: # %bb.0: -; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m2, ta, ma -; RV64ZVFHMIN-NEXT: vfmv.v.f v10, fa5 -; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, m1, tu, mu -; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t -; RV64ZVFHMIN-NEXT: ret - %elt.head = insertelement poison, bfloat %a, i32 0 - %va = shufflevector %elt.head, poison, zeroinitializer - %v = call @llvm.vp.merge.nxv4bf16( %m, %va, %vb, i32 %evl) - ret %v -} - -declare @llvm.vp.merge.nxv8bf16(, , , i32) - -define @vpmerge_vv_nxv8bf16( %va, %vb, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpmerge_vv_nxv8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m2, tu, ma -; CHECK-NEXT: vmerge.vvm v10, v10, v8, v0 -; CHECK-NEXT: vmv2r.v v8, v10 -; CHECK-NEXT: ret - %v = call @llvm.vp.merge.nxv8bf16( %m, %va, %vb, i32 %evl) - ret %v -} - -define @vpmerge_vf_nxv8bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { -; RV32ZVFH-LABEL: vpmerge_vf_nxv8bf16: -; RV32ZVFH: # %bb.0: -; RV32ZVFH-NEXT: vsetvli zero, a0, e16, m2, tu, ma -; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; RV32ZVFH-NEXT: ret -; -; RV64ZVFH-LABEL: vpmerge_vf_nxv8bf16: -; RV64ZVFH: # %bb.0: -; RV64ZVFH-NEXT: vsetvli zero, a0, e16, m2, tu, ma -; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; RV64ZVFH-NEXT: ret -; -; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv8bf16: -; RV32ZVFHMIN: # %bb.0: -; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m4, ta, ma -; RV32ZVFHMIN-NEXT: vfmv.v.f v12, fa5 -; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, m2, tu, mu -; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t -; RV32ZVFHMIN-NEXT: ret -; -; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv8bf16: -; RV64ZVFHMIN: # %bb.0: -; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m4, ta, ma -; RV64ZVFHMIN-NEXT: vfmv.v.f v12, fa5 -; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, m2, tu, mu -; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t -; RV64ZVFHMIN-NEXT: ret - %elt.head = insertelement poison, bfloat %a, i32 0 - %va = shufflevector %elt.head, poison, zeroinitializer - %v = call @llvm.vp.merge.nxv8bf16( %m, %va, %vb, i32 %evl) - ret %v -} - -declare @llvm.vp.merge.nxv16bf16(, , , i32) - -define @vpmerge_vv_nxv16bf16( %va, %vb, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpmerge_vv_nxv16bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m4, tu, ma -; CHECK-NEXT: vmerge.vvm v12, v12, v8, v0 -; CHECK-NEXT: vmv4r.v v8, v12 -; CHECK-NEXT: ret - %v = call @llvm.vp.merge.nxv16bf16( %m, %va, %vb, i32 %evl) - ret %v -} - -define @vpmerge_vf_nxv16bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { -; RV32ZVFH-LABEL: vpmerge_vf_nxv16bf16: -; RV32ZVFH: # %bb.0: -; RV32ZVFH-NEXT: vsetvli zero, a0, e16, m4, tu, ma -; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; RV32ZVFH-NEXT: ret -; -; RV64ZVFH-LABEL: vpmerge_vf_nxv16bf16: -; RV64ZVFH: # %bb.0: -; RV64ZVFH-NEXT: vsetvli zero, a0, e16, m4, tu, ma -; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; RV64ZVFH-NEXT: ret -; -; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv16bf16: -; RV32ZVFHMIN: # %bb.0: -; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m8, ta, ma -; RV32ZVFHMIN-NEXT: vfmv.v.f v16, fa5 -; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, tu, mu -; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t -; RV32ZVFHMIN-NEXT: ret -; -; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv16bf16: -; RV64ZVFHMIN: # %bb.0: -; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m8, ta, ma -; RV64ZVFHMIN-NEXT: vfmv.v.f v16, fa5 -; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, tu, mu -; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t -; RV64ZVFHMIN-NEXT: ret - %elt.head = insertelement poison, bfloat %a, i32 0 - %va = shufflevector %elt.head, poison, zeroinitializer - %v = call @llvm.vp.merge.nxv16bf16( %m, %va, %vb, i32 %evl) - ret %v -} - -declare @llvm.vp.merge.nxv32bf16(, , , i32) - -define @vpmerge_vv_nxv32bf16( %va, %vb, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpmerge_vv_nxv32bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m8, tu, ma -; CHECK-NEXT: vmerge.vvm v16, v16, v8, v0 -; CHECK-NEXT: vmv8r.v v8, v16 -; CHECK-NEXT: ret - %v = call @llvm.vp.merge.nxv32bf16( %m, %va, %vb, i32 %evl) - ret %v -} - -define @vpmerge_vf_nxv32bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { -; RV32ZVFH-LABEL: vpmerge_vf_nxv32bf16: -; RV32ZVFH: # %bb.0: -; RV32ZVFH-NEXT: vsetvli zero, a0, e16, m8, tu, ma -; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; RV32ZVFH-NEXT: ret -; -; RV64ZVFH-LABEL: vpmerge_vf_nxv32bf16: -; RV64ZVFH: # %bb.0: -; RV64ZVFH-NEXT: vsetvli zero, a0, e16, m8, tu, ma -; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; RV64ZVFH-NEXT: ret -; -; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv32bf16: -; RV32ZVFHMIN: # %bb.0: -; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m8, ta, ma -; RV32ZVFHMIN-NEXT: vfmv.v.f v24, fa5 -; RV32ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v16, v24 -; RV32ZVFHMIN-NEXT: vmv.v.v v20, v16 -; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, m8, tu, ma -; RV32ZVFHMIN-NEXT: vmerge.vvm v8, v8, v16, v0 -; RV32ZVFHMIN-NEXT: ret -; -; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv32bf16: -; RV64ZVFHMIN: # %bb.0: -; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m8, ta, ma -; RV64ZVFHMIN-NEXT: vfmv.v.f v24, fa5 -; RV64ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v16, v24 -; RV64ZVFHMIN-NEXT: vmv.v.v v20, v16 -; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, m8, tu, ma -; RV64ZVFHMIN-NEXT: vmerge.vvm v8, v8, v16, v0 -; RV64ZVFHMIN-NEXT: ret - %elt.head = insertelement poison, bfloat %a, i32 0 - %va = shufflevector %elt.head, poison, zeroinitializer - %v = call @llvm.vp.merge.nxv32bf16( %m, %va, %vb, i32 %evl) - ret %v -} diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll index 82c2fe3273bd..53b8e4a78b75 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFH -; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFH -; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFHMIN -; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfhmin,+v -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFHMIN define @vfmerge_vv_nxv1f16( %va, %vb, %cond) { @@ -512,219 +512,3 @@ define void @vselect_legalize_regression( %a, %sel, ptr %out ret void } - -define @vfmerge_vv_nxv1bf16( %va, %vb, %cond) { -; CHECK-LABEL: vfmerge_vv_nxv1bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %vc = select %cond, %va, %vb - ret %vc -} - -define @vfmerge_fv_nxv1bf16( %va, bfloat %b, %cond) { -; CHECK-ZVFH-LABEL: vfmerge_fv_nxv1bf16: -; CHECK-ZVFH: # %bb.0: -; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, ma -; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; CHECK-ZVFH-NEXT: ret -; -; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv1bf16: -; CHECK-ZVFHMIN: # %bb.0: -; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, mf2, ta, ma -; CHECK-ZVFHMIN-NEXT: vfmv.v.f v9, fa5 -; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, mu -; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t -; CHECK-ZVFHMIN-NEXT: ret - %head = insertelement poison, bfloat %b, i32 0 - %splat = shufflevector %head, poison, zeroinitializer - %vc = select %cond, %splat, %va - ret %vc -} - -define @vfmerge_vv_nxv2bf16( %va, %vb, %cond) { -; CHECK-LABEL: vfmerge_vv_nxv2bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %vc = select %cond, %va, %vb - ret %vc -} - -define @vfmerge_fv_nxv2bf16( %va, bfloat %b, %cond) { -; CHECK-ZVFH-LABEL: vfmerge_fv_nxv2bf16: -; CHECK-ZVFH: # %bb.0: -; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, mf2, ta, ma -; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; CHECK-ZVFH-NEXT: ret -; -; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv2bf16: -; CHECK-ZVFHMIN: # %bb.0: -; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m1, ta, ma -; CHECK-ZVFHMIN-NEXT: vfmv.v.f v9, fa5 -; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, mu -; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t -; CHECK-ZVFHMIN-NEXT: ret - %head = insertelement poison, bfloat %b, i32 0 - %splat = shufflevector %head, poison, zeroinitializer - %vc = select %cond, %splat, %va - ret %vc -} - -define @vfmerge_vv_nxv4bf16( %va, %vb, %cond) { -; CHECK-LABEL: vfmerge_vv_nxv4bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %vc = select %cond, %va, %vb - ret %vc -} - -define @vfmerge_fv_nxv4bf16( %va, bfloat %b, %cond) { -; CHECK-ZVFH-LABEL: vfmerge_fv_nxv4bf16: -; CHECK-ZVFH: # %bb.0: -; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, m1, ta, ma -; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; CHECK-ZVFH-NEXT: ret -; -; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv4bf16: -; CHECK-ZVFHMIN: # %bb.0: -; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m2, ta, ma -; CHECK-ZVFHMIN-NEXT: vfmv.v.f v10, fa5 -; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, mu -; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t -; CHECK-ZVFHMIN-NEXT: ret - %head = insertelement poison, bfloat %b, i32 0 - %splat = shufflevector %head, poison, zeroinitializer - %vc = select %cond, %splat, %va - ret %vc -} - -define @vfmerge_vv_nxv8bf16( %va, %vb, %cond) { -; CHECK-LABEL: vfmerge_vv_nxv8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 -; CHECK-NEXT: ret - %vc = select %cond, %va, %vb - ret %vc -} - -define @vfmerge_fv_nxv8bf16( %va, bfloat %b, %cond) { -; CHECK-ZVFH-LABEL: vfmerge_fv_nxv8bf16: -; CHECK-ZVFH: # %bb.0: -; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, m2, ta, ma -; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; CHECK-ZVFH-NEXT: ret -; -; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv8bf16: -; CHECK-ZVFHMIN: # %bb.0: -; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m4, ta, ma -; CHECK-ZVFHMIN-NEXT: vfmv.v.f v12, fa5 -; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, mu -; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t -; CHECK-ZVFHMIN-NEXT: ret - %head = insertelement poison, bfloat %b, i32 0 - %splat = shufflevector %head, poison, zeroinitializer - %vc = select %cond, %splat, %va - ret %vc -} - -define @vfmerge_zv_nxv8bf16( %va, %cond) { -; CHECK-LABEL: vfmerge_zv_nxv8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma -; CHECK-NEXT: vmerge.vim v8, v8, 0, v0 -; CHECK-NEXT: ret - %vc = select %cond, splat (bfloat zeroinitializer), %va - ret %vc -} - -define @vmerge_truelhs_nxv8bf16_0( %va, %vb) { -; CHECK-LABEL: vmerge_truelhs_nxv8bf16_0: -; CHECK: # %bb.0: -; CHECK-NEXT: ret - %vc = select splat (i1 1), %va, %vb - ret %vc -} - -define @vmerge_falselhs_nxv8bf16_0( %va, %vb) { -; CHECK-LABEL: vmerge_falselhs_nxv8bf16_0: -; CHECK: # %bb.0: -; CHECK-NEXT: vmv2r.v v8, v10 -; CHECK-NEXT: ret - %vc = select zeroinitializer, %va, %vb - ret %vc -} - -define @vfmerge_vv_nxv16bf16( %va, %vb, %cond) { -; CHECK-LABEL: vfmerge_vv_nxv16bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0 -; CHECK-NEXT: ret - %vc = select %cond, %va, %vb - ret %vc -} - -define @vfmerge_fv_nxv16bf16( %va, bfloat %b, %cond) { -; CHECK-ZVFH-LABEL: vfmerge_fv_nxv16bf16: -; CHECK-ZVFH: # %bb.0: -; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, m4, ta, ma -; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; CHECK-ZVFH-NEXT: ret -; -; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv16bf16: -; CHECK-ZVFHMIN: # %bb.0: -; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m8, ta, ma -; CHECK-ZVFHMIN-NEXT: vfmv.v.f v16, fa5 -; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, mu -; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t -; CHECK-ZVFHMIN-NEXT: ret - %head = insertelement poison, bfloat %b, i32 0 - %splat = shufflevector %head, poison, zeroinitializer - %vc = select %cond, %splat, %va - ret %vc -} - -define @vfmerge_vv_nxv32bf16( %va, %vb, %cond) { -; CHECK-LABEL: vfmerge_vv_nxv32bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e16, m8, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v16, v8, v0 -; CHECK-NEXT: ret - %vc = select %cond, %va, %vb - ret %vc -} - -define @vfmerge_fv_nxv32bf16( %va, bfloat %b, %cond) { -; CHECK-ZVFH-LABEL: vfmerge_fv_nxv32bf16: -; CHECK-ZVFH: # %bb.0: -; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, m8, ta, ma -; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 -; CHECK-ZVFH-NEXT: ret -; -; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv32bf16: -; CHECK-ZVFHMIN: # %bb.0: -; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 -; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m8, ta, ma -; CHECK-ZVFHMIN-NEXT: vfmv.v.f v24, fa5 -; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma -; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v16, v24 -; CHECK-ZVFHMIN-NEXT: vmv.v.v v20, v16 -; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e16, m8, ta, ma -; CHECK-ZVFHMIN-NEXT: vmerge.vvm v8, v8, v16, v0 -; CHECK-ZVFHMIN-NEXT: ret - %head = insertelement poison, bfloat %b, i32 0 - %splat = shufflevector %head, poison, zeroinitializer - %vc = select %cond, %splat, %va - ret %vc -} diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll index d1049e14fa29..ee0617c93148 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfh,+v -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfh,+v -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfhmin,+v -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfhmin,+v -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s declare @llvm.vp.select.nxv1i1(, , , i32) @@ -922,75 +922,3 @@ define @select_unknown_T_T( %x, @llvm.vp.select.nxv2i1( %x, %y, %y, i32 %evl) ret %a } - -declare @llvm.vp.select.nxv1bf16(, , , i32) - -define @select_nxv1bf16( %a, %b, %c, i32 zeroext %evl) { -; CHECK-LABEL: select_nxv1bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, mf4, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %v = call @llvm.vp.select.nxv1bf16( %a, %b, %c, i32 %evl) - ret %v -} - -declare @llvm.vp.select.nxv2bf16(, , , i32) - -define @select_nxv2bf16( %a, %b, %c, i32 zeroext %evl) { -; CHECK-LABEL: select_nxv2bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %v = call @llvm.vp.select.nxv2bf16( %a, %b, %c, i32 %evl) - ret %v -} - -declare @llvm.vp.select.nxv4bf16(, , , i32) - -define @select_nxv4bf16( %a, %b, %c, i32 zeroext %evl) { -; CHECK-LABEL: select_nxv4bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 -; CHECK-NEXT: ret - %v = call @llvm.vp.select.nxv4bf16( %a, %b, %c, i32 %evl) - ret %v -} - -declare @llvm.vp.select.nxv8bf16(, , , i32) - -define @select_nxv8bf16( %a, %b, %c, i32 zeroext %evl) { -; CHECK-LABEL: select_nxv8bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m2, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 -; CHECK-NEXT: ret - %v = call @llvm.vp.select.nxv8bf16( %a, %b, %c, i32 %evl) - ret %v -} - -declare @llvm.vp.select.nxv16bf16(, , , i32) - -define @select_nxv16bf16( %a, %b, %c, i32 zeroext %evl) { -; CHECK-LABEL: select_nxv16bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m4, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0 -; CHECK-NEXT: ret - %v = call @llvm.vp.select.nxv16bf16( %a, %b, %c, i32 %evl) - ret %v -} - -declare @llvm.vp.select.nxv32bf16(, , , i32) - -define @select_nxv32bf16( %a, %b, %c, i32 zeroext %evl) { -; CHECK-LABEL: select_nxv32bf16: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e16, m8, ta, ma -; CHECK-NEXT: vmerge.vvm v8, v16, v8, v0 -; CHECK-NEXT: ret - %v = call @llvm.vp.select.nxv32bf16( %a, %b, %c, i32 %evl) - ret %v -} -- GitLab From 68413219dba47104fef388364a9973b584126f42 Mon Sep 17 00:00:00 2001 From: jiajie zhang <56027356+JumpMasterJJ@users.noreply.github.com> Date: Thu, 6 Jun 2024 12:28:02 +0800 Subject: [PATCH 040/462] [flang] Add GETCWD runtime and lowering intrinsics implementation (#92746) This patch add support of intrinsics GNU extension GETCWD https://github.com/llvm/llvm-project/issues/84203. Some usage info and example has been added to `flang/docs/Intrinsics.md`. The patch contains both the lowering and the runtime code and works on both Windows and Linux. | System | Implmentation | |-----------|--------------------| | Windows | _getcwd | | Linux |getcwd | --- flang/docs/Intrinsics.md | 31 +++++++++++++ .../flang/Optimizer/Builder/IntrinsicCall.h | 2 + .../flang/Optimizer/Builder/Runtime/Command.h | 5 +++ flang/include/flang/Runtime/command.h | 4 ++ flang/include/flang/Runtime/magic-numbers.h | 5 +++ flang/lib/Evaluate/intrinsics.cpp | 12 ++++- flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 35 +++++++++++++++ .../lib/Optimizer/Builder/Runtime/Command.cpp | 13 ++++++ flang/runtime/command.cpp | 26 +++++++++++ flang/runtime/stat.h | 1 + .../test/Lower/Intrinsics/getcwd-function.f90 | 23 ++++++++++ .../test/Lower/Intrinsics/getcwd-optional.f90 | 29 ++++++++++++ flang/test/Lower/Intrinsics/getcwd.f90 | 44 +++++++++++++++++++ flang/test/Semantics/getcwd.f90 | 35 +++++++++++++++ 14 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 flang/test/Lower/Intrinsics/getcwd-function.f90 create mode 100644 flang/test/Lower/Intrinsics/getcwd-optional.f90 create mode 100644 flang/test/Lower/Intrinsics/getcwd.f90 create mode 100644 flang/test/Semantics/getcwd.f90 diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md index 41129b10083b..8853d4d9e1c7 100644 --- a/flang/docs/Intrinsics.md +++ b/flang/docs/Intrinsics.md @@ -967,4 +967,35 @@ program test_etime print *, tarray(1) print *, tarray(2) end program test_etime +``` + +### Non-Standard Intrinsics: GETCWD + +#### Description +`GETCWD(C, STATUS)` returns current working directory. + +This intrinsic is provided in both subroutine and function forms; however, only one form can be used in any given program unit. + +*C* and *STATUS* are `INTENT(OUT)` and provide the following: + +| | | +|------------|---------------------------------------------------------------------------------------------------| +| `C` | Current work directory. The type shall be `CHARACTER` and of default kind. | +| `STATUS` | (Optional) Status flag. Returns 0 on success, a system specific and nonzero error code otherwise. The type shall be `INTEGER` and of a kind greater or equal to 4. | + +#### Usage and Info + +- **Standard:** GNU extension +- **Class:** Subroutine, function +- **Syntax:** `CALL GETCWD(C, STATUS)`, `STATUS = GETCWD(C)` + +#### Example +```Fortran +PROGRAM example_getcwd + CHARACTER(len=255) :: cwd + INTEGER :: status + CALL getcwd(cwd, status) + PRINT *, cwd + PRINT *, status +END PROGRAM ``` \ No newline at end of file diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h index c47e41eab18b..8ef5d59b92f0 100644 --- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h +++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h @@ -232,6 +232,8 @@ struct IntrinsicLibrary { mlir::Value genFloor(mlir::Type, llvm::ArrayRef); mlir::Value genFraction(mlir::Type resultType, mlir::ArrayRef args); + fir::ExtendedValue genGetCwd(std::optional resultType, + llvm::ArrayRef args); void genGetCommand(mlir::ArrayRef args); mlir::Value genGetPID(mlir::Type resultType, llvm::ArrayRef args); diff --git a/flang/include/flang/Optimizer/Builder/Runtime/Command.h b/flang/include/flang/Optimizer/Builder/Runtime/Command.h index 976fb3aa0b6f..0d60a367d999 100644 --- a/flang/include/flang/Optimizer/Builder/Runtime/Command.h +++ b/flang/include/flang/Optimizer/Builder/Runtime/Command.h @@ -53,5 +53,10 @@ mlir::Value genGetEnvVariable(fir::FirOpBuilder &, mlir::Location, mlir::Value length, mlir::Value trimName, mlir::Value errmsg); +/// Generate a call to the GetCwd runtime function which implements +/// the GETCWD intrinsic. +mlir::Value genGetCwd(fir::FirOpBuilder &builder, mlir::Location loc, + mlir::Value c); + } // namespace fir::runtime #endif // FORTRAN_OPTIMIZER_BUILDER_RUNTIME_COMMAND_H diff --git a/flang/include/flang/Runtime/command.h b/flang/include/flang/Runtime/command.h index c67d171c8e2f..7ab3f6442dcf 100644 --- a/flang/include/flang/Runtime/command.h +++ b/flang/include/flang/Runtime/command.h @@ -55,6 +55,10 @@ std::int32_t RTNAME(GetEnvVariable)(const Descriptor &name, const Descriptor *value = nullptr, const Descriptor *length = nullptr, bool trim_name = true, const Descriptor *errmsg = nullptr, const char *sourceFile = nullptr, int line = 0); + +// Calls getcwd() +std::int32_t RTNAME(GetCwd)( + const Descriptor &cwd, const char *sourceFile, int line); } } // namespace Fortran::runtime diff --git a/flang/include/flang/Runtime/magic-numbers.h b/flang/include/flang/Runtime/magic-numbers.h index 38ccc5e7d3df..1cded1fd6323 100644 --- a/flang/include/flang/Runtime/magic-numbers.h +++ b/flang/include/flang/Runtime/magic-numbers.h @@ -68,6 +68,11 @@ Additional status code for a bad pointer DEALLOCATE. #endif #define FORTRAN_RUNTIME_STAT_BAD_POINTER_DEALLOCATION 110 +#if 0 +Status codes for GETCWD. +#endif +#define FORTRAN_RUNTIME_STAT_MISSING_CWD 111 + #if 0 ieee_class_type values The sequence is that of F18 Clause 17.2p3, but nothing depends on that. diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp index 12d13422574b..58c1b6989f49 100644 --- a/flang/lib/Evaluate/intrinsics.cpp +++ b/flang/lib/Evaluate/intrinsics.cpp @@ -514,6 +514,10 @@ static const IntrinsicInterface genericIntrinsicFunction[]{ {"gamma", {{"x", SameReal}}, SameReal}, {"get_team", {{"level", DefaultInt, Rank::scalar, Optionality::optional}}, TeamType, Rank::scalar, IntrinsicClass::transformationalFunction}, + {"getcwd", + {{"c", DefaultChar, Rank::scalar, Optionality::required, + common::Intent::Out}}, + TypePattern{IntType, KindCode::greaterOrEqualToKind, 4}}, {"getpid", {}, DefaultInt}, {"huge", {{"x", SameIntOrReal, Rank::anyOrAssumedRank, Optionality::required, @@ -1406,6 +1410,12 @@ static const IntrinsicInterface intrinsicSubroutine[]{ {"errmsg", DefaultChar, Rank::scalar, Optionality::optional, common::Intent::InOut}}, {}, Rank::elemental, IntrinsicClass::impureSubroutine}, + {"getcwd", + {{"c", DefaultChar, Rank::scalar, Optionality::required, + common::Intent::Out}, + {"status", TypePattern{IntType, KindCode::greaterOrEqualToKind, 4}, + Rank::scalar, Optionality::optional, common::Intent::Out}}, + {}, Rank::elemental, IntrinsicClass::impureSubroutine}, {"move_alloc", {{"from", SameType, Rank::known, Optionality::required, common::Intent::InOut}, @@ -2574,7 +2584,7 @@ bool IntrinsicProcTable::Implementation::IsDualIntrinsic( const std::string &name) const { // Collection for some intrinsics with function and subroutine form, // in order to pass the semantic check. - static const std::string dualIntrinsic[]{{"etime"}}; + static const std::string dualIntrinsic[]{{"etime"}, {"getcwd"}}; return std::find_if(std::begin(dualIntrinsic), std::end(dualIntrinsic), [&name](const std::string &dualName) { diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp index 072899751bc8..d3f6fa16ac80 100644 --- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -280,6 +280,10 @@ static constexpr IntrinsicHandler handlers[]{ {"trim_name", asAddr, handleDynamicOptional}, {"errmsg", asBox, handleDynamicOptional}}}, /*isElemental=*/false}, + {"getcwd", + &I::genGetCwd, + {{{"c", asBox}, {"status", asAddr, handleDynamicOptional}}}, + /*isElemental=*/false}, {"getpid", &I::genGetPID}, {"iachar", &I::genIchar}, {"iall", @@ -3476,6 +3480,37 @@ mlir::Value IntrinsicLibrary::genFraction(mlir::Type resultType, fir::runtime::genFraction(builder, loc, fir::getBase(args[0]))); } +// GETCWD +fir::ExtendedValue +IntrinsicLibrary::genGetCwd(std::optional resultType, + llvm::ArrayRef args) { + assert((args.size() == 1 && resultType.has_value()) || + (args.size() >= 1 && !resultType.has_value())); + + mlir::Value cwd = fir::getBase(args[0]); + mlir::Value statusValue = fir::runtime::genGetCwd(builder, loc, cwd); + + if (resultType.has_value()) { + // Function form, return status. + return statusValue; + } else { + // Subroutine form, store status and return none. + const fir::ExtendedValue &status = args[1]; + if (!isStaticallyAbsent(status)) { + mlir::Value statusAddr = fir::getBase(status); + mlir::Value statusIsPresentAtRuntime = + builder.genIsNotNullAddr(loc, statusAddr); + builder.genIfThen(loc, statusIsPresentAtRuntime) + .genThen([&]() { + builder.createStoreWithConvert(loc, statusValue, statusAddr); + }) + .end(); + } + } + + return {}; +} + // GET_COMMAND void IntrinsicLibrary::genGetCommand(llvm::ArrayRef args) { assert(args.size() == 4); diff --git a/flang/lib/Optimizer/Builder/Runtime/Command.cpp b/flang/lib/Optimizer/Builder/Runtime/Command.cpp index 1d719e7bbd9a..8320d89493b3 100644 --- a/flang/lib/Optimizer/Builder/Runtime/Command.cpp +++ b/flang/lib/Optimizer/Builder/Runtime/Command.cpp @@ -88,3 +88,16 @@ mlir::Value fir::runtime::genGetEnvVariable(fir::FirOpBuilder &builder, sourceFile, sourceLine); return builder.create(loc, runtimeFunc, args).getResult(0); } + +mlir::Value fir::runtime::genGetCwd(fir::FirOpBuilder &builder, + mlir::Location loc, mlir::Value cwd) { + mlir::func::FuncOp func = + fir::runtime::getRuntimeFunc(loc, builder); + auto runtimeFuncTy = func.getFunctionType(); + mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc); + mlir::Value sourceLine = + fir::factory::locationToLineNo(builder, loc, runtimeFuncTy.getInput(2)); + llvm::SmallVector args = fir::runtime::createArguments( + builder, loc, runtimeFuncTy, cwd, sourceFile, sourceLine); + return builder.create(loc, func, args).getResult(0); +} diff --git a/flang/runtime/command.cpp b/flang/runtime/command.cpp index b573c5dfd797..e642248a25e6 100644 --- a/flang/runtime/command.cpp +++ b/flang/runtime/command.cpp @@ -17,12 +17,19 @@ #ifdef _WIN32 #include "flang/Common/windows-include.h" +#include +#define getcwd _getcwd +#define PATH_MAX MAX_PATH // On Windows GetCurrentProcessId returns a DWORD aka uint32_t #include inline pid_t getpid() { return GetCurrentProcessId(); } #else #include //getpid() + +#ifndef PATH_MAX +#define PATH_MAX 4096 +#endif #endif namespace Fortran::runtime { @@ -239,4 +246,23 @@ std::int32_t RTNAME(GetEnvVariable)(const Descriptor &name, return StatOk; } +std::int32_t RTNAME(GetCwd)( + const Descriptor &cwd, const char *sourceFile, int line) { + Terminator terminator{sourceFile, line}; + + RUNTIME_CHECK(terminator, IsValidCharDescriptor(&cwd)); + + char *buf{(char *)AllocateMemoryOrCrash(terminator, PATH_MAX)}; + + if (!getcwd(buf, PATH_MAX)) { + return StatMissingCurrentWorkDirectory; + } + + std::int64_t strLen{StringLength(buf)}; + std::int32_t status{CopyCharsToDescriptor(cwd, buf, strLen)}; + + std::free(buf); + return status; +} + } // namespace Fortran::runtime diff --git a/flang/runtime/stat.h b/flang/runtime/stat.h index 4f46f52ecb29..71faeb027d90 100644 --- a/flang/runtime/stat.h +++ b/flang/runtime/stat.h @@ -41,6 +41,7 @@ enum Stat { StatLocked = FORTRAN_RUNTIME_STAT_LOCKED, StatLockedOtherImage = FORTRAN_RUNTIME_STAT_LOCKED_OTHER_IMAGE, StatMissingEnvVariable = FORTRAN_RUNTIME_STAT_MISSING_ENV_VAR, + StatMissingCurrentWorkDirectory = FORTRAN_RUNTIME_STAT_MISSING_CWD, StatStoppedImage = FORTRAN_RUNTIME_STAT_STOPPED_IMAGE, StatUnlocked = FORTRAN_RUNTIME_STAT_UNLOCKED, StatUnlockedFailedImage = FORTRAN_RUNTIME_STAT_UNLOCKED_FAILED_IMAGE, diff --git a/flang/test/Lower/Intrinsics/getcwd-function.f90 b/flang/test/Lower/Intrinsics/getcwd-function.f90 new file mode 100644 index 000000000000..50b64729294f --- /dev/null +++ b/flang/test/Lower/Intrinsics/getcwd-function.f90 @@ -0,0 +1,23 @@ +! Test GETCWD with dynamically optional arguments. +! RUN: bbc -emit-fir %s -o - | FileCheck %s + +! CHECK-LABEL: func.func @_QPtest( +! CHECK-SAME: %[[cwdArg:.*]]: !fir.boxchar<1> {fir.bindc_name = "cwd"}) -> i32 { +integer function test(cwd) + CHARACTER(len=255) :: cwd + test = getcwd(cwd) + ! CHECK-NEXT: %[[c8:.*]] = arith.constant 8 : i32 + ! CHECK-NEXT: %[[c255:.*]] = arith.constant 255 : index + ! CHECK-NEXT: %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope + ! CHECK-NEXT: %[[cwdUnbox:.*]]:2 = fir.unboxchar %[[cwdArg]] : (!fir.boxchar<1>) -> (!fir.ref>, index) + ! CHECK-NEXT: %[[cwdCast:.*]] = fir.convert %[[cwdUnbox]]#0 : (!fir.ref>) -> !fir.ref> + ! CHECK-NEXT: %[[cwdDeclare:.*]] = fir.declare %[[cwdCast]] typeparams %[[c255]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFtestEcwd"} : (!fir.ref>, index, !fir.dscope) -> !fir.ref> + ! CHECK-NEXT: %[[test:.*]] = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFtestEtest"} + ! CHECK-NEXT: %[[testAddr:.*]] = fir.declare %[[test]] {uniq_name = "_QFtestEtest"} : (!fir.ref) -> !fir.ref + ! CHECK-NEXT: %[[cwdBox:.*]] = fir.embox %[[cwdDeclare]] : (!fir.ref>) -> !fir.box> + ! CHECK: %[[cwd:.*]] = fir.convert %[[cwdBox]] : (!fir.box>) -> !fir.box + ! CHECK: %[[statusValue:.*]] = fir.call @_FortranAGetCwd(%[[cwd]], %[[VAL_9:.*]], %[[c8]]) fastmath : (!fir.box, !fir.ref, i32) -> i32 + ! CHECK-NEXT: fir.store %[[statusValue]] to %[[testAddr]] : !fir.ref + ! CHECK-NEXT: %[[returnValue:.*]] = fir.load %[[testAddr]] : !fir.ref + ! CHECK-NEXT: return %[[returnValue]] : i32 +end function diff --git a/flang/test/Lower/Intrinsics/getcwd-optional.f90 b/flang/test/Lower/Intrinsics/getcwd-optional.f90 new file mode 100644 index 000000000000..3e2a221f0c3f --- /dev/null +++ b/flang/test/Lower/Intrinsics/getcwd-optional.f90 @@ -0,0 +1,29 @@ +! Test GETCWD with dynamically optional arguments. +! RUN: bbc -emit-fir %s -o - | FileCheck %s + + +! CHECK-LABEL: func.func @_QPtest( +! CHECK-SAME: %[[cwdArg:.*]]: !fir.boxchar<1> {fir.bindc_name = "cwd"}, +! CHECK-SAME: %[[statusArg:.*]]: !fir.ref {fir.bindc_name = "status", fir.optional}) { +subroutine test(cwd, status) + CHARACTER(len=255) :: cwd + INTEGER, OPTIONAL :: status + call getcwd(cwd, status) + ! CHECK-NEXT: %[[c0:.*]] = arith.constant 0 : i64 + ! CHECK-NEXT: %[[c11:.*]] = arith.constant 11 : i32 + ! CHECK-NEXT: %[[c255:.*]] = arith.constant 255 : index + ! CHECK-NEXT: %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope + ! CHECK-NEXT: %[[cwdUnbox:.*]]:2 = fir.unboxchar %[[cwdArg]] : (!fir.boxchar<1>) -> (!fir.ref>, index) + ! CHECK-NEXT: %[[cwdCast:.*]] = fir.convert %[[cwdUnbox]]#0 : (!fir.ref>) -> !fir.ref> + ! CHECK-NEXT: %[[cwdDeclare:.*]] = fir.declare %[[cwdCast]] typeparams %[[c255]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFtestEcwd"} : (!fir.ref>, index, !fir.dscope) -> !fir.ref> + ! CHECK-NEXT: %[[statusAddr:.*]] = fir.declare %[[statusArg]] dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtestEstatus"} : (!fir.ref, !fir.dscope) -> !fir.ref + ! CHECK-NEXT: %[[cwdBox:.*]] = fir.embox %[[cwdDeclare]] : (!fir.ref>) -> !fir.box> + ! CHECK: %[[cwd:.*]] = fir.convert %[[cwdBox]] : (!fir.box>) -> !fir.box + ! CHECK: %[[statusValue:.*]] = fir.call @_FortranAGetCwd(%[[cwd]], %[[VAL_8:.*]], %[[c11]]) fastmath : (!fir.box, !fir.ref, i32) -> i32 + ! CHECK-NEXT: %[[statusCast:.*]] = fir.convert %[[statusAddr]] : (!fir.ref) -> i64 + ! CHECK-NEXT: %[[isPresent:.*]] = arith.cmpi ne, %[[statusCast]], %[[c0]] : i64 + ! CHECK-NEXT: fir.if %[[isPresent]] { + ! CHECK-NEXT: fir.store %[[statusValue]] to %[[statusAddr]] : !fir.ref + ! CHECK-NEXT: } + ! CHECK-NEXT: return +end subroutine diff --git a/flang/test/Lower/Intrinsics/getcwd.f90 b/flang/test/Lower/Intrinsics/getcwd.f90 new file mode 100644 index 000000000000..fe207854aff0 --- /dev/null +++ b/flang/test/Lower/Intrinsics/getcwd.f90 @@ -0,0 +1,44 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s + +! CHECK-LABEL: func.func @_QPcwd_only( +! CHECK-SAME: %[[cwdArg:.*]]: !fir.boxchar<1> {fir.bindc_name = "cwd"}) { +subroutine cwd_only(cwd) + CHARACTER(len=255) :: cwd + call getcwd(cwd) + ! CHECK-NEXT: %[[c7:.*]] = arith.constant 7 : i32 + ! CHECK-NEXT: %[[c255:.*]] = arith.constant 255 : index + ! CHECK-NEXT: %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope + ! CHECK-NEXT: %[[cwdUnbox:.*]]:2 = fir.unboxchar %[[cwdArg]] : (!fir.boxchar<1>) -> (!fir.ref>, index) + ! CHECK-NEXT: %[[cwdCast:.*]] = fir.convert %[[cwdUnbox]]#0 : (!fir.ref>) -> !fir.ref> + ! CHECK-NEXT: %[[cwdDeclare:.*]] = fir.declare %[[cwdCast]] typeparams %[[c255]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFcwd_onlyEcwd"} : (!fir.ref>, index, !fir.dscope) -> !fir.ref> + ! CHECK-NEXT: %[[cwdBox:.*]] = fir.embox %[[cwdDeclare]] : (!fir.ref>) -> !fir.box> + ! CHECK: %[[cwd:.*]] = fir.convert %[[cwdBox]] : (!fir.box>) -> !fir.box + ! CHECK: %[[statusValue:.*]] = fir.call @_FortranAGetCwd(%[[cwd]], %[[VAL_7:.*]], %[[c7]]) fastmath : (!fir.box, !fir.ref, i32) -> i32 + ! CHECK-NEXT: return +end subroutine cwd_only + +! CHECK-LABEL: func.func @_QPall_arguments( +! CHECK-SAME: %[[cwdArg:.*]]: !fir.boxchar<1> {fir.bindc_name = "cwd"}, +! CHECK-SAME: %[[statusArg:.*]]: !fir.ref {fir.bindc_name = "status"}) { +subroutine all_arguments(cwd, status) + CHARACTER(len=255) :: cwd + INTEGER :: status + call getcwd(cwd, status) + ! CHECK-NEXT: %[[c0:.*]] = arith.constant 0 : i64 + ! CHECK-NEXT: %[[c26:.*]] = arith.constant 26 : i32 + ! CHECK-NEXT: %[[c255:.*]] = arith.constant 255 : index + ! CHECK-NEXT: %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope + ! CHECK-NEXT: %[[cwdUnbox:.*]]:2 = fir.unboxchar %[[cwdArg]] : (!fir.boxchar<1>) -> (!fir.ref>, index) + ! CHECK-NEXT: %[[cwdCast:.*]] = fir.convert %[[cwdUnbox]]#0 : (!fir.ref>) -> !fir.ref> + ! CHECK-NEXT: %[[cwdDeclare:.*]] = fir.declare %[[cwdCast]] typeparams %[[c255]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFall_argumentsEcwd"} : (!fir.ref>, index, !fir.dscope) -> !fir.ref> + ! CHECK-NEXT: %[[statusAddr:.*]] = fir.declare %[[statusArg]] dummy_scope %0 {uniq_name = "_QFall_argumentsEstatus"} : (!fir.ref, !fir.dscope) -> !fir.ref + ! CHECK-NEXT: %[[cwdBox:.*]] = fir.embox %[[cwdDeclare]] : (!fir.ref>) -> !fir.box> + ! CHECK: %[[cwd:.*]] = fir.convert %[[cwdBox]] : (!fir.box>) -> !fir.box + ! CHECK: %[[statusValue:.*]] = fir.call @_FortranAGetCwd(%[[cwd]], %[[VAL_8:.*]], %[[c26]]) fastmath : (!fir.box, !fir.ref, i32) -> i32 + ! CHECK-NEXT: %[[statusCast:.*]] = fir.convert %[[statusAddr]] : (!fir.ref) -> i64 + ! CHECK-NEXT: %[[isPresent:.*]] = arith.cmpi ne, %[[statusCast]], %[[c0]] : i64 + ! CHECK-NEXT: fir.if %[[isPresent]] { + ! CHECK-NEXT: fir.store %[[statusValue]] to %[[statusAddr]] : !fir.ref + ! CHECK-NEXT: } + ! CHECK-NEXT: return +end subroutine all_arguments \ No newline at end of file diff --git a/flang/test/Semantics/getcwd.f90 b/flang/test/Semantics/getcwd.f90 new file mode 100644 index 000000000000..b6ff16eeec5a --- /dev/null +++ b/flang/test/Semantics/getcwd.f90 @@ -0,0 +1,35 @@ +! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic +! Tests for the GETCWD intrinsics + +subroutine bad_kind_error(cwd, status) + CHARACTER(len=255) :: cwd + INTEGER(2) :: status + !ERROR: Actual argument for 'status=' has bad type or kind 'INTEGER(2)' + call getcwd(cwd, status) +end subroutine bad_kind_error + +subroutine bad_args_error() + !ERROR: missing mandatory 'c=' argument + call getcwd() +end subroutine bad_args_error + +subroutine bad_apply_form(cwd) + CHARACTER(len=255) :: cwd + INTEGER :: status + !Declaration of 'getcwd' + call getcwd(cwd, status) + !ERROR: Cannot call subroutine 'getcwd' like a function + status = getcwd(cwd) +end subroutine bad_apply_form + +subroutine good_subroutine(cwd, status) + CHARACTER(len=255) :: cwd + INTEGER :: status + call getcwd(cwd, status) +end subroutine good_subroutine + +subroutine good_function(cwd, status) + CHARACTER(len=255) :: cwd + INTEGER :: status + status = getcwd(cwd) +end subroutine good_function \ No newline at end of file -- GitLab From f6c1e65ddfa9e1a07919104be543a1f9eccbb519 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Thu, 6 Jun 2024 07:28:02 +0200 Subject: [PATCH 041/462] [clang] Implement a __is_bitwise_cloneable builtin type trait. (#86512) This patch implements a `__is_bitwise_cloneable` builtin in clang. The builtin is used as a guard to check a type can be safely bitwise copied by memcpy. It's functionally similar to `__is_trivially_copyable`, but covers a wider range of types (e.g. classes with virtual functions). The compiler guarantees that after copy, the destination object has the same object representations as the source object. And it is up to user to guarantee that program semantic constraints are satisfied. Context: https://discourse.llvm.org/t/extension-for-creating-objects-via-memcpy --- clang/docs/LanguageExtensions.rst | 24 ++++++++++++ clang/docs/ReleaseNotes.rst | 3 ++ clang/include/clang/AST/Type.h | 14 +++++++ clang/include/clang/Basic/TokenKinds.def | 2 + clang/lib/AST/Type.cpp | 37 +++++++++++++++++++ clang/lib/Sema/SemaExprCXX.cpp | 3 ++ ...builtin-is-bitwise-cloneable-fsanitize.cpp | 34 +++++++++++++++++ .../SemaCXX/builtin-is-bitwise-cloneable.cpp | 8 ++++ clang/test/SemaObjCXX/arc-type-traits.mm | 9 +++++ 9 files changed, 134 insertions(+) create mode 100644 clang/test/SemaCXX/builtin-is-bitwise-cloneable-fsanitize.cpp create mode 100644 clang/test/SemaCXX/builtin-is-bitwise-cloneable.cpp diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst index 46f99d0bbdd0..a49e4122ffc1 100644 --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -4016,6 +4016,30 @@ Note that the `size` argument must be a compile time constant. Note that this intrinsic cannot yet be called in a ``constexpr`` context. +``__is_bitwise_cloneable`` +-------------------------- + +A type trait is used to check whether a type can be safely copied by memcpy. + +**Syntax**: + +.. code-block:: c++ + + bool __is_bitwise_cloneable(Type) + +**Description**: + +Objects of bitwise cloneable types can be bitwise copied by memcpy/memmove. The +Clang compiler warrants that this behavior is well defined, and won't be +broken by compiler optimizations and sanitizers. + +For implicit-lifetime types, the lifetime of the new object is implicitly +started after the copy. For other types (e.g., classes with virtual methods), +the lifetime isn't started, and using the object results in undefined behavior +according to the C++ Standard. + +This builtin can be used in constant expressions. + Atomic Min/Max builtins with memory ordering -------------------------------------------- diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 69ac08133c9f..b9c9070fcb22 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -340,6 +340,9 @@ Non-comprehensive list of changes in this release ``-Winvalid-constexpr`` is not enabled for the function definition, which should result in mild compile-time performance improvements. +- Added ``__is_bitwise_cloneable`` which is used to check whether a type + can be safely copied by memcpy/memmove. + New Compiler Flags ------------------ - ``-fsanitize=implicit-bitfield-conversion`` checks implicit truncation and diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h index 263b632df23c..9eb3f6c09e3d 100644 --- a/clang/include/clang/AST/Type.h +++ b/clang/include/clang/AST/Type.h @@ -1120,6 +1120,20 @@ public: /// Return true if this is a trivially copyable type (C++0x [basic.types]p9) bool isTriviallyCopyableType(const ASTContext &Context) const; + /// Return true if the type is safe to bitwise copy using memcpy/memmove. + /// + /// This is an extension in clang: bitwise cloneable types act as trivially + /// copyable types, meaning their underlying bytes can be safely copied by + /// memcpy or memmove. After the copy, the destination object has the same + /// object representation. + /// + /// However, there are cases where it is not safe to copy: + /// - When sanitizers, such as AddressSanitizer, add padding with poison, + /// which can cause issues if those poisoned padding bits are accessed. + /// - Types with Objective-C lifetimes, where specific runtime + /// semantics may not be preserved during a bitwise copy. + bool isBitwiseCloneableType(const ASTContext &Context) const; + /// Return true if this is a trivially copyable type bool isTriviallyCopyConstructibleType(const ASTContext &Context) const; diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def index b5a0e9df9f7a..9c4b17465e18 100644 --- a/clang/include/clang/Basic/TokenKinds.def +++ b/clang/include/clang/Basic/TokenKinds.def @@ -542,6 +542,8 @@ TYPE_TRAIT_2(__reference_converts_from_temporary, ReferenceConvertsFromTemporary // is not exposed to users. TYPE_TRAIT_2(/*EmptySpellingName*/, IsDeducible, KEYCXX) +TYPE_TRAIT_1(__is_bitwise_cloneable, IsBitwiseCloneable, KEYALL) + // Embarcadero Expression Traits EXPRESSION_TRAIT(__is_lvalue_expr, IsLValueExpr, KEYCXX) EXPRESSION_TRAIT(__is_rvalue_expr, IsRValueExpr, KEYCXX) diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp index 2097b29b7e0b..3f88c3441eaf 100644 --- a/clang/lib/AST/Type.cpp +++ b/clang/lib/AST/Type.cpp @@ -2749,6 +2749,43 @@ bool QualType::isTriviallyCopyableType(const ASTContext &Context) const { /*IsCopyConstructible=*/false); } +// FIXME: each call will trigger a full computation, cache the result. +bool QualType::isBitwiseCloneableType(const ASTContext &Context) const { + auto CanonicalType = getCanonicalType(); + if (CanonicalType.hasNonTrivialObjCLifetime()) + return false; + if (CanonicalType->isArrayType()) + return Context.getBaseElementType(CanonicalType) + .isBitwiseCloneableType(Context); + + if (CanonicalType->isIncompleteType()) + return false; + const auto *RD = CanonicalType->getAsRecordDecl(); // struct/union/class + if (!RD) + return true; + + // Never allow memcpy when we're adding poisoned padding bits to the struct. + // Accessing these posioned bits will trigger false alarms on + // SanitizeAddressFieldPadding etc. + if (RD->mayInsertExtraPadding()) + return false; + + for (auto *const Field : RD->fields()) { + if (!Field->getType().isBitwiseCloneableType(Context)) + return false; + } + + if (const auto *CXXRD = dyn_cast(RD)) { + for (auto Base : CXXRD->bases()) + if (!Base.getType().isBitwiseCloneableType(Context)) + return false; + for (auto VBase : CXXRD->vbases()) + if (!VBase.getType().isBitwiseCloneableType(Context)) + return false; + } + return true; +} + bool QualType::isTriviallyCopyConstructibleType( const ASTContext &Context) const { return isTriviallyCopyableTypeImpl(*this, Context, diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp index 4487c618862c..eb0d987f63da 100644 --- a/clang/lib/Sema/SemaExprCXX.cpp +++ b/clang/lib/Sema/SemaExprCXX.cpp @@ -5126,6 +5126,7 @@ static bool CheckUnaryTypeTraitTypeCompleteness(Sema &S, TypeTrait UTT, case UTT_IsStandardLayout: case UTT_IsPOD: case UTT_IsLiteral: + case UTT_IsBitwiseCloneable: // By analogy, is_trivially_relocatable and is_trivially_equality_comparable // impose the same constraints. case UTT_IsTriviallyRelocatable: @@ -5619,6 +5620,8 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, TypeTrait UTT, return C.hasUniqueObjectRepresentations(T); case UTT_IsTriviallyRelocatable: return T.isTriviallyRelocatableType(C); + case UTT_IsBitwiseCloneable: + return T.isBitwiseCloneableType(C); case UTT_IsReferenceable: return T.isReferenceable(); case UTT_CanPassInRegs: diff --git a/clang/test/SemaCXX/builtin-is-bitwise-cloneable-fsanitize.cpp b/clang/test/SemaCXX/builtin-is-bitwise-cloneable-fsanitize.cpp new file mode 100644 index 000000000000..d47a39a0754c --- /dev/null +++ b/clang/test/SemaCXX/builtin-is-bitwise-cloneable-fsanitize.cpp @@ -0,0 +1,34 @@ +// RUN: %clang_cc1 -triple x86_64-unknown-linux -DSANITIZER_ENABLED -fsanitize=address -fsanitize-address-field-padding=1 %s +// RUN: %clang_cc1 -triple x86_64-unknown-linux %s + +struct S { + ~S() {} + virtual void foo() {} + + int buffer[1]; + int other_field = 0; +}; + +union U { + S s; +}; + +struct Derived : S {}; + +static_assert(!__is_trivially_copyable(S)); +#ifdef SANITIZER_ENABLED +// Don't allow memcpy when the struct has poisoned padding bits. +// The sanitizer adds posion padding bits to struct S. +static_assert(sizeof(S) > 16); +static_assert(!__is_bitwise_cloneable(S)); +static_assert(sizeof(U) == sizeof(S)); // no padding bit for U. +static_assert(!__is_bitwise_cloneable(U)); +static_assert(!__is_bitwise_cloneable(S[2])); +static_assert(!__is_bitwise_cloneable(Derived)); +#else +static_assert(sizeof(S) == 16); +static_assert(__is_bitwise_cloneable(S)); +static_assert(__is_bitwise_cloneable(U)); +static_assert(__is_bitwise_cloneable(S[2])); +static_assert(__is_bitwise_cloneable(Derived)); +#endif diff --git a/clang/test/SemaCXX/builtin-is-bitwise-cloneable.cpp b/clang/test/SemaCXX/builtin-is-bitwise-cloneable.cpp new file mode 100644 index 000000000000..1781cf48449f --- /dev/null +++ b/clang/test/SemaCXX/builtin-is-bitwise-cloneable.cpp @@ -0,0 +1,8 @@ +// RUN: %clang_cc1 -fsyntax-only -verify %s +// +struct DynamicClass { virtual int Foo(); }; +static_assert(!__is_trivially_copyable(DynamicClass)); +static_assert(__is_bitwise_cloneable(DynamicClass)); + +struct InComplete; // expected-note{{forward declaration}} +static_assert(!__is_bitwise_cloneable(InComplete)); // expected-error{{incomplete type 'InComplete' used in type trait expression}} diff --git a/clang/test/SemaObjCXX/arc-type-traits.mm b/clang/test/SemaObjCXX/arc-type-traits.mm index 2d30ae450f3b..25bc8b362140 100644 --- a/clang/test/SemaObjCXX/arc-type-traits.mm +++ b/clang/test/SemaObjCXX/arc-type-traits.mm @@ -221,3 +221,12 @@ TRAIT_IS_TRUE(__is_trivially_relocatable, __unsafe_unretained id); TRAIT_IS_TRUE(__is_trivially_relocatable, HasStrong); TRAIT_IS_FALSE(__is_trivially_relocatable, HasWeak); TRAIT_IS_TRUE(__is_trivially_relocatable, HasUnsafeUnretained); + +// __is_bitwise_cloneable +TRAIT_IS_FALSE(__is_bitwise_cloneable, __strong id); +TRAIT_IS_FALSE(__is_bitwise_cloneable, __weak id); +TRAIT_IS_FALSE(__is_bitwise_cloneable, __autoreleasing id); +TRAIT_IS_TRUE(__is_trivial, __unsafe_unretained id); +TRAIT_IS_FALSE(__is_bitwise_cloneable, HasStrong); +TRAIT_IS_FALSE(__is_bitwise_cloneable, HasWeak); +TRAIT_IS_TRUE(__is_bitwise_cloneable, HasUnsafeUnretained); -- GitLab From 46edc02eaac81bfdace3e1f906751cad114790da Mon Sep 17 00:00:00 2001 From: hev Date: Thu, 6 Jun 2024 14:05:56 +0800 Subject: [PATCH 042/462] [LoongArch] Adjust LA64 data layout by using n32:64 in layout string (#93814) Although i32 type is illegal in the backend, LA64 has pretty good support for i32 types by using W instructions. By adding n32 to the DataLayout string, middle end optimizations will consider i32 to be a native type. One known effect of this is enabling LoopStrengthReduce on loops with i32 induction variables. This can be beneficial because C/C++ code often has loops with i32 induction variables due to the use of `int` or `unsigned int`. If this patch exposes performance issues, those are better addressed by tuning LSR or other passes. --- clang/lib/Basic/Targets/LoongArch.h | 2 +- llvm/docs/ReleaseNotes.rst | 4 ++++ llvm/lib/IR/AutoUpgrade.cpp | 4 ++-- llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp | 2 +- llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp | 5 +++++ 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/clang/lib/Basic/Targets/LoongArch.h b/clang/lib/Basic/Targets/LoongArch.h index 68572843f2d7..5fc223483951 100644 --- a/clang/lib/Basic/Targets/LoongArch.h +++ b/clang/lib/Basic/Targets/LoongArch.h @@ -133,7 +133,7 @@ public: LongWidth = LongAlign = PointerWidth = PointerAlign = 64; IntMaxType = Int64Type = SignedLong; HasUnalignedAccess = true; - resetDataLayout("e-m:e-p:64:64-i64:64-i128:128-n64-S128"); + resetDataLayout("e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"); // TODO: select appropriate ABI. setABI("lp64d"); } diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst index 393b97efa547..c4a85620fc3e 100644 --- a/llvm/docs/ReleaseNotes.rst +++ b/llvm/docs/ReleaseNotes.rst @@ -116,6 +116,10 @@ Changes to the Hexagon Backend Changes to the LoongArch Backend -------------------------------- +* i32 is now a native type in the datalayout string. This enables + LoopStrengthReduce for loops with i32 induction variables, among other + optimizations. + Changes to the MIPS Backend --------------------------- diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index a7ed2de6e8a5..2f4b8351e747 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -5368,8 +5368,8 @@ std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) { return DL.empty() ? std::string("G1") : (DL + "-G1").str(); } - if (T.isRISCV64()) { - // Make i32 a native type for 64-bit RISC-V. + if (T.isLoongArch64() || T.isRISCV64()) { + // Make i32 a native type for 64-bit LoongArch and RISC-V. auto I = DL.find("-n64-"); if (I != StringRef::npos) return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str(); diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp index 83466d53f84d..c29c1b593321 100644 --- a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp @@ -46,7 +46,7 @@ static cl::opt static std::string computeDataLayout(const Triple &TT) { if (TT.isArch64Bit()) - return "e-m:e-p:64:64-i64:64-i128:128-n64-S128"; + return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"; assert(TT.isArch32Bit() && "only LA32 and LA64 are currently supported"); return "e-m:e-p:32:32-i64:64-n32-S128"; } diff --git a/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp index 7148e2902fa7..ca50187e5e5e 100644 --- a/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp +++ b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp @@ -63,6 +63,11 @@ TEST(DataLayoutUpgradeTest, ValidDataLayoutUpgrade) { "riscv64"), "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"); + // Check that LoongArch64 upgrades -n64 to -n32:64. + EXPECT_EQ(UpgradeDataLayoutString("e-m:e-p:64:64-i64:64-i128:128-n64-S128", + "loongarch64"), + "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"); + // Check that SPIR && SPIRV targets add -G1 if it's not present. EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32", "spir"), "e-p:32:32-G1"); EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32", "spir64"), "e-p:32:32-G1"); -- GitLab From 11c08327dab425b67d80e99850e654e4c6c17864 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Thu, 6 Jun 2024 08:23:00 +0200 Subject: [PATCH 043/462] [MLIR][LLVM] Improve module translation comment (NFC) (#94577) This commit enhances the docsting of `translateModuleToLLVMIR` as a followup to https://github.com/llvm/llvm-project/pull/94445 --- mlir/include/mlir/Target/LLVMIR/Export.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mlir/include/mlir/Target/LLVMIR/Export.h b/mlir/include/mlir/Target/LLVMIR/Export.h index 224496865513..893aaaa4faff 100644 --- a/mlir/include/mlir/Target/LLVMIR/Export.h +++ b/mlir/include/mlir/Target/LLVMIR/Export.h @@ -20,10 +20,11 @@ class Module; namespace mlir { class Operation; -/// Translate operation that satisfies LLVM dialect module requirements into an -/// LLVM IR module living in the given context. This translates operations from -/// any dilalect that has a registered implementation of -/// LLVMTranslationDialectInterface. +/// Translates a given LLVM dialect `module` into an LLVM IR module living in +/// the given context. Operates on any operation from dialects that provide a +/// registered implementation of the LLVMTranslationDialectInterface. Returns +/// nullptr when the translation fails. +/// Verifies the produced LLVM module, except when `disableVerification` is set. std::unique_ptr translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext, llvm::StringRef name = "LLVMDialectModule", -- GitLab From fb0c705dbf27e3ab84d726ad30e172806a530c21 Mon Sep 17 00:00:00 2001 From: Matheus Izvekov Date: Thu, 6 Jun 2024 04:14:10 -0300 Subject: [PATCH 044/462] [clang] NFCI: Make ASTContext optional in the AST text dumper again (#94522) --- clang/lib/AST/TextNodeDumper.cpp | 21 +++++++++++++-------- clang/lib/AST/Type.cpp | 1 - 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp index 8bacceea0f22..1076dcd40a69 100644 --- a/clang/lib/AST/TextNodeDumper.cpp +++ b/clang/lib/AST/TextNodeDumper.cpp @@ -958,6 +958,9 @@ void TextNodeDumper::dumpTemplateArgument(const TemplateArgument &TA) { } OS << " '" << Str << "'"; + if (!Context) + return; + if (TemplateArgument CanonTA = Context->getCanonicalTemplateArgument(TA); !CanonTA.structurallyEquals(TA)) { llvm::SmallString<128> CanonStr; @@ -1139,15 +1142,17 @@ void TextNodeDumper::dumpTemplateName(TemplateName TN, StringRef Label) { } OS << " '" << Str << "'"; - if (TemplateName CanonTN = Context->getCanonicalTemplateName(TN); - CanonTN != TN) { - llvm::SmallString<128> CanonStr; - { - llvm::raw_svector_ostream SS(CanonStr); - CanonTN.print(SS, PrintPolicy); + if (Context) { + if (TemplateName CanonTN = Context->getCanonicalTemplateName(TN); + CanonTN != TN) { + llvm::SmallString<128> CanonStr; + { + llvm::raw_svector_ostream SS(CanonStr); + CanonTN.print(SS, PrintPolicy); + } + if (CanonStr != Str) + OS << ":'" << CanonStr << "'"; } - if (CanonStr != Str) - OS << ":'" << CanonStr << "'"; } } dumpBareTemplateName(TN); diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp index 3f88c3441eaf..33acae2cbafa 100644 --- a/clang/lib/AST/Type.cpp +++ b/clang/lib/AST/Type.cpp @@ -4481,7 +4481,6 @@ static CachedProperties computeCachedProperties(const Type *T) { #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class,Base) case Type::Class: #include "clang/AST/TypeNodes.inc" // Treat instantiation-dependent types as external. - if (!T->isInstantiationDependentType()) T->dump(); assert(T->isInstantiationDependentType()); return CachedProperties(Linkage::External, false); -- GitLab From 60429fbf88607dc3081e9faafb767bfbe996cfaa Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 6 Jun 2024 09:31:52 +0200 Subject: [PATCH 045/462] [InstCombine] Add more tests for select equivalence fold (NFC) --- llvm/test/Transforms/InstCombine/select.ll | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll index d3cf2af4be65..d110cc9f9eb4 100644 --- a/llvm/test/Transforms/InstCombine/select.ll +++ b/llvm/test/Transforms/InstCombine/select.ll @@ -2797,6 +2797,19 @@ define <2 x i8> @select_replacement_add_eq_vec_poison(<2 x i8> %x, <2 x i8> %y) ret <2 x i8> %sel } +define <2 x i8> @select_replacement_add_eq_vec_undef(<2 x i8> %x, <2 x i8> %y) { +; CHECK-LABEL: @select_replacement_add_eq_vec_undef( +; CHECK-NEXT: [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[ADD:%.*]] = add <2 x i8> [[X]], +; CHECK-NEXT: [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[ADD]], <2 x i8> [[Y:%.*]] +; CHECK-NEXT: ret <2 x i8> [[SEL]] +; + %cmp = icmp eq <2 x i8> %x, + %add = add <2 x i8> %x, + %sel = select <2 x i1> %cmp, <2 x i8> %add, <2 x i8> %y + ret <2 x i8> %sel +} + define i8 @select_replacement_add_ne(i8 %x, i8 %y) { ; CHECK-LABEL: @select_replacement_add_ne( ; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[X:%.*]], 1 @@ -2835,6 +2848,21 @@ define i8 @select_replacement_sub_noundef(i8 %x, i8 noundef %y, i8 %z) { ret i8 %sel } +define i8 @select_replacement_sub_noundef_but_may_be_poison(i8 %x, i8 noundef %yy, i8 %z) { +; CHECK-LABEL: @select_replacement_sub_noundef_but_may_be_poison( +; CHECK-NEXT: [[Y:%.*]] = shl nuw i8 [[YY:%.*]], 1 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[Y]], [[X:%.*]] +; CHECK-NEXT: [[SUB:%.*]] = sub i8 [[X]], [[Y]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], i8 [[SUB]], i8 [[Z:%.*]] +; CHECK-NEXT: ret i8 [[SEL]] +; + %y = shl nuw i8 %yy, 1 + %cmp = icmp eq i8 %x, %y + %sub = sub i8 %x, %y + %sel = select i1 %cmp, i8 %sub, i8 %z + ret i8 %sel +} + ; TODO: The transform is also safe without noundef. define i8 @select_replacement_sub(i8 %x, i8 %y, i8 %z) { ; CHECK-LABEL: @select_replacement_sub( -- GitLab From 9bea770b63861250c8bbec46f68df73da4f09508 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 6 Jun 2024 09:34:26 +0200 Subject: [PATCH 046/462] [InstCombine] Only requite not-undef in select equiv fold As the comment already indicates, only replacement with undef is problematic, as it introduces an additional use of undef. Use the correct ValueTracking helper. --- llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp | 6 ++---- llvm/test/Transforms/InstCombine/select.ll | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index 313beb7b6407..d2aaa5e23054 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -1294,8 +1294,7 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel, // X == Y ? X : Z with X == Y ? Y : Z, as that would lead to an infinite // replacement cycle. Value *CmpLHS = Cmp.getOperand(0), *CmpRHS = Cmp.getOperand(1); - if (TrueVal != CmpLHS && - isGuaranteedNotToBeUndefOrPoison(CmpRHS, SQ.AC, &Sel, &DT)) { + if (TrueVal != CmpLHS && isGuaranteedNotToBeUndef(CmpRHS, SQ.AC, &Sel, &DT)) { if (Value *V = simplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, SQ, /* AllowRefinement */ true)) // Require either the replacement or the simplification result to be a @@ -1316,8 +1315,7 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel, if (replaceInInstruction(TrueVal, CmpLHS, CmpRHS)) return &Sel; } - if (TrueVal != CmpRHS && - isGuaranteedNotToBeUndefOrPoison(CmpLHS, SQ.AC, &Sel, &DT)) + if (TrueVal != CmpRHS && isGuaranteedNotToBeUndef(CmpLHS, SQ.AC, &Sel, &DT)) if (Value *V = simplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, SQ, /* AllowRefinement */ true)) if (isa(CmpLHS) || isa(V)) diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll index d110cc9f9eb4..29b4d602ffa0 100644 --- a/llvm/test/Transforms/InstCombine/select.ll +++ b/llvm/test/Transforms/InstCombine/select.ll @@ -2852,8 +2852,7 @@ define i8 @select_replacement_sub_noundef_but_may_be_poison(i8 %x, i8 noundef %y ; CHECK-LABEL: @select_replacement_sub_noundef_but_may_be_poison( ; CHECK-NEXT: [[Y:%.*]] = shl nuw i8 [[YY:%.*]], 1 ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[Y]], [[X:%.*]] -; CHECK-NEXT: [[SUB:%.*]] = sub i8 [[X]], [[Y]] -; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], i8 [[SUB]], i8 [[Z:%.*]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], i8 0, i8 [[Z:%.*]] ; CHECK-NEXT: ret i8 [[SEL]] ; %y = shl nuw i8 %yy, 1 -- GitLab From 16e2ec82ac45701f9c55ab917e30f38dbae6f79a Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 6 Jun 2024 09:39:33 +0200 Subject: [PATCH 047/462] [ValueTracking] Make undef element check more precise If we're only checking for undef, then also only look for undef elements in the vector (rather than undef and poison). --- llvm/lib/Analysis/ValueTracking.cpp | 11 +++++++---- llvm/test/Transforms/InstCombine/select.ll | 3 +-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 08138a5e2f2d..782c28c94483 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -7296,10 +7296,13 @@ static bool isGuaranteedNotToBeUndefOrPoison( isa(C) || isa(C)) return true; - if (C->getType()->isVectorTy() && !isa(C)) - return (!includesUndef(Kind) ? !C->containsPoisonElement() - : !C->containsUndefOrPoisonElement()) && - !C->containsConstantExpression(); + if (C->getType()->isVectorTy() && !isa(C)) { + if (includesUndef(Kind) && C->containsUndefElement()) + return false; + if (includesPoison(Kind) && C->containsPoisonElement()) + return false; + return !C->containsConstantExpression(); + } } // Strip cast operations from a pointer value. diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll index 29b4d602ffa0..a0ee4383495c 100644 --- a/llvm/test/Transforms/InstCombine/select.ll +++ b/llvm/test/Transforms/InstCombine/select.ll @@ -2787,8 +2787,7 @@ define <2 x i8> @select_replacement_add_eq_vec_nonuniform(<2 x i8> %x, <2 x i8> define <2 x i8> @select_replacement_add_eq_vec_poison(<2 x i8> %x, <2 x i8> %y) { ; CHECK-LABEL: @select_replacement_add_eq_vec_poison( ; CHECK-NEXT: [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[ADD:%.*]] = add <2 x i8> [[X]], -; CHECK-NEXT: [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[ADD]], <2 x i8> [[Y:%.*]] +; CHECK-NEXT: [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> , <2 x i8> [[Y:%.*]] ; CHECK-NEXT: ret <2 x i8> [[SEL]] ; %cmp = icmp eq <2 x i8> %x, -- GitLab From e0ac087ff004f7a63ba64b9685f4f098d6ee54c5 Mon Sep 17 00:00:00 2001 From: Sameer Sahasrabuddhe Date: Thu, 6 Jun 2024 13:13:46 +0530 Subject: [PATCH 048/462] [LoopUnroll] Consider convergence control tokens when unrolling (#91715) - There is no restriction on a loop with controlled convergent operations when the relevant tokens are defined and used within the loop. - When a token defined outside a loop is used inside (also called a loop convergence heart), unrolling is allowed only in the absence of remainder or runtime checks. - When a token defined inside a loop is used outside, such a loop is said to be "extended". This loop can only be unrolled by also duplicating the extended part lying outside the loop. Such unrolling is disabled for now. - Clean up loop hearts: When unrolling a loop with a heart, duplicating the heart will introduce multiple static uses of a convergence control token in a cycle that does not contain its definition. This violates the static rules for tokens, and needs to be cleaned up into a single occurrence of the intrinsic. - Spell out the initializer for UnrollLoopOptions to improve readability. Original implementation [D85605] by Nicolai Haehnle . --- llvm/include/llvm/Analysis/CodeMetrics.h | 9 +- llvm/include/llvm/Analysis/LoopInfo.h | 3 + llvm/include/llvm/IR/InstrTypes.h | 8 + llvm/include/llvm/IR/IntrinsicInst.h | 19 +- .../llvm/Transforms/Utils/UnrollLoop.h | 7 +- llvm/lib/Analysis/CodeMetrics.cpp | 53 +- llvm/lib/Analysis/LoopInfo.cpp | 20 + llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 2 +- .../Transforms/Scalar/DFAJumpThreading.cpp | 3 +- .../Scalar/LoopUnrollAndJamPass.cpp | 8 +- llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 57 +- .../Transforms/Utils/LoopRotationUtils.cpp | 2 +- llvm/lib/Transforms/Utils/LoopUnroll.cpp | 47 +- .../Transforms/Utils/LoopUnrollRuntime.cpp | 17 +- .../LoopUnroll/convergent.controlled.ll | 562 ++++++++++++++++++ 15 files changed, 748 insertions(+), 69 deletions(-) create mode 100644 llvm/test/Transforms/LoopUnroll/convergent.controlled.ll diff --git a/llvm/include/llvm/Analysis/CodeMetrics.h b/llvm/include/llvm/Analysis/CodeMetrics.h index a9431bca1125..d09018daf954 100644 --- a/llvm/include/llvm/Analysis/CodeMetrics.h +++ b/llvm/include/llvm/Analysis/CodeMetrics.h @@ -20,12 +20,15 @@ namespace llvm { class AssumptionCache; class BasicBlock; +class Instruction; class Loop; class Function; template class SmallPtrSetImpl; class TargetTransformInfo; class Value; +enum struct ConvergenceKind { None, Controlled, ExtendedLoop, Uncontrolled }; + /// Utility to calculate the size and a few similar metrics for a set /// of basic blocks. struct CodeMetrics { @@ -42,8 +45,8 @@ struct CodeMetrics { /// one or more 'noduplicate' instructions. bool notDuplicatable = false; - /// True if this function contains a call to a convergent function. - bool convergent = false; + /// The kind of convergence specified in this function. + ConvergenceKind Convergence = ConvergenceKind::None; /// True if this function calls alloca (in the C sense). bool usesDynamicAlloca = false; @@ -77,7 +80,7 @@ struct CodeMetrics { /// Add information about a block to the current state. void analyzeBasicBlock(const BasicBlock *BB, const TargetTransformInfo &TTI, const SmallPtrSetImpl &EphValues, - bool PrepareForLTO = false); + bool PrepareForLTO = false, const Loop *L = nullptr); /// Collect a loop's ephemeral values (those used only by an assume /// or similar intrinsics in the loop). diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h index 52084630560c..4f06a7e889f9 100644 --- a/llvm/include/llvm/Analysis/LoopInfo.h +++ b/llvm/include/llvm/Analysis/LoopInfo.h @@ -649,6 +649,9 @@ int getIntLoopAttribute(const Loop *TheLoop, StringRef Name, int Default = 0); std::optional findStringMetadataForLoop(const Loop *TheLoop, StringRef Name); +/// Find the convergence heart of the loop. +CallBase *getLoopConvergenceHeart(const Loop *TheLoop); + /// Look for the loop attribute that requires progress within the loop. /// Note: Most consumers probably want "isMustProgress" which checks /// the containing function attribute too. diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h index 9dd1bb455a71..441e6a1e7984 100644 --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -1588,6 +1588,14 @@ public: static CallBase *removeOperandBundle(CallBase *CB, uint32_t ID, BasicBlock::iterator InsertPt); + /// Return the convergence control token for this call, if it exists. + Value *getConvergenceControlToken() const { + if (auto Bundle = getOperandBundle(llvm::LLVMContext::OB_convergencectrl)) { + return Bundle->Inputs[0].get(); + } + return nullptr; + } + static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Call || I->getOpcode() == Instruction::Invoke || diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index fcd3a1025ac1..9010e1a1c896 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -1799,17 +1799,14 @@ public: return isa(V) && classof(cast(V)); } - // Returns the convergence intrinsic referenced by |I|'s convergencectrl - // attribute if any. - static IntrinsicInst *getParentConvergenceToken(Instruction *I) { - auto *CI = dyn_cast(I); - if (!CI) - return nullptr; - - auto Bundle = CI->getOperandBundle(llvm::LLVMContext::OB_convergencectrl); - assert(Bundle->Inputs.size() == 1 && - Bundle->Inputs[0]->getType()->isTokenTy()); - return dyn_cast(Bundle->Inputs[0].get()); + bool isAnchor() { + return getIntrinsicID() == Intrinsic::experimental_convergence_anchor; + } + bool isEntry() { + return getIntrinsicID() == Intrinsic::experimental_convergence_entry; + } + bool isLoop() { + return getIntrinsicID() == Intrinsic::experimental_convergence_loop; } }; diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h index bd804dc11266..797c082333a7 100644 --- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h +++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h @@ -16,6 +16,7 @@ #define LLVM_TRANSFORMS_UTILS_UNROLLLOOP_H #include "llvm/ADT/DenseMap.h" +#include "llvm/Analysis/CodeMetrics.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Support/InstructionCost.h" @@ -73,6 +74,7 @@ struct UnrollLoopOptions { bool AllowExpensiveTripCount; bool UnrollRemainder; bool ForgetAllSCEV; + const Instruction *Heart = nullptr; }; LoopUnrollResult UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI, @@ -128,14 +130,15 @@ class UnrollCostEstimator { public: unsigned NumInlineCandidates; - bool Convergent; + ConvergenceKind Convergence; + bool ConvergenceAllowsRuntime; UnrollCostEstimator(const Loop *L, const TargetTransformInfo &TTI, const SmallPtrSetImpl &EphValues, unsigned BEInsns); /// Whether it is legal to unroll this loop. - bool canUnroll() const { return LoopSize.isValid() && !NotDuplicatable; } + bool canUnroll() const; uint64_t getRolledLoopSize() const { return *LoopSize.getValue(); } diff --git a/llvm/lib/Analysis/CodeMetrics.cpp b/llvm/lib/Analysis/CodeMetrics.cpp index 2637e2f97dbb..ea67b526423b 100644 --- a/llvm/lib/Analysis/CodeMetrics.cpp +++ b/llvm/lib/Analysis/CodeMetrics.cpp @@ -16,6 +16,7 @@ #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/Function.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/Support/Debug.h" #include "llvm/Support/InstructionCost.h" @@ -111,11 +112,24 @@ void CodeMetrics::collectEphemeralValues( completeEphemeralValues(Visited, Worklist, EphValues); } +static bool extendsConvergenceOutsideLoop(const Instruction &I, const Loop *L) { + if (!L) + return false; + if (!isa(I)) + return false; + for (const auto *U : I.users()) { + if (!L->contains(cast(U))) + return true; + } + return false; +} + /// Fill in the current structure with information gleaned from the specified /// block. void CodeMetrics::analyzeBasicBlock( const BasicBlock *BB, const TargetTransformInfo &TTI, - const SmallPtrSetImpl &EphValues, bool PrepareForLTO) { + const SmallPtrSetImpl &EphValues, bool PrepareForLTO, + const Loop *L) { ++NumBlocks; InstructionCost NumInstsBeforeThisBB = NumInsts; for (const Instruction &I : *BB) { @@ -163,19 +177,38 @@ void CodeMetrics::analyzeBasicBlock( if (isa(I) || I.getType()->isVectorTy()) ++NumVectorInsts; - if (I.getType()->isTokenTy() && I.isUsedOutsideOfBlock(BB)) + if (I.getType()->isTokenTy() && !isa(I) && + I.isUsedOutsideOfBlock(BB)) { + LLVM_DEBUG(dbgs() << I + << "\n Cannot duplicate a token value used outside " + "the current block (except convergence control).\n"); notDuplicatable = true; - - if (const CallInst *CI = dyn_cast(&I)) { - if (CI->cannotDuplicate()) - notDuplicatable = true; - if (CI->isConvergent()) - convergent = true; } - if (const InvokeInst *InvI = dyn_cast(&I)) - if (InvI->cannotDuplicate()) + if (const CallBase *CB = dyn_cast(&I)) { + if (CB->cannotDuplicate()) notDuplicatable = true; + // Compute a meet over the visited blocks for the following partial order: + // + // None -> { Controlled, ExtendedLoop, Uncontrolled} + // Controlled -> ExtendedLoop + if (Convergence <= ConvergenceKind::Controlled && CB->isConvergent()) { + if (isa(CB) || + CB->getConvergenceControlToken()) { + assert(Convergence != ConvergenceKind::Uncontrolled); + LLVM_DEBUG(dbgs() << "Found controlled convergence:\n" << I << "\n"); + if (extendsConvergenceOutsideLoop(I, L)) + Convergence = ConvergenceKind::ExtendedLoop; + else { + assert(Convergence != ConvergenceKind::ExtendedLoop); + Convergence = ConvergenceKind::Controlled; + } + } else { + assert(Convergence == ConvergenceKind::None); + Convergence = ConvergenceKind::Uncontrolled; + } + } + } NumInsts += TTI.getInstructionCost(&I, TargetTransformInfo::TCK_CodeSize); } diff --git a/llvm/lib/Analysis/LoopInfo.cpp b/llvm/lib/Analysis/LoopInfo.cpp index 369ab087ffc0..c34c4974382e 100644 --- a/llvm/lib/Analysis/LoopInfo.cpp +++ b/llvm/lib/Analysis/LoopInfo.cpp @@ -1105,6 +1105,26 @@ int llvm::getIntLoopAttribute(const Loop *TheLoop, StringRef Name, return getOptionalIntLoopAttribute(TheLoop, Name).value_or(Default); } +CallBase *llvm::getLoopConvergenceHeart(const Loop *TheLoop) { + BasicBlock *H = TheLoop->getHeader(); + for (Instruction &II : *H) { + if (auto *CB = dyn_cast(&II)) { + if (!CB->isConvergent()) + continue; + // This is the heart if it uses a token defined outside the loop. The + // verifier has already checked that only the loop intrinsic can use such + // a token. + if (auto *Token = CB->getConvergenceControlToken()) { + auto *TokenDef = cast(Token); + if (!TheLoop->contains(TokenDef->getParent())) + return CB; + } + return nullptr; + } + } + return nullptr; +} + bool llvm::isFinite(const Loop *L) { return L->getHeader()->getParent()->willReturn(); } diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index 2c4b45255d05..92213e19c9d9 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -3961,7 +3961,7 @@ static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI) { UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns); // Loop is not unrollable if the loop contains certain instructions. - if (!UCE.canUnroll() || UCE.Convergent) { + if (!UCE.canUnroll()) { LLVM_DEBUG(dbgs() << "Loop not considered unrollable\n"); return 1; } diff --git a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp index ba2546b8db0e..4371b821eae6 100644 --- a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp +++ b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp @@ -827,7 +827,8 @@ private: return false; } - if (Metrics.convergent) { + // FIXME: Allow jump threading with controlled convergence. + if (Metrics.Convergence != ConvergenceKind::None) { LLVM_DEBUG(dbgs() << "DFA Jump Threading: Not jump threading, contains " << "convergent instructions.\n"); ORE->emit([&]() { diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp index 7b4c54370e48..f8e2f1f28088 100644 --- a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp +++ b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp @@ -327,8 +327,7 @@ tryToUnrollAndJamLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, UnrollCostEstimator OuterUCE(L, TTI, EphValues, UP.BEInsns); if (!InnerUCE.canUnroll() || !OuterUCE.canUnroll()) { - LLVM_DEBUG(dbgs() << " Not unrolling loop which contains instructions" - << " which cannot be duplicated or have invalid cost.\n"); + LLVM_DEBUG(dbgs() << " Loop not considered unrollable\n"); return LoopUnrollResult::Unmodified; } @@ -341,7 +340,10 @@ tryToUnrollAndJamLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, LLVM_DEBUG(dbgs() << " Not unrolling loop with inlinable calls.\n"); return LoopUnrollResult::Unmodified; } - if (InnerUCE.Convergent || OuterUCE.Convergent) { + // FIXME: The call to canUnroll() allows some controlled convergent + // operations, but we block them here for future changes. + if (InnerUCE.Convergence != ConvergenceKind::None || + OuterUCE.Convergence != ConvergenceKind::None) { LLVM_DEBUG( dbgs() << " Not unrolling loop with convergent instructions.\n"); return LoopUnrollResult::Unmodified; diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp index 10fc9e9303e8..cbc35b6dd429 100644 --- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp +++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -684,11 +684,15 @@ UnrollCostEstimator::UnrollCostEstimator( const SmallPtrSetImpl &EphValues, unsigned BEInsns) { CodeMetrics Metrics; for (BasicBlock *BB : L->blocks()) - Metrics.analyzeBasicBlock(BB, TTI, EphValues); + Metrics.analyzeBasicBlock(BB, TTI, EphValues, /* PrepareForLTO= */ false, + L); NumInlineCandidates = Metrics.NumInlineCandidates; NotDuplicatable = Metrics.notDuplicatable; - Convergent = Metrics.convergent; + Convergence = Metrics.Convergence; LoopSize = Metrics.NumInsts; + ConvergenceAllowsRuntime = + Metrics.Convergence != ConvergenceKind::Uncontrolled && + !getLoopConvergenceHeart(L); // Don't allow an estimate of size zero. This would allows unrolling of loops // with huge iteration counts, which is a compile time problem even if it's @@ -701,6 +705,25 @@ UnrollCostEstimator::UnrollCostEstimator( LoopSize = BEInsns + 1; } +bool UnrollCostEstimator::canUnroll() const { + switch (Convergence) { + case ConvergenceKind::ExtendedLoop: + LLVM_DEBUG(dbgs() << " Convergence prevents unrolling.\n"); + return false; + default: + break; + } + if (!LoopSize.isValid()) { + LLVM_DEBUG(dbgs() << " Invalid loop size prevents unrolling.\n"); + return false; + } + if (NotDuplicatable) { + LLVM_DEBUG(dbgs() << " Non-duplicatable blocks prevent unrolling.\n"); + return false; + } + return true; +} + uint64_t UnrollCostEstimator::getUnrolledLoopSize( const TargetTransformInfo::UnrollingPreferences &UP, unsigned CountOverwrite) const { @@ -1206,8 +1229,7 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE, UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns); if (!UCE.canUnroll()) { - LLVM_DEBUG(dbgs() << " Not unrolling loop which contains instructions" - << " which cannot be duplicated or have invalid cost.\n"); + LLVM_DEBUG(dbgs() << " Loop not considered unrollable.\n"); return LoopUnrollResult::Unmodified; } @@ -1254,15 +1276,9 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE, // is unsafe -- it adds a control-flow dependency to the convergent // operation. Therefore restrict remainder loop (try unrolling without). // - // TODO: This is quite conservative. In practice, convergent_op() - // is likely to be called unconditionally in the loop. In this - // case, the program would be ill-formed (on most architectures) - // unless n were the same on all threads in a thread group. - // Assuming n is the same on all threads, any kind of unrolling is - // safe. But currently llvm's notion of convergence isn't powerful - // enough to express this. - if (UCE.Convergent) - UP.AllowRemainder = false; + // TODO: This is somewhat conservative; we could allow the remainder if the + // trip count is uniform. + UP.AllowRemainder &= UCE.ConvergenceAllowsRuntime; // Try to find the trip count upper bound if we cannot find the exact trip // count. @@ -1282,6 +1298,8 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE, if (!UP.Count) return LoopUnrollResult::Unmodified; + UP.Runtime &= UCE.ConvergenceAllowsRuntime; + if (PP.PeelCount) { assert(UP.Count == 1 && "Cannot perform peel and unroll in the same step"); LLVM_DEBUG(dbgs() << "PEELING loop %" << L->getHeader()->getName() @@ -1324,11 +1342,16 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE, // Unroll the loop. Loop *RemainderLoop = nullptr; + UnrollLoopOptions ULO; + ULO.Count = UP.Count; + ULO.Force = UP.Force; + ULO.AllowExpensiveTripCount = UP.AllowExpensiveTripCount; + ULO.UnrollRemainder = UP.UnrollRemainder; + ULO.Runtime = UP.Runtime; + ULO.ForgetAllSCEV = ForgetAllSCEV; + ULO.Heart = getLoopConvergenceHeart(L); LoopUnrollResult UnrollResult = UnrollLoop( - L, - {UP.Count, UP.Force, UP.Runtime, UP.AllowExpensiveTripCount, - UP.UnrollRemainder, ForgetAllSCEV}, - LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop, AA); + L, ULO, LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop, AA); if (UnrollResult == LoopUnrollResult::Unmodified) return LoopUnrollResult::Unmodified; diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp index 08ba65d9483e..3d950b151cd3 100644 --- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp @@ -460,7 +460,7 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) { L->dump()); return Rotated; } - if (Metrics.convergent) { + if (Metrics.Convergence != ConvergenceKind::None) { LLVM_DEBUG(dbgs() << "LoopRotation: NOT rotating - contains convergent " "instructions: "; L->dump()); diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp index 1216538195fb..90d7b99e9d81 100644 --- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp +++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp @@ -419,6 +419,26 @@ void llvm::simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI, } } +// Loops containing convergent instructions that are uncontrolled or controlled +// from outside the loop must have a count that divides their TripMultiple. +LLVM_ATTRIBUTE_USED +static bool canHaveUnrollRemainder(const Loop *L) { + if (getLoopConvergenceHeart(L)) + return false; + + // Check for uncontrolled convergent operations. + for (auto &BB : L->blocks()) { + for (auto &I : *BB) { + if (isa(I)) + return true; + if (auto *CB = dyn_cast(&I)) + if (CB->isConvergent()) + return CB->getConvergenceControlToken(); + } + } + return true; +} + /// Unroll the given loop by Count. The loop must be in LCSSA form. Unrolling /// can only fail when the loop's latch block is not terminated by a conditional /// branch instruction. However, if the trip count (and multiple) are not known, @@ -564,19 +584,8 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI, return LoopUnrollResult::Unmodified; } - // Loops containing convergent instructions cannot use runtime unrolling, - // as the prologue/epilogue may add additional control-dependencies to - // convergent operations. - LLVM_DEBUG( - { - bool HasConvergent = false; - for (auto &BB : L->blocks()) - for (auto &I : *BB) - if (auto *CB = dyn_cast(&I)) - HasConvergent |= CB->isConvergent(); - assert((!HasConvergent || !ULO.Runtime) && - "Can't runtime unroll if loop contains a convergent operation."); - }); + assert((!ULO.Runtime || canHaveUnrollRemainder(L)) && + "Can't runtime unroll if loop contains a convergent operation."); bool EpilogProfitability = UnrollRuntimeEpilog.getNumOccurrences() ? UnrollRuntimeEpilog @@ -722,7 +731,7 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI, if (OldLoop) LoopsToSimplify.insert(NewLoops[OldLoop]); - if (*BB == Header) + if (*BB == Header) { // Loop over all of the PHI nodes in the block, changing them to use // the incoming values from the previous block. for (PHINode *OrigPHI : OrigPHINode) { @@ -735,6 +744,16 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI, NewPHI->eraseFromParent(); } + // Eliminate copies of the loop heart intrinsic, if any. + if (ULO.Heart) { + auto it = VMap.find(ULO.Heart); + assert(it != VMap.end()); + Instruction *heartCopy = cast(it->second); + heartCopy->eraseFromParent(); + VMap.erase(it); + } + } + // Update our running map of newest clones LastValueMap[*BB] = New; for (ValueToValueMapTy::iterator VI = VMap.begin(), VE = VMap.end(); diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp index e1af02829c1d..dd7150bc63ec 100644 --- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp +++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp @@ -1016,12 +1016,17 @@ bool llvm::UnrollRuntimeLoopRemainder( auto UnrollResult = LoopUnrollResult::Unmodified; if (remainderLoop && UnrollRemainder) { LLVM_DEBUG(dbgs() << "Unrolling remainder loop\n"); - UnrollResult = - UnrollLoop(remainderLoop, - {/*Count*/ Count - 1, /*Force*/ false, /*Runtime*/ false, - /*AllowExpensiveTripCount*/ false, - /*UnrollRemainder*/ false, ForgetAllSCEV}, - LI, SE, DT, AC, TTI, /*ORE*/ nullptr, PreserveLCSSA); + UnrollLoopOptions ULO; + ULO.Count = Count - 1; + ULO.Force = false; + ULO.Runtime = false; + ULO.AllowExpensiveTripCount = false; + ULO.UnrollRemainder = false; + ULO.ForgetAllSCEV = ForgetAllSCEV; + assert(!getLoopConvergenceHeart(L) && + "A loop with a convergence heart does not allow runtime unrolling."); + UnrollResult = UnrollLoop(remainderLoop, ULO, LI, SE, DT, AC, TTI, + /*ORE*/ nullptr, PreserveLCSSA); } if (ResultLoop && UnrollResult != LoopUnrollResult::FullyUnrolled) diff --git a/llvm/test/Transforms/LoopUnroll/convergent.controlled.ll b/llvm/test/Transforms/LoopUnroll/convergent.controlled.ll new file mode 100644 index 000000000000..7fd4eb18f16e --- /dev/null +++ b/llvm/test/Transforms/LoopUnroll/convergent.controlled.ll @@ -0,0 +1,562 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -passes=loop-unroll -unroll-runtime -unroll-allow-partial -S | FileCheck %s + +declare void @f() convergent +declare void @g() + +; Although this loop contains a convergent instruction, it should be +; fully unrolled. +define i32 @full_unroll() { +; CHECK-LABEL: @full_unroll( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: br label [[L3:%.*]] +; CHECK: l3: +; CHECK-NEXT: [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ] +; CHECK-NEXT: br label [[A:%.*]] +; CHECK: a: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: br label [[A_1:%.*]] +; CHECK: a.1: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: br label [[A_2:%.*]] +; CHECK: a.2: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: ret i32 0 +; +entry: + %anchor = call token @llvm.experimental.convergence.anchor() + br label %l3 + +l3: + %x.0 = phi i32 [ 0, %entry ], [ %inc, %a ] + %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ] + %inc = add nsw i32 %x.0, 1 + %exitcond = icmp eq i32 %inc, 3 + br label %a + +a: + call void @f() [ "convergencectrl"(token %tok.loop) ] + br i1 %exitcond, label %exit, label %l3 + +exit: + ret i32 0 +} + +; This loop contains a convergent instruction, but it should be partially +; unrolled. The unroll count is the largest power of 2 that divides the +; multiple -- 4, in this case. +define i32 @runtime_unroll(i32 %n) { +; CHECK-LABEL: @runtime_unroll( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: [[LOOP_CTL:%.*]] = mul nsw i32 [[N:%.*]], 12 +; CHECK-NEXT: br label [[L3:%.*]] +; CHECK: l3: +; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_3:%.*]], [[A_3:%.*]] ] +; CHECK-NEXT: [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ] +; CHECK-NEXT: br label [[A:%.*]] +; CHECK: a: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: br label [[A_1:%.*]] +; CHECK: a.1: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: br label [[A_2:%.*]] +; CHECK: a.2: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: br label [[A_3]] +; CHECK: a.3: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: [[INC_3]] = add nsw i32 [[X_0]], 4 +; CHECK-NEXT: [[EXITCOND_3:%.*]] = icmp eq i32 [[INC_3]], [[LOOP_CTL]] +; CHECK-NEXT: br i1 [[EXITCOND_3]], label [[EXIT:%.*]], label [[L3]] +; CHECK: exit: +; CHECK-NEXT: ret i32 0 +; +entry: + %anchor = call token @llvm.experimental.convergence.anchor() + %loop_ctl = mul nsw i32 %n, 12 + br label %l3 + +l3: + %x.0 = phi i32 [ 0, %entry ], [ %inc, %a ] + %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ] + br label %a + +a: + call void @f() [ "convergencectrl"(token %tok.loop) ] + %inc = add nsw i32 %x.0, 1 + %exitcond = icmp eq i32 %inc, %loop_ctl + br i1 %exitcond, label %exit, label %l3 + +exit: + ret i32 0 +} + +; This loop contains a convergent instruction, so its partial unroll +; count must divide its trip multiple. This overrides its unroll +; pragma -- we unroll exactly 8 times, even though 16 is requested. +define i32 @pragma_unroll(i32 %n) { +; CHECK-LABEL: @pragma_unroll( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: [[LOOP_CTL:%.*]] = mul nsw i32 [[N:%.*]], 24 +; CHECK-NEXT: br label [[L3:%.*]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: l3: +; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_7:%.*]], [[A_7:%.*]] ] +; CHECK-NEXT: [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ] +; CHECK-NEXT: br label [[A:%.*]] +; CHECK: a: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: br label [[A_1:%.*]] +; CHECK: a.1: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: br label [[A_2:%.*]] +; CHECK: a.2: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: br label [[A_3:%.*]] +; CHECK: a.3: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: br label [[A_4:%.*]] +; CHECK: a.4: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: br label [[A_5:%.*]] +; CHECK: a.5: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: br label [[A_6:%.*]] +; CHECK: a.6: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: br label [[A_7]] +; CHECK: a.7: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: [[INC_7]] = add nsw i32 [[X_0]], 8 +; CHECK-NEXT: [[EXITCOND_7:%.*]] = icmp eq i32 [[INC_7]], [[LOOP_CTL]] +; CHECK-NEXT: br i1 [[EXITCOND_7]], label [[EXIT:%.*]], label [[L3]], !llvm.loop [[LOOP2:![0-9]+]] +; CHECK: exit: +; CHECK-NEXT: ret i32 0 +; +entry: + %anchor = call token @llvm.experimental.convergence.anchor() + %loop_ctl = mul nsw i32 %n, 24 + br label %l3, !llvm.loop !0 + +l3: + %x.0 = phi i32 [ 0, %entry ], [ %inc, %a ] + %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ] + br label %a + +a: + call void @f() [ "convergencectrl"(token %tok.loop) ] + %inc = add nsw i32 %x.0, 1 + %exitcond = icmp eq i32 %inc, %loop_ctl + br i1 %exitcond, label %exit, label %l3, !llvm.loop !0 + +exit: + ret i32 0 +} + +; This loop contains a convergent instruction. Since the pragma loop unroll +; count 2 divides trip count 4. The loop unroll should respect the pragma. +define void @pragma_unroll_divisible_trip_count() { +; CHECK-LABEL: @pragma_unroll_divisible_trip_count( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: br label [[L3:%.*]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK: l3: +; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_1:%.*]], [[L3]] ] +; CHECK-NEXT: [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ] +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: [[INC_1]] = add nuw nsw i32 [[X_0]], 2 +; CHECK-NEXT: [[EXITCOND_1:%.*]] = icmp eq i32 [[INC_1]], 4 +; CHECK-NEXT: br i1 [[EXITCOND_1]], label [[EXIT:%.*]], label [[L3]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + %anchor = call token @llvm.experimental.convergence.anchor() + br label %l3, !llvm.loop !1 + +l3: + %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ] + %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ] + call void @f() [ "convergencectrl"(token %tok.loop) ] + %inc = add nsw i32 %x.0, 1 + %exitcond = icmp eq i32 %inc, 4 + br i1 %exitcond, label %exit, label %l3, !llvm.loop !1 + +exit: + ret void +} + +; This loop contains a convergent instruction. Since the pragma loop unroll +; count 2 divides trip multiple 2. The loop unroll should respect the pragma. +define i32 @pragma_unroll_divisible_trip_multiple(i32 %n) { +; CHECK-LABEL: @pragma_unroll_divisible_trip_multiple( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: [[LOOP_CTL:%.*]] = mul nsw i32 [[N:%.*]], 2 +; CHECK-NEXT: br label [[L3:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l3: +; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_1:%.*]], [[L3]] ] +; CHECK-NEXT: [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ] +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: [[INC_1]] = add nsw i32 [[X_0]], 2 +; CHECK-NEXT: [[EXITCOND_1:%.*]] = icmp eq i32 [[INC_1]], [[LOOP_CTL]] +; CHECK-NEXT: br i1 [[EXITCOND_1]], label [[EXIT:%.*]], label [[L3]], !llvm.loop [[LOOP7:![0-9]+]] +; CHECK: exit: +; CHECK-NEXT: ret i32 0 +; +entry: + %anchor = call token @llvm.experimental.convergence.anchor() + %loop_ctl = mul nsw i32 %n, 2 + br label %l3, !llvm.loop !1 + +l3: + %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ] + %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ] + call void @f() [ "convergencectrl"(token %tok.loop) ] + %inc = add nsw i32 %x.0, 1 + %exitcond = icmp eq i32 %inc, %loop_ctl + br i1 %exitcond, label %exit, label %l3, !llvm.loop !1 + +exit: + ret i32 0 +} + +; This loop contains a convergent instruction. Since the pragma loop unroll +; count 2 is unknown to divide runtime trip count, the loop is not unrolled +; since remainder is forbidden for unrolling convergent loop. +define i32 @pragma_unroll_indivisible_runtime_trip_count(i32 %n) { +; CHECK-LABEL: @pragma_unroll_indivisible_runtime_trip_count( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: br label [[L3:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l3: +; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[L3]] ] +; CHECK-NEXT: [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ] +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: [[INC]] = add nsw i32 [[X_0]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N:%.*]] +; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[L3]], !llvm.loop [[LOOP4]] +; CHECK: exit: +; CHECK-NEXT: ret i32 0 +; +entry: + %anchor = call token @llvm.experimental.convergence.anchor() + br label %l3, !llvm.loop !1 + +l3: + %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ] + %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ] + call void @f() [ "convergencectrl"(token %tok.loop) ] + %inc = add nsw i32 %x.0, 1 + %exitcond = icmp eq i32 %inc, %n + br i1 %exitcond, label %exit, label %l3, !llvm.loop !1 + +exit: + ret i32 0 +} + +; This loop contains a convergent instruction. Since the pragma loop unroll +; count 2 does not divide trip count 5, the loop is not unrolled by 2 +; since remainder is forbidden for unrolling convergent loop. Instead, the +; loop gets fully unrolled. +define i32 @pragma_unroll_indivisible_trip_count() { +; CHECK-LABEL: @pragma_unroll_indivisible_trip_count( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ANCHOR:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: br label [[L3:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l3: +; CHECK-NEXT: [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[ANCHOR]]) ] +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: ret i32 0 +; +entry: + %anchor = call token @llvm.experimental.convergence.anchor() + br label %l3, !llvm.loop !1 + +l3: + %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ] + %tok.loop = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %anchor) ] + call void @f() [ "convergencectrl"(token %tok.loop) ] + %inc = add nsw i32 %x.0, 1 + %exitcond = icmp eq i32 %inc, 5 + br i1 %exitcond, label %exit, label %l3, !llvm.loop !1 + +exit: + ret i32 0 +} + +; This loop contains a convergent instruction that is anchored inside the loop +; itself. It is unrolled by 2 with remainder, as requested by the loop metadata. +define i32 @pragma_unroll_with_remainder(i32 %n) { +; CHECK-LABEL: @pragma_unroll_with_remainder( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = freeze i32 [[N:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], -1 +; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[TMP0]], 1 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i32 [[TMP1]], 1 +; CHECK-NEXT: br i1 [[TMP2]], label [[EXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; CHECK: entry.new: +; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[TMP0]], [[XTRAITER]] +; CHECK-NEXT: br label [[L3:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l3: +; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 0, [[ENTRY_NEW]] ], [ [[INC_1:%.*]], [[L3]] ] +; CHECK-NEXT: [[NITER:%.*]] = phi i32 [ 0, [[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], [[L3]] ] +; CHECK-NEXT: [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: [[TOK_LOOP_1:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP_1]]) ] +; CHECK-NEXT: [[INC_1]] = add nsw i32 [[X_0]], 2 +; CHECK-NEXT: [[NITER_NEXT_1]] = add i32 [[NITER]], 2 +; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i32 [[NITER_NEXT_1]], [[UNROLL_ITER]] +; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[L3]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK: exit.unr-lcssa.loopexit: +; CHECK-NEXT: br label [[EXIT_UNR_LCSSA]] +; CHECK: exit.unr-lcssa: +; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[L3_EPIL_PREHEADER:%.*]], label [[EXIT:%.*]] +; CHECK: l3.epil.preheader: +; CHECK-NEXT: br label [[L3_EPIL:%.*]] +; CHECK: l3.epil: +; CHECK-NEXT: [[TOK_LOOP_EPIL:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP_EPIL]]) ] +; CHECK-NEXT: br label [[EXIT]] +; CHECK: exit: +; CHECK-NEXT: ret i32 0 +; +entry: + br label %l3, !llvm.loop !1 + +l3: + %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ] + %tok.loop = call token @llvm.experimental.convergence.anchor() + call void @f() [ "convergencectrl"(token %tok.loop) ] + %inc = add nsw i32 %x.0, 1 + %exitcond = icmp eq i32 %inc, %n + br i1 %exitcond, label %exit, label %l3, !llvm.loop !1 + +exit: + ret i32 0 +} + +; Don't unroll a loop that is extended by convergence controls. +; +; We could theoretically duplicate the extension part, but this is not +; implemented. +define i32 @extended_loop(i32 %n) { +; CHECK-LABEL: @extended_loop( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[L3:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l3: +; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[L3]] ] +; CHECK-NEXT: [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: [[INC]] = add nsw i32 [[X_0]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N:%.*]] +; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[L3]], !llvm.loop [[LOOP4]] +; CHECK: exit: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_LOOP]]) ] +; CHECK-NEXT: ret i32 0 +; +entry: + br label %l3, !llvm.loop !1 + +l3: + %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ] + %tok.loop = call token @llvm.experimental.convergence.anchor() + %inc = add nsw i32 %x.0, 1 + %exitcond = icmp eq i32 %inc, %n + br i1 %exitcond, label %exit, label %l3, !llvm.loop !1 + +exit: + call void @f() [ "convergencectrl"(token %tok.loop) ] + ret i32 0 +} + +; Inner loop is extended beyond the outer loop. No unrolling possible. + +define i32 @extended_inner_loop_1(i32 %n, i1 %cond) { +; CHECK-LABEL: @extended_inner_loop_1( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[L3:%.*]] +; CHECK: l3: +; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[LATCH:%.*]] ] +; CHECK-NEXT: [[TOK_LOOP:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: [[INC]] = add nsw i32 [[X_0]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 4 +; CHECK-NEXT: br label [[L2:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l2: +; CHECK-NEXT: [[TOK_L2:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2]]) ] +; CHECK-NEXT: br i1 [[COND:%.*]], label [[L2]], label [[LATCH]], !llvm.loop [[LOOP4]] +; CHECK: latch: +; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[L3]] +; CHECK: exit: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2]]) ] +; CHECK-NEXT: ret i32 0 +; +entry: + br label %l3 + +l3: + %x.0 = phi i32 [ 0, %entry ], [ %inc, %latch ] + %tok.loop = call token @llvm.experimental.convergence.anchor() + %inc = add nsw i32 %x.0, 1 + %exitcond = icmp eq i32 %inc, 4 + br label %l2, !llvm.loop !1 + +l2: + %tok.l2 = call token @llvm.experimental.convergence.anchor() + call void @f() [ "convergencectrl"(token %tok.l2) ] + br i1 %cond, label %l2, label %latch, !llvm.loop !1 + +latch: + br i1 %exitcond, label %exit, label %l3 + +exit: + call void @f() [ "convergencectrl"(token %tok.l2) ] + ret i32 0 +} + +; Inner loop is extended inside the outer loop. Outer loop is unrolled. + +define i32 @extended_inner_loop_2(i32 %n, i1 %cond) { +; CHECK-LABEL: @extended_inner_loop_2( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[L3:%.*]] +; CHECK: l3: +; CHECK-NEXT: br label [[L2:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l2: +; CHECK-NEXT: [[TOK_L2:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2]]) ] +; CHECK-NEXT: br i1 [[COND:%.*]], label [[L2]], label [[LATCH:%.*]], !llvm.loop [[LOOP4]] +; CHECK: latch: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2]]) ] +; CHECK-NEXT: br label [[L2_1:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l2.1: +; CHECK-NEXT: [[TOK_L2_1:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2_1]]) ] +; CHECK-NEXT: br i1 [[COND]], label [[L2_1]], label [[LATCH_1:%.*]], !llvm.loop [[LOOP4]] +; CHECK: latch.1: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2_1]]) ] +; CHECK-NEXT: br label [[L2_2:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l2.2: +; CHECK-NEXT: [[TOK_L2_2:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2_2]]) ] +; CHECK-NEXT: br i1 [[COND]], label [[L2_2]], label [[LATCH_2:%.*]], !llvm.loop [[LOOP4]] +; CHECK: latch.2: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2_2]]) ] +; CHECK-NEXT: br label [[L2_3:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l2.3: +; CHECK-NEXT: [[TOK_L2_3:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2_3]]) ] +; CHECK-NEXT: br i1 [[COND]], label [[L2_3]], label [[LATCH_3:%.*]], !llvm.loop [[LOOP4]] +; CHECK: latch.3: +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2_3]]) ] +; CHECK-NEXT: ret i32 0 +; +entry: + br label %l3 + +l3: + %x.0 = phi i32 [ 0, %entry ], [ %inc, %latch ] + %tok.loop = call token @llvm.experimental.convergence.anchor() + %inc = add nsw i32 %x.0, 1 + %exitcond = icmp eq i32 %inc, 4 + br label %l2, !llvm.loop !1 + +l2: + %tok.l2 = call token @llvm.experimental.convergence.anchor() + call void @f() [ "convergencectrl"(token %tok.l2) ] + br i1 %cond, label %l2, label %latch, !llvm.loop !1 + +latch: + call void @f() [ "convergencectrl"(token %tok.l2) ] + br i1 %exitcond, label %exit, label %l3 + +exit: + ret i32 0 +} + +; No extension. Both loops unrolled. + +define i32 @unroll_nest(i32 %n, i1 %cond) { +; CHECK-LABEL: @unroll_nest( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[L3:%.*]] +; CHECK: l3: +; CHECK-NEXT: br label [[L2:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l2: +; CHECK-NEXT: [[TOK_L2:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2]]) ] +; CHECK-NEXT: br i1 [[COND:%.*]], label [[L2_1:%.*]], label [[LATCH:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l2.1: +; CHECK-NEXT: [[TOK_L2_1:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2_1]]) ] +; CHECK-NEXT: br i1 [[COND]], label [[L2]], label [[LATCH]], !llvm.loop [[LOOP9:![0-9]+]] +; CHECK: latch: +; CHECK-NEXT: br label [[L2_12:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l2.12: +; CHECK-NEXT: [[TOK_L2_11:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2_11]]) ] +; CHECK-NEXT: br i1 [[COND]], label [[L2_1_1:%.*]], label [[LATCH_1:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l2.1.1: +; CHECK-NEXT: [[TOK_L2_1_1:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2_1_1]]) ] +; CHECK-NEXT: br i1 [[COND]], label [[L2_12]], label [[LATCH_1]], !llvm.loop [[LOOP9]] +; CHECK: latch.1: +; CHECK-NEXT: br label [[L2_2:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l2.2: +; CHECK-NEXT: [[TOK_L2_2:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2_2]]) ] +; CHECK-NEXT: br i1 [[COND]], label [[L2_1_2:%.*]], label [[LATCH_2:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l2.1.2: +; CHECK-NEXT: [[TOK_L2_1_2:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2_1_2]]) ] +; CHECK-NEXT: br i1 [[COND]], label [[L2_2]], label [[LATCH_2]], !llvm.loop [[LOOP9]] +; CHECK: latch.2: +; CHECK-NEXT: br label [[L2_3:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l2.3: +; CHECK-NEXT: [[TOK_L2_3:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2_3]]) ] +; CHECK-NEXT: br i1 [[COND]], label [[L2_1_3:%.*]], label [[LATCH_3:%.*]], !llvm.loop [[LOOP4]] +; CHECK: l2.1.3: +; CHECK-NEXT: [[TOK_L2_1_3:%.*]] = call token @llvm.experimental.convergence.anchor() +; CHECK-NEXT: call void @f() [ "convergencectrl"(token [[TOK_L2_1_3]]) ] +; CHECK-NEXT: br i1 [[COND]], label [[L2_3]], label [[LATCH_3]], !llvm.loop [[LOOP9]] +; CHECK: latch.3: +; CHECK-NEXT: ret i32 0 +; +entry: + br label %l3 + +l3: + %x.0 = phi i32 [ 0, %entry ], [ %inc, %latch ] + %tok.loop = call token @llvm.experimental.convergence.anchor() + %inc = add nsw i32 %x.0, 1 + %exitcond = icmp eq i32 %inc, 4 + br label %l2, !llvm.loop !1 + +l2: + %tok.l2 = call token @llvm.experimental.convergence.anchor() + call void @f() [ "convergencectrl"(token %tok.l2) ] + br i1 %cond, label %l2, label %latch, !llvm.loop !1 + +latch: + br i1 %exitcond, label %exit, label %l3 + +exit: + ret i32 0 +} + +declare token @llvm.experimental.convergence.anchor() +declare token @llvm.experimental.convergence.loop() + +!0 = !{!0, !{!"llvm.loop.unroll.count", i32 16}} +!1 = !{!1, !{!"llvm.loop.unroll.count", i32 2}} -- GitLab From afc7292626e193a2b09b81d227eac90428f0abe3 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 6 Jun 2024 09:59:18 +0200 Subject: [PATCH 049/462] [SDPatternMatch] Do not use std::forward and rvalue references (NFC) (#93806) The m_ZExtOrSelf() family of matchers currently incorrectly calls std::forward twice on the same value. However, just removing those causes other complications, because then template arguments get incorrectly inferred to const references instead of the underlying value types. Things become a mess. Instead, just completely remove the use of std::forward and rvalue references from SDPatternMatch. I don't think they really provide value in this context, especially as they're not used consistently in the first place. --- llvm/include/llvm/CodeGen/SDPatternMatch.h | 55 +++++++++------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h index 7970441c8371..071a27a79506 100644 --- a/llvm/include/llvm/CodeGen/SDPatternMatch.h +++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h @@ -330,9 +330,7 @@ template struct And { template struct And : And { Pred P; - And(Pred &&p, Preds &&...preds) - : And(std::forward(preds)...), P(std::forward(p)) { - } + And(const Pred &p, const Preds &...preds) : And(preds...), P(p) {} template bool match(const MatchContext &Ctx, SDValue N) { @@ -349,8 +347,7 @@ template struct Or { template struct Or : Or { Pred P; - Or(Pred &&p, Preds &&...preds) - : Or(std::forward(preds)...), P(std::forward(p)) {} + Or(const Pred &p, const Preds &...preds) : Or(preds...), P(p) {} template bool match(const MatchContext &Ctx, SDValue N) { @@ -376,16 +373,16 @@ template inline Not m_Unless(const Pred &P) { return Not{P}; } -template And m_AllOf(Preds &&...preds) { - return And(std::forward(preds)...); +template And m_AllOf(const Preds &...preds) { + return And(preds...); } -template Or m_AnyOf(Preds &&...preds) { - return Or(std::forward(preds)...); +template Or m_AnyOf(const Preds &...preds) { + return Or(preds...); } -template auto m_NoneOf(Preds &&...preds) { - return m_Unless(m_AnyOf(std::forward(preds)...)); +template auto m_NoneOf(const Preds &...preds) { + return m_Unless(m_AnyOf(preds...)); } // === Generic node matching === @@ -402,10 +399,8 @@ struct Operands_match : Operands_match { OpndPred P; - Operands_match(OpndPred &&p, OpndPreds &&...preds) - : Operands_match( - std::forward(preds)...), - P(std::forward(p)) {} + Operands_match(const OpndPred &p, const OpndPreds &...preds) + : Operands_match(preds...), P(p) {} template bool match(const MatchContext &Ctx, SDValue N) { @@ -419,9 +414,8 @@ struct Operands_match }; template -auto m_Node(unsigned Opcode, OpndPreds &&...preds) { - return m_AllOf(m_Opc(Opcode), Operands_match<0, OpndPreds...>( - std::forward(preds)...)); +auto m_Node(unsigned Opcode, const OpndPreds &...preds) { + return m_AllOf(m_Opc(Opcode), Operands_match<0, OpndPreds...>(preds...)); } /// Provide number of operands that are not chain or glue, as well as the first @@ -647,10 +641,9 @@ template inline UnaryOpc_match m_ZExt(const Opnd &Op) { return UnaryOpc_match(ISD::ZERO_EXTEND, Op); } -template inline auto m_SExt(Opnd &&Op) { - return m_AnyOf( - UnaryOpc_match(ISD::SIGN_EXTEND, Op), - m_Node(ISD::SIGN_EXTEND_INREG, std::forward(Op), m_Value())); +template inline auto m_SExt(const Opnd &Op) { + return m_AnyOf(UnaryOpc_match(ISD::SIGN_EXTEND, Op), + m_Node(ISD::SIGN_EXTEND_INREG, Op, m_Value())); } template inline UnaryOpc_match m_AnyExt(const Opnd &Op) { @@ -663,30 +656,28 @@ template inline UnaryOpc_match m_Trunc(const Opnd &Op) { /// Match a zext or identity /// Allows to peek through optional extensions -template inline auto m_ZExtOrSelf(Opnd &&Op) { - return m_AnyOf(m_ZExt(std::forward(Op)), std::forward(Op)); +template inline auto m_ZExtOrSelf(const Opnd &Op) { + return m_AnyOf(m_ZExt(Op), Op); } /// Match a sext or identity /// Allows to peek through optional extensions -template inline auto m_SExtOrSelf(Opnd &&Op) { - return m_AnyOf(m_SExt(std::forward(Op)), std::forward(Op)); +template inline auto m_SExtOrSelf(const Opnd &Op) { + return m_AnyOf(m_SExt(Op), Op); } /// Match a aext or identity /// Allows to peek through optional extensions template -inline Or, Opnd> m_AExtOrSelf(Opnd &&Op) { - return Or, Opnd>(m_AnyExt(std::forward(Op)), - std::forward(Op)); +inline Or, Opnd> m_AExtOrSelf(const Opnd &Op) { + return Or, Opnd>(m_AnyExt(Op), Op); } /// Match a trunc or identity /// Allows to peek through optional truncations template -inline Or, Opnd> m_TruncOrSelf(Opnd &&Op) { - return Or, Opnd>(m_Trunc(std::forward(Op)), - std::forward(Op)); +inline Or, Opnd> m_TruncOrSelf(const Opnd &Op) { + return Or, Opnd>(m_Trunc(Op), Op); } // === Constants === -- GitLab From c3c443bb41ab5eab6d954bcc051549104ff6c235 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Tue, 4 Jun 2024 12:14:46 -0500 Subject: [PATCH 050/462] [InstCombine] Add transforms `(icmp spred (and X, Y), X)` if `X` or `Y` are known signed/unsigned Several transforms: 1) If known `Y < 0`: - slt -> ult: https://alive2.llvm.org/ce/z/9zt2iK - sle -> ule: https://alive2.llvm.org/ce/z/SPoPNF - sgt -> ugt: https://alive2.llvm.org/ce/z/IGNxAk - sge -> uge: https://alive2.llvm.org/ce/z/joqTvR 2) If known `Y >= 0`: - `(X & PosY) s> X --> X s< 0` - https://alive2.llvm.org/ce/z/7e-5BQ - `(X & PosY) s> X --> X s< 0` - https://alive2.llvm.org/ce/z/jvT4Gb 3) If known `X < 0`: - `(NegX & Y) s> NegX --> Y s>= 0` - https://alive2.llvm.org/ce/z/ApkaEh - `(NegX & Y) s<= NegX --> Y s< 0` - https://alive2.llvm.org/ce/z/oRnfHp Closes #94417 --- .../InstCombine/InstCombineCompares.cpp | 23 ++++++ ...t-low-bit-mask-and-icmp-sge-to-icmp-sle.ll | 3 +- ...t-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll | 3 +- ...t-low-bit-mask-and-icmp-sle-to-icmp-sle.ll | 3 +- ...t-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll | 3 +- .../InstCombine/icmp-and-lowbit-mask.ll | 72 +++++++++---------- .../Transforms/InstCombine/icmp-of-and-x.ll | 38 ++++------ 7 files changed, 74 insertions(+), 71 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp index 89193f8ff94b..803475bfaba5 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -4745,6 +4745,29 @@ static Instruction *foldICmpAndXX(ICmpInst &I, const SimplifyQuery &Q, Constant::getNullValue(Op1->getType())); } + if (!ICmpInst::isSigned(Pred)) + return nullptr; + + KnownBits KnownY = IC.computeKnownBits(A, /*Depth=*/0, &I); + // (X & NegY) spred X --> (X & NegY) upred X + if (KnownY.isNegative()) + return new ICmpInst(ICmpInst::getUnsignedPredicate(Pred), Op0, Op1); + + if (Pred != ICmpInst::ICMP_SLE && Pred != ICmpInst::ICMP_SGT) + return nullptr; + + if (KnownY.isNonNegative()) + // (X & PosY) s<= X --> X s>= 0 + // (X & PosY) s> X --> X s< 0 + return new ICmpInst(ICmpInst::getSwappedPredicate(Pred), Op1, + Constant::getNullValue(Op1->getType())); + + if (isKnownNegative(Op1, IC.getSimplifyQuery().getWithInstruction(&I))) + // (NegX & Y) s<= NegX --> Y s< 0 + // (NegX & Y) s> NegX --> Y s>= 0 + return new ICmpInst(ICmpInst::getFlippedStrictnessPredicate(Pred), A, + Constant::getNullValue(A->getType())); + return nullptr; } diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll index ae503bfb1cfe..e103fe944098 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll @@ -98,8 +98,7 @@ declare i8 @gen8() define i1 @c0() { ; CHECK-LABEL: @c0( ; CHECK-NEXT: [[X:%.*]] = call i8 @gen8() -; CHECK-NEXT: [[TMP0:%.*]] = and i8 [[X]], 3 -; CHECK-NEXT: [[RET:%.*]] = icmp sge i8 [[X]], [[TMP0]] +; CHECK-NEXT: [[RET:%.*]] = icmp sgt i8 [[X]], -1 ; CHECK-NEXT: ret i1 [[RET]] ; %x = call i8 @gen8() diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll index d1dd411ee86b..bbd733e86a32 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll @@ -125,8 +125,7 @@ define i1 @oneuse0() { define i1 @c0(i8 %x) { ; CHECK-LABEL: @c0( -; CHECK-NEXT: [[TMP0:%.*]] = and i8 [[X:%.*]], 3 -; CHECK-NEXT: [[RET:%.*]] = icmp sgt i8 [[TMP0]], [[X]] +; CHECK-NEXT: [[RET:%.*]] = icmp slt i8 [[X:%.*]], 0 ; CHECK-NEXT: ret i1 [[RET]] ; %tmp0 = and i8 %x, 3 diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll index 4bed21a525f0..b167c8ad25aa 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll @@ -113,8 +113,7 @@ define i1 @oneuse0() { define i1 @c0(i8 %x) { ; CHECK-LABEL: @c0( -; CHECK-NEXT: [[TMP0:%.*]] = and i8 [[X:%.*]], 3 -; CHECK-NEXT: [[RET:%.*]] = icmp sle i8 [[TMP0]], [[X]] +; CHECK-NEXT: [[RET:%.*]] = icmp sgt i8 [[X:%.*]], -1 ; CHECK-NEXT: ret i1 [[RET]] ; %tmp0 = and i8 %x, 3 diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll index 8415204a4915..828150244773 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll @@ -108,8 +108,7 @@ declare i8 @gen8() define i1 @c0() { ; CHECK-LABEL: @c0( ; CHECK-NEXT: [[X:%.*]] = call i8 @gen8() -; CHECK-NEXT: [[TMP0:%.*]] = and i8 [[X]], 3 -; CHECK-NEXT: [[RET:%.*]] = icmp slt i8 [[X]], [[TMP0]] +; CHECK-NEXT: [[RET:%.*]] = icmp slt i8 [[X]], 0 ; CHECK-NEXT: ret i1 [[RET]] ; %x = call i8 @gen8() diff --git a/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll b/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll index 8bb7fd0e522c..0aace5f52c96 100644 --- a/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll +++ b/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll @@ -7,8 +7,8 @@ define i1 @src_is_mask_zext(i16 %x_in, i8 %y) { ; CHECK-LABEL: @src_is_mask_zext( ; CHECK-NEXT: [[M_IN:%.*]] = lshr i8 -1, [[Y:%.*]] ; CHECK-NEXT: [[MASK:%.*]] = zext i8 [[M_IN]] to i16 -; CHECK-NEXT: [[X:%.*]] = xor i16 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[R:%.*]] = icmp ule i16 [[X]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i16 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[R:%.*]] = icmp ule i16 [[TMP1]], [[MASK]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i16 %x_in, 123 @@ -83,8 +83,8 @@ define i1 @src_is_mask_and(i8 %x_in, i8 %y, i8 %z) { ; CHECK-NEXT: [[MY:%.*]] = lshr i8 7, [[Y:%.*]] ; CHECK-NEXT: [[MZ:%.*]] = lshr i8 -1, [[Z:%.*]] ; CHECK-NEXT: [[MASK:%.*]] = and i8 [[MY]], [[MZ]] -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[X]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[TMP1]], [[MASK]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 @@ -121,8 +121,8 @@ define i1 @src_is_mask_or(i8 %x_in, i8 %y) { ; CHECK-LABEL: @src_is_mask_or( ; CHECK-NEXT: [[MY:%.*]] = lshr i8 -1, [[Y:%.*]] ; CHECK-NEXT: [[MASK:%.*]] = and i8 [[MY]], 7 -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[X]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[TMP1]], [[MASK]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 @@ -138,8 +138,8 @@ define i1 @src_is_mask_xor(i8 %x_in, i8 %y) { ; CHECK-LABEL: @src_is_mask_xor( ; CHECK-NEXT: [[Y_M1:%.*]] = add i8 [[Y:%.*]], -1 ; CHECK-NEXT: [[MASK:%.*]] = xor i8 [[Y_M1]], [[Y]] -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 @@ -173,8 +173,8 @@ define i1 @src_is_mask_select(i8 %x_in, i8 %y, i1 %cond) { ; CHECK-NEXT: [[Y_M1:%.*]] = add i8 [[Y:%.*]], -1 ; CHECK-NEXT: [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]] ; CHECK-NEXT: [[MASK:%.*]] = select i1 [[COND:%.*]], i8 [[YMASK]], i8 15 -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 @@ -249,8 +249,8 @@ define i1 @src_is_mask_lshr(i8 %x_in, i8 %y, i8 %z, i1 %cond) { ; CHECK-NEXT: [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]] ; CHECK-NEXT: [[SMASK:%.*]] = select i1 [[COND:%.*]], i8 [[YMASK]], i8 15 ; CHECK-NEXT: [[MASK:%.*]] = lshr i8 [[SMASK]], [[Z:%.*]] -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 @@ -269,8 +269,8 @@ define i1 @src_is_mask_ashr(i8 %x_in, i8 %y, i8 %z, i1 %cond) { ; CHECK-NEXT: [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]] ; CHECK-NEXT: [[SMASK:%.*]] = select i1 [[COND:%.*]], i8 [[YMASK]], i8 15 ; CHECK-NEXT: [[MASK:%.*]] = ashr i8 [[SMASK]], [[Z:%.*]] -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 @@ -287,8 +287,8 @@ define i1 @src_is_mask_p2_m1(i8 %x_in, i8 %y) { ; CHECK-LABEL: @src_is_mask_p2_m1( ; CHECK-NEXT: [[P2ORZ:%.*]] = shl i8 2, [[Y:%.*]] ; CHECK-NEXT: [[MASK:%.*]] = add i8 [[P2ORZ]], -1 -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 @@ -304,8 +304,8 @@ define i1 @src_is_mask_umax(i8 %x_in, i8 %y) { ; CHECK-NEXT: [[Y_M1:%.*]] = add i8 [[Y:%.*]], -1 ; CHECK-NEXT: [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]] ; CHECK-NEXT: [[MASK:%.*]] = call i8 @llvm.umax.i8(i8 [[YMASK]], i8 3) -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 @@ -324,8 +324,8 @@ define i1 @src_is_mask_umin(i8 %x_in, i8 %y, i8 %z) { ; CHECK-NEXT: [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]] ; CHECK-NEXT: [[ZMASK:%.*]] = lshr i8 15, [[Z:%.*]] ; CHECK-NEXT: [[MASK:%.*]] = call i8 @llvm.umin.i8(i8 [[YMASK]], i8 [[ZMASK]]) -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[X]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[TMP1]], [[MASK]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 @@ -364,8 +364,8 @@ define i1 @src_is_mask_smax(i8 %x_in, i8 %y) { ; CHECK-NEXT: [[Y_M1:%.*]] = add i8 [[Y:%.*]], -1 ; CHECK-NEXT: [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]] ; CHECK-NEXT: [[MASK:%.*]] = call i8 @llvm.smax.i8(i8 [[YMASK]], i8 -1) -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[X]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[TMP1]], [[MASK]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 @@ -383,8 +383,8 @@ define i1 @src_is_mask_smin(i8 %x_in, i8 %y) { ; CHECK-NEXT: [[Y_M1:%.*]] = add i8 [[Y:%.*]], -1 ; CHECK-NEXT: [[YMASK:%.*]] = xor i8 [[Y_M1]], [[Y]] ; CHECK-NEXT: [[MASK:%.*]] = call i8 @llvm.smin.i8(i8 [[YMASK]], i8 0) -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[X]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[TMP1]], [[MASK]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 @@ -401,8 +401,8 @@ define i1 @src_is_mask_bitreverse_not_mask(i8 %x_in, i8 %y) { ; CHECK-LABEL: @src_is_mask_bitreverse_not_mask( ; CHECK-NEXT: [[NMASK:%.*]] = shl nsw i8 -1, [[Y:%.*]] ; CHECK-NEXT: [[MASK:%.*]] = call i8 @llvm.bitreverse.i8(i8 [[NMASK]]) -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[X]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[TMP1]], [[MASK]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 @@ -455,9 +455,9 @@ define i1 @src_is_notmask_shl(i8 %x_in, i8 %y, i1 %cond) { define i1 @src_is_notmask_x_xor_neg_x(i8 %x_in, i8 %y, i1 %cond) { ; CHECK-LABEL: @src_is_notmask_x_xor_neg_x( ; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[NEG_Y:%.*]] = add i8 [[Y:%.*]], -1 -; CHECK-NEXT: [[NOTMASK0:%.*]] = xor i8 [[NEG_Y]], [[Y]] -; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[COND:%.*]], i8 [[NOTMASK0]], i8 7 +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[Y:%.*]], -1 +; CHECK-NEXT: [[TMP2:%.*]] = xor i8 [[TMP1]], [[Y]] +; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[COND:%.*]], i8 [[TMP2]], i8 7 ; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[X]], [[TMP3]] ; CHECK-NEXT: ret i1 [[R]] ; @@ -473,9 +473,9 @@ define i1 @src_is_notmask_x_xor_neg_x(i8 %x_in, i8 %y, i1 %cond) { define i1 @src_is_notmask_x_xor_neg_x_inv(i8 %x_in, i8 %y, i1 %cond) { ; CHECK-LABEL: @src_is_notmask_x_xor_neg_x_inv( ; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[NEG_Y:%.*]] = add i8 [[Y:%.*]], -1 -; CHECK-NEXT: [[NOTMASK0:%.*]] = xor i8 [[NEG_Y]], [[Y]] -; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[COND:%.*]], i8 [[NOTMASK0]], i8 7 +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[Y:%.*]], -1 +; CHECK-NEXT: [[TMP2:%.*]] = xor i8 [[TMP1]], [[Y]] +; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[COND:%.*]], i8 [[TMP2]], i8 7 ; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[X]], [[TMP3]] ; CHECK-NEXT: ret i1 [[R]] ; @@ -625,9 +625,7 @@ define i1 @src_is_notmask_xor_fail(i8 %x_in, i8 %y) { define i1 @src_is_mask_const_slt(i8 %x_in) { ; CHECK-LABEL: @src_is_mask_const_slt( -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[AND:%.*]] = and i8 [[X]], 7 -; CHECK-NEXT: [[R:%.*]] = icmp slt i8 [[X]], [[AND]] +; CHECK-NEXT: [[R:%.*]] = icmp slt i8 [[X_IN:%.*]], 0 ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 @@ -650,9 +648,7 @@ define i1 @src_is_mask_const_sgt(i8 %x_in) { define i1 @src_is_mask_const_sle(i8 %x_in) { ; CHECK-LABEL: @src_is_mask_const_sle( -; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[AND:%.*]] = and i8 [[X]], 31 -; CHECK-NEXT: [[R:%.*]] = icmp sle i8 [[AND]], [[X]] +; CHECK-NEXT: [[R:%.*]] = icmp sgt i8 [[X_IN:%.*]], -1 ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 diff --git a/llvm/test/Transforms/InstCombine/icmp-of-and-x.ll b/llvm/test/Transforms/InstCombine/icmp-of-and-x.ll index 0f26be12c39c..75badabda01a 100644 --- a/llvm/test/Transforms/InstCombine/icmp-of-and-x.ll +++ b/llvm/test/Transforms/InstCombine/icmp-of-and-x.ll @@ -58,7 +58,7 @@ define i1 @icmp_sge_x_negy(i8 %x, i8 %y) { ; CHECK-NEXT: [[CY:%.*]] = icmp slt i8 [[Y:%.*]], 0 ; CHECK-NEXT: call void @llvm.assume(i1 [[CY]]) ; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], [[Y]] -; CHECK-NEXT: [[Z:%.*]] = icmp sge i8 [[AND]], [[X]] +; CHECK-NEXT: [[Z:%.*]] = icmp eq i8 [[AND]], [[X]] ; CHECK-NEXT: ret i1 [[Z]] ; %cy = icmp slt i8 %y, 0 @@ -74,7 +74,7 @@ define i1 @icmp_slt_x_negy(i8 %x, i8 %y) { ; CHECK-NEXT: br i1 [[CY]], label [[NEGY:%.*]], label [[POSY:%.*]] ; CHECK: negy: ; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], [[Y]] -; CHECK-NEXT: [[Z:%.*]] = icmp slt i8 [[AND]], [[X]] +; CHECK-NEXT: [[Z:%.*]] = icmp ne i8 [[AND]], [[X]] ; CHECK-NEXT: ret i1 [[Z]] ; CHECK: posy: ; CHECK-NEXT: [[R:%.*]] = call i1 @barrier() @@ -116,10 +116,7 @@ posy: define i1 @icmp_sle_x_negy(i8 %x, i8 %yy) { ; CHECK-LABEL: @icmp_sle_x_negy( -; CHECK-NEXT: [[Y:%.*]] = or i8 [[YY:%.*]], -128 -; CHECK-NEXT: [[AND:%.*]] = and i8 [[Y]], [[X:%.*]] -; CHECK-NEXT: [[Z:%.*]] = icmp sle i8 [[AND]], [[X]] -; CHECK-NEXT: ret i1 [[Z]] +; CHECK-NEXT: ret i1 true ; %y = or i8 %yy, 128 %and = and i8 %y, %x @@ -129,10 +126,7 @@ define i1 @icmp_sle_x_negy(i8 %x, i8 %yy) { define <2 x i1> @icmp_sgt_x_negy(<2 x i8> %x, <2 x i8> %yy) { ; CHECK-LABEL: @icmp_sgt_x_negy( -; CHECK-NEXT: [[Y:%.*]] = or <2 x i8> [[YY:%.*]], -; CHECK-NEXT: [[AND:%.*]] = and <2 x i8> [[Y]], [[X:%.*]] -; CHECK-NEXT: [[Z:%.*]] = icmp sgt <2 x i8> [[AND]], [[X]] -; CHECK-NEXT: ret <2 x i1> [[Z]] +; CHECK-NEXT: ret <2 x i1> zeroinitializer ; %y = or <2 x i8> %yy, %and = and <2 x i8> %y, %x @@ -155,9 +149,7 @@ define <2 x i1> @icmp_sgt_x_negy_fail_partial(<2 x i8> %x, <2 x i8> %yy) { define <2 x i1> @icmp_sle_x_posy(<2 x i8> %x, <2 x i8> %yy) { ; CHECK-LABEL: @icmp_sle_x_posy( -; CHECK-NEXT: [[Y:%.*]] = and <2 x i8> [[YY:%.*]], -; CHECK-NEXT: [[AND:%.*]] = and <2 x i8> [[Y]], [[X:%.*]] -; CHECK-NEXT: [[Z:%.*]] = icmp sle <2 x i8> [[AND]], [[X]] +; CHECK-NEXT: [[Z:%.*]] = icmp sgt <2 x i8> [[X:%.*]], ; CHECK-NEXT: ret <2 x i1> [[Z]] ; %y = and <2 x i8> %yy, @@ -183,8 +175,7 @@ define i1 @icmp_sgt_x_posy(i8 %x, i8 %y) { ; CHECK-LABEL: @icmp_sgt_x_posy( ; CHECK-NEXT: [[CY:%.*]] = icmp sgt i8 [[Y:%.*]], -1 ; CHECK-NEXT: call void @llvm.assume(i1 [[CY]]) -; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], [[Y]] -; CHECK-NEXT: [[Z:%.*]] = icmp sgt i8 [[AND]], [[X]] +; CHECK-NEXT: [[Z:%.*]] = icmp slt i8 [[X:%.*]], 0 ; CHECK-NEXT: ret i1 [[Z]] ; %cy = icmp sge i8 %y, 0 @@ -196,9 +187,7 @@ define i1 @icmp_sgt_x_posy(i8 %x, i8 %y) { define <2 x i1> @icmp_sgt_negx_y(<2 x i8> %xx, <2 x i8> %y) { ; CHECK-LABEL: @icmp_sgt_negx_y( -; CHECK-NEXT: [[X:%.*]] = or <2 x i8> [[XX:%.*]], -; CHECK-NEXT: [[AND:%.*]] = and <2 x i8> [[X]], [[Y:%.*]] -; CHECK-NEXT: [[Z:%.*]] = icmp sgt <2 x i8> [[AND]], [[X]] +; CHECK-NEXT: [[Z:%.*]] = icmp sgt <2 x i8> [[Y:%.*]], ; CHECK-NEXT: ret <2 x i1> [[Z]] ; %x = or <2 x i8> %xx, @@ -211,8 +200,7 @@ define i1 @icmp_sle_negx_y(i8 %x, i8 %y) { ; CHECK-LABEL: @icmp_sle_negx_y( ; CHECK-NEXT: [[CX:%.*]] = icmp slt i8 [[X:%.*]], 0 ; CHECK-NEXT: call void @llvm.assume(i1 [[CX]]) -; CHECK-NEXT: [[AND:%.*]] = and i8 [[X]], [[Y:%.*]] -; CHECK-NEXT: [[Z:%.*]] = icmp sle i8 [[AND]], [[X]] +; CHECK-NEXT: [[Z:%.*]] = icmp slt i8 [[Y:%.*]], 0 ; CHECK-NEXT: ret i1 [[Z]] ; %cx = icmp slt i8 %x, 0 @@ -239,9 +227,9 @@ define i1 @icmp_sle_negx_y_fail_maybe_zero(i8 %x, i8 %y) { define i1 @icmp_eq_x_invertable_y_todo(i8 %x, i1 %y) { ; CHECK-LABEL: @icmp_eq_x_invertable_y_todo( -; CHECK-NEXT: [[YY:%.*]] = select i1 [[Y:%.*]], i8 -8, i8 -25 -; CHECK-NEXT: [[AND:%.*]] = and i8 [[YY]], [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[Y:%.*]], i8 -8, i8 -25 +; CHECK-NEXT: [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = icmp eq i8 [[TMP2]], 0 ; CHECK-NEXT: ret i1 [[R]] ; %yy = select i1 %y, i8 7, i8 24 @@ -252,8 +240,8 @@ define i1 @icmp_eq_x_invertable_y_todo(i8 %x, i1 %y) { define i1 @icmp_eq_x_invertable_y(i8 %x, i8 %y) { ; CHECK-LABEL: @icmp_eq_x_invertable_y( -; CHECK-NEXT: [[AND:%.*]] = and i8 [[YY:%.*]], [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = and i8 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[R:%.*]] = icmp eq i8 [[TMP1]], 0 ; CHECK-NEXT: ret i1 [[R]] ; %yy = xor i8 %y, -1 -- GitLab From 91825512d1eed67d90a000bec2a5a46f0d417c04 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 6 Jun 2024 08:53:28 +0100 Subject: [PATCH 051/462] [ARM] vabd.ll - regenerate test checks Cleanup for #94504 --- llvm/test/CodeGen/ARM/vabd.ll | 194 +++++++++++++++++++++++++++------- 1 file changed, 154 insertions(+), 40 deletions(-) diff --git a/llvm/test/CodeGen/ARM/vabd.ll b/llvm/test/CodeGen/ARM/vabd.ll index eb5eed83d4ca..4184e9275a25 100644 --- a/llvm/test/CodeGen/ARM/vabd.ll +++ b/llvm/test/CodeGen/ARM/vabd.ll @@ -1,8 +1,14 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 ; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s define <8 x i8> @vabds8(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabds8: -;CHECK: vabd.s8 +; CHECK-LABEL: vabds8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r1] +; CHECK-NEXT: vldr d17, [r0] +; CHECK-NEXT: vabd.s8 d16, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr %tmp1 = load <8 x i8>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = call <8 x i8> @llvm.arm.neon.vabds.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2) @@ -10,8 +16,13 @@ define <8 x i8> @vabds8(ptr %A, ptr %B) nounwind { } define <4 x i16> @vabds16(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabds16: -;CHECK: vabd.s16 +; CHECK-LABEL: vabds16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r1] +; CHECK-NEXT: vldr d17, [r0] +; CHECK-NEXT: vabd.s16 d16, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr %tmp1 = load <4 x i16>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = call <4 x i16> @llvm.arm.neon.vabds.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2) @@ -19,8 +30,13 @@ define <4 x i16> @vabds16(ptr %A, ptr %B) nounwind { } define <2 x i32> @vabds32(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabds32: -;CHECK: vabd.s32 +; CHECK-LABEL: vabds32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r1] +; CHECK-NEXT: vldr d17, [r0] +; CHECK-NEXT: vabd.s32 d16, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr %tmp1 = load <2 x i32>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = call <2 x i32> @llvm.arm.neon.vabds.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2) @@ -28,8 +44,13 @@ define <2 x i32> @vabds32(ptr %A, ptr %B) nounwind { } define <8 x i8> @vabdu8(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdu8: -;CHECK: vabd.u8 +; CHECK-LABEL: vabdu8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r1] +; CHECK-NEXT: vldr d17, [r0] +; CHECK-NEXT: vabd.u8 d16, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr %tmp1 = load <8 x i8>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = call <8 x i8> @llvm.arm.neon.vabdu.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2) @@ -37,8 +58,13 @@ define <8 x i8> @vabdu8(ptr %A, ptr %B) nounwind { } define <4 x i16> @vabdu16(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdu16: -;CHECK: vabd.u16 +; CHECK-LABEL: vabdu16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r1] +; CHECK-NEXT: vldr d17, [r0] +; CHECK-NEXT: vabd.u16 d16, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr %tmp1 = load <4 x i16>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = call <4 x i16> @llvm.arm.neon.vabdu.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2) @@ -46,8 +72,13 @@ define <4 x i16> @vabdu16(ptr %A, ptr %B) nounwind { } define <2 x i32> @vabdu32(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdu32: -;CHECK: vabd.u32 +; CHECK-LABEL: vabdu32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r1] +; CHECK-NEXT: vldr d17, [r0] +; CHECK-NEXT: vabd.u32 d16, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr %tmp1 = load <2 x i32>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = call <2 x i32> @llvm.arm.neon.vabdu.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2) @@ -55,8 +86,13 @@ define <2 x i32> @vabdu32(ptr %A, ptr %B) nounwind { } define <2 x float> @vabdf32(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdf32: -;CHECK: vabd.f32 +; CHECK-LABEL: vabdf32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r1] +; CHECK-NEXT: vldr d17, [r0] +; CHECK-NEXT: vabd.f32 d16, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr %tmp1 = load <2 x float>, ptr %A %tmp2 = load <2 x float>, ptr %B %tmp3 = call <2 x float> @llvm.arm.neon.vabds.v2f32(<2 x float> %tmp1, <2 x float> %tmp2) @@ -64,8 +100,14 @@ define <2 x float> @vabdf32(ptr %A, ptr %B) nounwind { } define <16 x i8> @vabdQs8(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdQs8: -;CHECK: vabd.s8 +; CHECK-LABEL: vabdQs8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r1] +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vabd.s8 q8, q9, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr %tmp1 = load <16 x i8>, ptr %A %tmp2 = load <16 x i8>, ptr %B %tmp3 = call <16 x i8> @llvm.arm.neon.vabds.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2) @@ -73,8 +115,14 @@ define <16 x i8> @vabdQs8(ptr %A, ptr %B) nounwind { } define <8 x i16> @vabdQs16(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdQs16: -;CHECK: vabd.s16 +; CHECK-LABEL: vabdQs16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r1] +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vabd.s16 q8, q9, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr %tmp1 = load <8 x i16>, ptr %A %tmp2 = load <8 x i16>, ptr %B %tmp3 = call <8 x i16> @llvm.arm.neon.vabds.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2) @@ -82,8 +130,14 @@ define <8 x i16> @vabdQs16(ptr %A, ptr %B) nounwind { } define <4 x i32> @vabdQs32(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdQs32: -;CHECK: vabd.s32 +; CHECK-LABEL: vabdQs32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r1] +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vabd.s32 q8, q9, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr %tmp1 = load <4 x i32>, ptr %A %tmp2 = load <4 x i32>, ptr %B %tmp3 = call <4 x i32> @llvm.arm.neon.vabds.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2) @@ -91,8 +145,14 @@ define <4 x i32> @vabdQs32(ptr %A, ptr %B) nounwind { } define <16 x i8> @vabdQu8(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdQu8: -;CHECK: vabd.u8 +; CHECK-LABEL: vabdQu8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r1] +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vabd.u8 q8, q9, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr %tmp1 = load <16 x i8>, ptr %A %tmp2 = load <16 x i8>, ptr %B %tmp3 = call <16 x i8> @llvm.arm.neon.vabdu.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2) @@ -100,8 +160,14 @@ define <16 x i8> @vabdQu8(ptr %A, ptr %B) nounwind { } define <8 x i16> @vabdQu16(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdQu16: -;CHECK: vabd.u16 +; CHECK-LABEL: vabdQu16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r1] +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vabd.u16 q8, q9, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr %tmp1 = load <8 x i16>, ptr %A %tmp2 = load <8 x i16>, ptr %B %tmp3 = call <8 x i16> @llvm.arm.neon.vabdu.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2) @@ -109,8 +175,14 @@ define <8 x i16> @vabdQu16(ptr %A, ptr %B) nounwind { } define <4 x i32> @vabdQu32(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdQu32: -;CHECK: vabd.u32 +; CHECK-LABEL: vabdQu32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r1] +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vabd.u32 q8, q9, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr %tmp1 = load <4 x i32>, ptr %A %tmp2 = load <4 x i32>, ptr %B %tmp3 = call <4 x i32> @llvm.arm.neon.vabdu.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2) @@ -118,8 +190,14 @@ define <4 x i32> @vabdQu32(ptr %A, ptr %B) nounwind { } define <4 x float> @vabdQf32(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdQf32: -;CHECK: vabd.f32 +; CHECK-LABEL: vabdQf32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r1] +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vabd.f32 q8, q9, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr %tmp1 = load <4 x float>, ptr %A %tmp2 = load <4 x float>, ptr %B %tmp3 = call <4 x float> @llvm.arm.neon.vabds.v4f32(<4 x float> %tmp1, <4 x float> %tmp2) @@ -147,8 +225,14 @@ declare <4 x i32> @llvm.arm.neon.vabdu.v4i32(<4 x i32>, <4 x i32>) nounwind read declare <4 x float> @llvm.arm.neon.vabds.v4f32(<4 x float>, <4 x float>) nounwind readnone define <8 x i16> @vabdls8(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdls8: -;CHECK: vabdl.s8 +; CHECK-LABEL: vabdls8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r1] +; CHECK-NEXT: vldr d17, [r0] +; CHECK-NEXT: vabdl.s8 q8, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr %tmp1 = load <8 x i8>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = call <8 x i8> @llvm.arm.neon.vabds.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2) @@ -157,8 +241,14 @@ define <8 x i16> @vabdls8(ptr %A, ptr %B) nounwind { } define <4 x i32> @vabdls16(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdls16: -;CHECK: vabdl.s16 +; CHECK-LABEL: vabdls16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r1] +; CHECK-NEXT: vldr d17, [r0] +; CHECK-NEXT: vabdl.s16 q8, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr %tmp1 = load <4 x i16>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = call <4 x i16> @llvm.arm.neon.vabds.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2) @@ -167,8 +257,14 @@ define <4 x i32> @vabdls16(ptr %A, ptr %B) nounwind { } define <2 x i64> @vabdls32(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdls32: -;CHECK: vabdl.s32 +; CHECK-LABEL: vabdls32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r1] +; CHECK-NEXT: vldr d17, [r0] +; CHECK-NEXT: vabdl.s32 q8, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr %tmp1 = load <2 x i32>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = call <2 x i32> @llvm.arm.neon.vabds.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2) @@ -177,8 +273,14 @@ define <2 x i64> @vabdls32(ptr %A, ptr %B) nounwind { } define <8 x i16> @vabdlu8(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdlu8: -;CHECK: vabdl.u8 +; CHECK-LABEL: vabdlu8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r1] +; CHECK-NEXT: vldr d17, [r0] +; CHECK-NEXT: vabdl.u8 q8, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr %tmp1 = load <8 x i8>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = call <8 x i8> @llvm.arm.neon.vabdu.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2) @@ -187,8 +289,14 @@ define <8 x i16> @vabdlu8(ptr %A, ptr %B) nounwind { } define <4 x i32> @vabdlu16(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdlu16: -;CHECK: vabdl.u16 +; CHECK-LABEL: vabdlu16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r1] +; CHECK-NEXT: vldr d17, [r0] +; CHECK-NEXT: vabdl.u16 q8, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr %tmp1 = load <4 x i16>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = call <4 x i16> @llvm.arm.neon.vabdu.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2) @@ -197,8 +305,14 @@ define <4 x i32> @vabdlu16(ptr %A, ptr %B) nounwind { } define <2 x i64> @vabdlu32(ptr %A, ptr %B) nounwind { -;CHECK-LABEL: vabdlu32: -;CHECK: vabdl.u32 +; CHECK-LABEL: vabdlu32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r1] +; CHECK-NEXT: vldr d17, [r0] +; CHECK-NEXT: vabdl.u32 q8, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr %tmp1 = load <2 x i32>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = call <2 x i32> @llvm.arm.neon.vabdu.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2) -- GitLab From fb8421a04525a1e73c26d69a7ab5356f3ac1450d Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 6 Jun 2024 08:54:01 +0100 Subject: [PATCH 052/462] [ARM] vaba.ll - regenerate test checks Cleanup for #94504 --- llvm/test/CodeGen/ARM/vaba.ll | 193 +++++++++++++++++++++++++++------- 1 file changed, 157 insertions(+), 36 deletions(-) diff --git a/llvm/test/CodeGen/ARM/vaba.ll b/llvm/test/CodeGen/ARM/vaba.ll index e4a61ea7d91f..14419a345d82 100644 --- a/llvm/test/CodeGen/ARM/vaba.ll +++ b/llvm/test/CodeGen/ARM/vaba.ll @@ -1,8 +1,15 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 ; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s define <8 x i8> @vabas8(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabas8: -;CHECK: vaba.s8 +; CHECK-LABEL: vabas8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r2] +; CHECK-NEXT: vldr d17, [r1] +; CHECK-NEXT: vldr d18, [r0] +; CHECK-NEXT: vaba.s8 d18, d17, d16 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: mov pc, lr %tmp1 = load <8 x i8>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = load <8 x i8>, ptr %C @@ -12,8 +19,14 @@ define <8 x i8> @vabas8(ptr %A, ptr %B, ptr %C) nounwind { } define <4 x i16> @vabas16(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabas16: -;CHECK: vaba.s16 +; CHECK-LABEL: vabas16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r2] +; CHECK-NEXT: vldr d17, [r1] +; CHECK-NEXT: vldr d18, [r0] +; CHECK-NEXT: vaba.s16 d18, d17, d16 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: mov pc, lr %tmp1 = load <4 x i16>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = load <4 x i16>, ptr %C @@ -23,8 +36,14 @@ define <4 x i16> @vabas16(ptr %A, ptr %B, ptr %C) nounwind { } define <2 x i32> @vabas32(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabas32: -;CHECK: vaba.s32 +; CHECK-LABEL: vabas32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r2] +; CHECK-NEXT: vldr d17, [r1] +; CHECK-NEXT: vldr d18, [r0] +; CHECK-NEXT: vaba.s32 d18, d17, d16 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: mov pc, lr %tmp1 = load <2 x i32>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = load <2 x i32>, ptr %C @@ -34,8 +53,14 @@ define <2 x i32> @vabas32(ptr %A, ptr %B, ptr %C) nounwind { } define <8 x i8> @vabau8(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabau8: -;CHECK: vaba.u8 +; CHECK-LABEL: vabau8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r2] +; CHECK-NEXT: vldr d17, [r1] +; CHECK-NEXT: vldr d18, [r0] +; CHECK-NEXT: vaba.u8 d18, d17, d16 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: mov pc, lr %tmp1 = load <8 x i8>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = load <8 x i8>, ptr %C @@ -45,8 +70,14 @@ define <8 x i8> @vabau8(ptr %A, ptr %B, ptr %C) nounwind { } define <4 x i16> @vabau16(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabau16: -;CHECK: vaba.u16 +; CHECK-LABEL: vabau16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r2] +; CHECK-NEXT: vldr d17, [r1] +; CHECK-NEXT: vldr d18, [r0] +; CHECK-NEXT: vaba.u16 d18, d17, d16 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: mov pc, lr %tmp1 = load <4 x i16>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = load <4 x i16>, ptr %C @@ -56,8 +87,14 @@ define <4 x i16> @vabau16(ptr %A, ptr %B, ptr %C) nounwind { } define <2 x i32> @vabau32(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabau32: -;CHECK: vaba.u32 +; CHECK-LABEL: vabau32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r2] +; CHECK-NEXT: vldr d17, [r1] +; CHECK-NEXT: vldr d18, [r0] +; CHECK-NEXT: vaba.u32 d18, d17, d16 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: mov pc, lr %tmp1 = load <2 x i32>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = load <2 x i32>, ptr %C @@ -67,8 +104,15 @@ define <2 x i32> @vabau32(ptr %A, ptr %B, ptr %C) nounwind { } define <16 x i8> @vabaQs8(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabaQs8: -;CHECK: vaba.s8 +; CHECK-LABEL: vabaQs8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r2] +; CHECK-NEXT: vld1.64 {d18, d19}, [r1] +; CHECK-NEXT: vld1.64 {d20, d21}, [r0] +; CHECK-NEXT: vaba.s8 q10, q9, q8 +; CHECK-NEXT: vmov r0, r1, d20 +; CHECK-NEXT: vmov r2, r3, d21 +; CHECK-NEXT: mov pc, lr %tmp1 = load <16 x i8>, ptr %A %tmp2 = load <16 x i8>, ptr %B %tmp3 = load <16 x i8>, ptr %C @@ -78,8 +122,15 @@ define <16 x i8> @vabaQs8(ptr %A, ptr %B, ptr %C) nounwind { } define <8 x i16> @vabaQs16(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabaQs16: -;CHECK: vaba.s16 +; CHECK-LABEL: vabaQs16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r2] +; CHECK-NEXT: vld1.64 {d18, d19}, [r1] +; CHECK-NEXT: vld1.64 {d20, d21}, [r0] +; CHECK-NEXT: vaba.s16 q10, q9, q8 +; CHECK-NEXT: vmov r0, r1, d20 +; CHECK-NEXT: vmov r2, r3, d21 +; CHECK-NEXT: mov pc, lr %tmp1 = load <8 x i16>, ptr %A %tmp2 = load <8 x i16>, ptr %B %tmp3 = load <8 x i16>, ptr %C @@ -89,8 +140,15 @@ define <8 x i16> @vabaQs16(ptr %A, ptr %B, ptr %C) nounwind { } define <4 x i32> @vabaQs32(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabaQs32: -;CHECK: vaba.s32 +; CHECK-LABEL: vabaQs32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r2] +; CHECK-NEXT: vld1.64 {d18, d19}, [r1] +; CHECK-NEXT: vld1.64 {d20, d21}, [r0] +; CHECK-NEXT: vaba.s32 q10, q9, q8 +; CHECK-NEXT: vmov r0, r1, d20 +; CHECK-NEXT: vmov r2, r3, d21 +; CHECK-NEXT: mov pc, lr %tmp1 = load <4 x i32>, ptr %A %tmp2 = load <4 x i32>, ptr %B %tmp3 = load <4 x i32>, ptr %C @@ -100,8 +158,15 @@ define <4 x i32> @vabaQs32(ptr %A, ptr %B, ptr %C) nounwind { } define <16 x i8> @vabaQu8(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabaQu8: -;CHECK: vaba.u8 +; CHECK-LABEL: vabaQu8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r2] +; CHECK-NEXT: vld1.64 {d18, d19}, [r1] +; CHECK-NEXT: vld1.64 {d20, d21}, [r0] +; CHECK-NEXT: vaba.u8 q10, q9, q8 +; CHECK-NEXT: vmov r0, r1, d20 +; CHECK-NEXT: vmov r2, r3, d21 +; CHECK-NEXT: mov pc, lr %tmp1 = load <16 x i8>, ptr %A %tmp2 = load <16 x i8>, ptr %B %tmp3 = load <16 x i8>, ptr %C @@ -111,8 +176,15 @@ define <16 x i8> @vabaQu8(ptr %A, ptr %B, ptr %C) nounwind { } define <8 x i16> @vabaQu16(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabaQu16: -;CHECK: vaba.u16 +; CHECK-LABEL: vabaQu16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r2] +; CHECK-NEXT: vld1.64 {d18, d19}, [r1] +; CHECK-NEXT: vld1.64 {d20, d21}, [r0] +; CHECK-NEXT: vaba.u16 q10, q9, q8 +; CHECK-NEXT: vmov r0, r1, d20 +; CHECK-NEXT: vmov r2, r3, d21 +; CHECK-NEXT: mov pc, lr %tmp1 = load <8 x i16>, ptr %A %tmp2 = load <8 x i16>, ptr %B %tmp3 = load <8 x i16>, ptr %C @@ -122,8 +194,15 @@ define <8 x i16> @vabaQu16(ptr %A, ptr %B, ptr %C) nounwind { } define <4 x i32> @vabaQu32(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabaQu32: -;CHECK: vaba.u32 +; CHECK-LABEL: vabaQu32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r2] +; CHECK-NEXT: vld1.64 {d18, d19}, [r1] +; CHECK-NEXT: vld1.64 {d20, d21}, [r0] +; CHECK-NEXT: vaba.u32 q10, q9, q8 +; CHECK-NEXT: vmov r0, r1, d20 +; CHECK-NEXT: vmov r2, r3, d21 +; CHECK-NEXT: mov pc, lr %tmp1 = load <4 x i32>, ptr %A %tmp2 = load <4 x i32>, ptr %B %tmp3 = load <4 x i32>, ptr %C @@ -149,8 +228,15 @@ declare <8 x i16> @llvm.arm.neon.vabdu.v8i16(<8 x i16>, <8 x i16>) nounwind read declare <4 x i32> @llvm.arm.neon.vabdu.v4i32(<4 x i32>, <4 x i32>) nounwind readnone define <8 x i16> @vabals8(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabals8: -;CHECK: vabal.s8 +; CHECK-LABEL: vabals8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r2] +; CHECK-NEXT: vldr d17, [r1] +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vabal.s8 q9, d17, d16 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: vmov r2, r3, d19 +; CHECK-NEXT: mov pc, lr %tmp1 = load <8 x i16>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = load <8 x i8>, ptr %C @@ -161,8 +247,15 @@ define <8 x i16> @vabals8(ptr %A, ptr %B, ptr %C) nounwind { } define <4 x i32> @vabals16(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabals16: -;CHECK: vabal.s16 +; CHECK-LABEL: vabals16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r2] +; CHECK-NEXT: vldr d17, [r1] +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vabal.s16 q9, d17, d16 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: vmov r2, r3, d19 +; CHECK-NEXT: mov pc, lr %tmp1 = load <4 x i32>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = load <4 x i16>, ptr %C @@ -173,8 +266,15 @@ define <4 x i32> @vabals16(ptr %A, ptr %B, ptr %C) nounwind { } define <2 x i64> @vabals32(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabals32: -;CHECK: vabal.s32 +; CHECK-LABEL: vabals32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r2] +; CHECK-NEXT: vldr d17, [r1] +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vabal.s32 q9, d17, d16 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: vmov r2, r3, d19 +; CHECK-NEXT: mov pc, lr %tmp1 = load <2 x i64>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = load <2 x i32>, ptr %C @@ -185,8 +285,15 @@ define <2 x i64> @vabals32(ptr %A, ptr %B, ptr %C) nounwind { } define <8 x i16> @vabalu8(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabalu8: -;CHECK: vabal.u8 +; CHECK-LABEL: vabalu8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r2] +; CHECK-NEXT: vldr d17, [r1] +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vabal.u8 q9, d17, d16 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: vmov r2, r3, d19 +; CHECK-NEXT: mov pc, lr %tmp1 = load <8 x i16>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = load <8 x i8>, ptr %C @@ -197,8 +304,15 @@ define <8 x i16> @vabalu8(ptr %A, ptr %B, ptr %C) nounwind { } define <4 x i32> @vabalu16(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabalu16: -;CHECK: vabal.u16 +; CHECK-LABEL: vabalu16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r2] +; CHECK-NEXT: vldr d17, [r1] +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vabal.u16 q9, d17, d16 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: vmov r2, r3, d19 +; CHECK-NEXT: mov pc, lr %tmp1 = load <4 x i32>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = load <4 x i16>, ptr %C @@ -209,8 +323,15 @@ define <4 x i32> @vabalu16(ptr %A, ptr %B, ptr %C) nounwind { } define <2 x i64> @vabalu32(ptr %A, ptr %B, ptr %C) nounwind { -;CHECK-LABEL: vabalu32: -;CHECK: vabal.u32 +; CHECK-LABEL: vabalu32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vldr d16, [r2] +; CHECK-NEXT: vldr d17, [r1] +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vabal.u32 q9, d17, d16 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: vmov r2, r3, d19 +; CHECK-NEXT: mov pc, lr %tmp1 = load <2 x i64>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = load <2 x i32>, ptr %C -- GitLab From a53ed2170aad4022ff6b730de3f7c63cc35c485d Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 6 Jun 2024 08:54:48 +0100 Subject: [PATCH 053/462] [MC][RISCV] relocations.s - add missing opcode to test check --- llvm/test/MC/RISCV/relocations.s | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/MC/RISCV/relocations.s b/llvm/test/MC/RISCV/relocations.s index 3cad3d44a615..f5f6417487f2 100644 --- a/llvm/test/MC/RISCV/relocations.s +++ b/llvm/test/MC/RISCV/relocations.s @@ -185,7 +185,7 @@ auipc a0, %tlsdesc_hi(a_symbol) lw a1, %tlsdesc_load_lo(.L5)(a0) # RELOC: R_RISCV_TLSDESC_LOAD_LO12 -# INSTR: a1, %tlsdesc_load_lo(.L5)(a0) +# INSTR: lw a1, %tlsdesc_load_lo(.L5)(a0) # FIXUP: fixup A - offset: 0, value: %tlsdesc_load_lo(.L5), kind: fixup_riscv_tlsdesc_load_lo12 addi a0, a0, %tlsdesc_add_lo(.L5) -- GitLab From ea32197daa8517ff67c0691ad24d25eb5cf905f4 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Thu, 6 Jun 2024 09:17:38 +0100 Subject: [PATCH 054/462] [DebugInfo][SelectionDAG] Fix position of salvaged 'dangling' DBG_VALUEs (#94458) `SelectionDAGBuilder::handleDebugValue` has a parameter `Order` which represents the insert-at position for the new DBG_VALUE. Prior to this patch `SelectionDAGBuilder::SDNodeOrder` is used instead of the `Order` parameter. The only code-paths where `Order != SDNodeOrder` are the two calls calls to `handleDebugValue` from `salvageUnresolvedDbgValue`. `salvageUnresolvedDbgValue` is called from `resolveOrClearDbgInfo` and `dropDanglingDebugInfo`. The former is called after SelectionDAG completes one block. Some dbg.values can't be lowered to DBG_VALUEs right away. These get recorded as 'dangling' - their order-number is saved - and get salvaged later through `dropDanglingDebugInfo`, or if we've still got dangling debug info once the whole block has been emitted, through `resolveOrClearDbgInfo`. Their saved order-number is passed to `handleDebugValue`. Prior to this patch, DBG_VALUEs inserted using these functions are inserted at the "current" `SDNodeOrder` rather than the intended position that is passed to the function. Fix and add test. --- .../SelectionDAG/SelectionDAGBuilder.cpp | 9 ++-- llvm/test/DebugInfo/X86/sdag-order.ll | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 llvm/test/DebugInfo/X86/sdag-order.ll diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 64d6fd458c62..2f3626f1c820 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -1684,7 +1684,7 @@ bool SelectionDAGBuilder::handleDebugValue(ArrayRef Values, if (!FragmentExpr) continue; SDDbgValue *SDV = DAG.getVRegDbgValue( - Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder); + Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, Order); DAG.AddDbgValue(SDV, false); Offset += RegisterSize; } @@ -1699,11 +1699,10 @@ bool SelectionDAGBuilder::handleDebugValue(ArrayRef Values, } // We have created a SDDbgOperand for each Value in Values. - // Should use Order instead of SDNodeOrder? assert(!LocationOps.empty()); - SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies, - /*IsIndirect=*/false, DbgLoc, - SDNodeOrder, IsVariadic); + SDDbgValue *SDV = + DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies, + /*IsIndirect=*/false, DbgLoc, Order, IsVariadic); DAG.AddDbgValue(SDV, /*isParameter=*/false); return true; } diff --git a/llvm/test/DebugInfo/X86/sdag-order.ll b/llvm/test/DebugInfo/X86/sdag-order.ll new file mode 100644 index 000000000000..f959a8065679 --- /dev/null +++ b/llvm/test/DebugInfo/X86/sdag-order.ll @@ -0,0 +1,46 @@ +; RUN: llc %s --stop-after=finalize-isel -o - | FileCheck %s + +;; Check the DBG_VALUE which is salvaged from the dbg.value using an otherwised +;; unused value is emitted at the correct position in the function. +;; Prior (-) to patching (+), these DBG_VALUEs would sink to the bottom of the +;; function: +;; │ bb.1.if.then: +;; │- $rax = COPY %1 +;; │ DBG_VALUE 0, $noreg, !9, !DIExpression(DW_OP_plus_uconst, 4, DW_OP_stack_value) +;; │+ $rax = COPY %1 +;; │ RET 0, $rax + +; CHECK: bb.1.if.then: +; CHECK-NEXT: DBG_VALUE 0, $noreg, ![[#]], !DIExpression(DW_OP_plus_uconst, 4, DW_OP_stack_value) + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @badger(ptr sret(i64) %sret) !dbg !5 { +entry: + %f.i = getelementptr i8, ptr null, i64 4 + br label %if.then + +if.then: ; preds = %entry + tail call void @llvm.dbg.value(metadata ptr %f.i, metadata !9, metadata !DIExpression()), !dbg !11 + ret void +} + +declare void @llvm.dbg.value(metadata, metadata, metadata) + +!llvm.dbg.cu = !{!0} +!llvm.debugify = !{!2, !3} +!llvm.module.flags = !{!4} + +!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug) +!1 = !DIFile(filename: "test.ll", directory: "/") +!2 = !{i32 3} +!3 = !{i32 1} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = distinct !DISubprogram(name: "_ZNK1d1gEv", linkageName: "_ZNK1d1gEv", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8) +!6 = !DISubroutineType(types: !7) +!7 = !{} +!8 = !{!9} +!9 = !DILocalVariable(name: "1", scope: !5, file: !1, line: 1, type: !10) +!10 = !DIBasicType(name: "ty64", size: 64, encoding: DW_ATE_unsigned) +!11 = !DILocation(line: 5, column: 1, scope: !5) -- GitLab From f1e78f776908f2bc1759eae25381f576f62728a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Thu, 6 Jun 2024 10:23:59 +0200 Subject: [PATCH 055/462] [clang][Interp] Handle lvalue APValues in visitAPValueInitializer() --- clang/lib/AST/Interp/ByteCodeExprGen.cpp | 2 +- clang/test/CodeGenCXX/template-param-objects-linkage.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index 3671c41ae703..ea259639dace 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -3188,7 +3188,7 @@ bool ByteCodeExprGen::visitAPValueInitializer(const APValue &Val, const APValue &F = Val.getStructField(I); const Record::Field *RF = R->getField(I); - if (F.isInt()) { + if (F.isInt() || F.isLValue()) { PrimType T = classifyPrim(RF->Decl->getType()); if (!this->visitAPValue(F, T, E)) return false; diff --git a/clang/test/CodeGenCXX/template-param-objects-linkage.cpp b/clang/test/CodeGenCXX/template-param-objects-linkage.cpp index 63e7d8c64686..9c148ed83753 100644 --- a/clang/test/CodeGenCXX/template-param-objects-linkage.cpp +++ b/clang/test/CodeGenCXX/template-param-objects-linkage.cpp @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++20 %s -emit-llvm -o - | FileCheck %s +// RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++20 %s -emit-llvm -o - -fexperimental-new-constant-interpreter | FileCheck %s struct S { char buf[32]; }; template constexpr const char* f() { return s.buf; } -- GitLab From c2e62c745996cbd4e19ac1ffcafc849960377b57 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Thu, 6 Jun 2024 09:31:17 +0100 Subject: [PATCH 056/462] [llvm-reduce] Remove DIGlobalVariableExpressions from DICompileUnit's globals (#94497) The 'metadata' delta pass will remove !dbg attachments from globals (which are DIGlobalVariableExpression nodes). The DIGlobalVariableExpressions don't get eliminated from the IR however if they are still referenced by the globals field in DICompileUnit. Teach the 'di-metadata' pass to try removing global variable operands from metadata tuples as well as DINodes. --- .../tools/llvm-reduce/remove-debug-info-nodes.ll | 8 +++----- llvm/tools/llvm-reduce/deltas/ReduceDIMetadata.cpp | 13 +++++++------ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/llvm/test/tools/llvm-reduce/remove-debug-info-nodes.ll b/llvm/test/tools/llvm-reduce/remove-debug-info-nodes.ll index 1ceeca8b6561..127543c64c33 100644 --- a/llvm/test/tools/llvm-reduce/remove-debug-info-nodes.ll +++ b/llvm/test/tools/llvm-reduce/remove-debug-info-nodes.ll @@ -2,7 +2,7 @@ ; DICompileUnit and DISuprogram. ; ; RUN: llvm-reduce --delta-passes=di-metadata --abort-on-invalid-reduction --test FileCheck --test-arg --check-prefixes=CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t -; RUN: FileCheck <%t --enable-var-scope %s +; RUN: FileCheck <%t --enable-var-scope %s --implicit-check-not=DIGlobalVariableExpression ; CHECK-INTERESTINGNESS: define void @test() !dbg [[SUBPROG:![0-9]+]] ; CHECK-INTERESTINGNESS: !llvm.module.flags = !{ @@ -21,12 +21,10 @@ ; CHECK: !llvm.dbg.cu = !{[[CU:.+]]} -; CHECK-DAG: [[CU]] = distinct !DICompileUnit(language: DW_LANG_C99,{{.*}}, retainedTypes: [[TYPES:![0-9]+]], globals: [[GLOBALS:![0-9]+]] -; CHECK-DAG: [[EMPTY:![0-9]+]] = !{} +; CHECK-DAG: [[CU]] = distinct !DICompileUnit(language: DW_LANG_C99,{{.*}}, retainedTypes: [[TYPES:![0-9]+]], globals: [[EMPTY:![0-9]+]] +; CHECK-DAG: [[EMPTY]] = !{} ; CHECK-DAG: [[TYPES]] = !{[[T0:![0-9]+]] ; CHECK-DAG: [[T0]] = !DIBasicType(name: "unsigned int", -; CHECK-DAG: [[GLOBALS]] = !{{{![0-9]+}} - ; CHECK-DAG: [[SUBPROG]] = distinct !DISubprogram(name: "test", {{.*}}retainedNodes: [[EMPTY]]) define void @test() !dbg !17 { diff --git a/llvm/tools/llvm-reduce/deltas/ReduceDIMetadata.cpp b/llvm/tools/llvm-reduce/deltas/ReduceDIMetadata.cpp index f4d8496aba4a..38352d6342d4 100644 --- a/llvm/tools/llvm-reduce/deltas/ReduceDIMetadata.cpp +++ b/llvm/tools/llvm-reduce/deltas/ReduceDIMetadata.cpp @@ -65,12 +65,13 @@ void identifyUninterestingMDNodes(Oracle &O, MDNodeList &MDs) { SmallVector TN; for (size_t I = 0; I < Tup->getNumOperands(); ++I) { // Ignore any operands that are not DebugInfo metadata nodes. - if (isa_and_nonnull(Tup->getOperand(I))) - // Don't add uninteresting operands to the tuple. - if (!O.shouldKeep()) - continue; - - TN.push_back(Tup->getOperand(I)); + if (Metadata *Op = Tup->getOperand(I).get()) { + if (isa(Op) || isa(Op)) + // Don't add uninteresting operands to the tuple. + if (!O.shouldKeep()) + continue; + TN.push_back(Op); + } } if (TN.size() != Tup->getNumOperands()) DbgNode->replaceOperandWith(OpIdx, DbgNode->get(DbgNode->getContext(), TN)); -- GitLab From 6c9bce88a876f48eb8ab4df172bb86375b29a87f Mon Sep 17 00:00:00 2001 From: Leandro Lupori Date: Thu, 6 Jun 2024 10:32:42 +0200 Subject: [PATCH 057/462] [flang][OpenMP] Fix privatization when critical is present (#94441) When a critical construct is present inside another construct where privatizations may occur, such as a parallel construct, some privatizations are skipped if the corresponding symbols are defined inside the critical section only (see the example below). This happens because, while critical constructs have a "body", they don't have a separate scope (which makes sense, since no privatizations can occur in them). Because of this, in semantics phase, it's not possible to insert a new host association symbol, but instead the symbol from the enclosing context is used directly. This makes symbol collection in DataSharingProcessor consider the new symbol to be defined by the critical construct, instead of by the enclosing one, which causes the privatization to be skipped. Example: ``` !$omp parallel default(firstprivate) !$omp critical i = 200 !$omp end critical !$omp end parallel ``` This patch fixes this by identifying constructs where privatizations may not happen and skipping them during the collection of nested symbols. Currently, this seems to happen only with critical constructs, but others can be easily added to the skip list, if needed. Fixes https://github.com/llvm/llvm-project/issues/75767 --- flang/lib/Lower/OpenMP/DataSharingProcessor.h | 4 +++- flang/test/Lower/OpenMP/critical.f90 | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h index 80a956de35ba..fb340e6fdb10 100644 --- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h +++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h @@ -44,7 +44,9 @@ private: void Post(const T &) {} bool Pre(const parser::OpenMPConstruct &omp) { - currentConstruct = &omp; + // Skip constructs that may not have privatizations. + if (!std::holds_alternative(omp.u)) + currentConstruct = &omp; return true; } diff --git a/flang/test/Lower/OpenMP/critical.f90 b/flang/test/Lower/OpenMP/critical.f90 index d62c58b3081a..c52ae688811e 100644 --- a/flang/test/Lower/OpenMP/critical.f90 +++ b/flang/test/Lower/OpenMP/critical.f90 @@ -51,3 +51,27 @@ subroutine predetermined_privatization() end do !$omp end parallel do end + +! https://github.com/llvm/llvm-project/issues/75767 +!CHECK-LABEL: func @_QPparallel_critical_privatization( +subroutine parallel_critical_privatization() + integer :: i + + !CHECK: %[[I:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFparallel_critical_privatizationEi"} + !CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I]] {uniq_name = "_QFparallel_critical_privatizationEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) + !CHECK: omp.parallel { + !CHECK: %[[PRIV_I:.*]] = fir.alloca i32 {bindc_name = "i", pinned, uniq_name = "_QFparallel_critical_privatizationEi"} + !CHECK: %[[PRIV_I_DECL:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFparallel_critical_privatizationEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) + !CHECK: %[[TEMP:.*]] = fir.load %[[I_DECL]]#0 : !fir.ref + !CHECK: hlfir.assign %[[TEMP]] to %[[PRIV_I_DECL]]#0 temporary_lhs : i32, !fir.ref + !$omp parallel default(firstprivate) + !CHECK: omp.critical { + !$omp critical + !CHECK: %[[C200:.*]] = arith.constant 200 : i32 + !CHECK: hlfir.assign %[[C200]] to %[[PRIV_I_DECL]]#0 : i32, !fir.ref + i = 200 + !CHECK: } + !$omp end critical + !CHECK: } + !$omp end parallel +end subroutine -- GitLab From 7091dfc0e49b8c79f9e1daf6ab0ca0e65f30f347 Mon Sep 17 00:00:00 2001 From: Vassil Vassilev Date: Thu, 6 Jun 2024 11:49:10 +0300 Subject: [PATCH 058/462] [clang-repl] Lay the foundation of pretty printing for C. (#89811) --- clang/lib/Interpreter/Interpreter.cpp | 165 ++++++++++++++------------ clang/lib/Parse/ParseStmt.cpp | 5 +- clang/test/Interpreter/pretty-print.c | 8 ++ 3 files changed, 100 insertions(+), 78 deletions(-) create mode 100644 clang/test/Interpreter/pretty-print.c diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index 683f87e8c8c7..7a9527891427 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -42,6 +42,9 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TargetParser/Host.h" + +#include + using namespace clang; // FIXME: Figure out how to unify with namespace init_convenience from @@ -270,14 +273,10 @@ Interpreter::~Interpreter() { // can't find the precise resource directory in unittests so we have to hard // code them. const char *const Runtimes = R"( + #define __CLANG_REPL__ 1 #ifdef __cplusplus + #define EXTERN_C extern "C" void *__clang_Interpreter_SetValueWithAlloc(void*, void*, void*); - void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*); - void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, void*); - void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, float); - void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, double); - void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, long double); - void __clang_Interpreter_SetValueNoAlloc(void*,void*,void*,unsigned long long); struct __clang_Interpreter_NewTag{} __ci_newtag; void* operator new(__SIZE_TYPE__, void* __p, __clang_Interpreter_NewTag) noexcept; template @@ -289,7 +288,11 @@ const char *const Runtimes = R"( void __clang_Interpreter_SetValueCopyArr(const T (*Src)[N], void* Placement, unsigned long Size) { __clang_Interpreter_SetValueCopyArr(Src[0], Placement, Size); } +#else + #define EXTERN_C extern #endif // __cplusplus + + EXTERN_C void __clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, ...); )"; llvm::Expected> @@ -588,15 +591,17 @@ std::unique_ptr Interpreter::FindRuntimeInterface() { if (!LookupInterface(ValuePrintingInfo[NoAlloc], MagicRuntimeInterface[NoAlloc])) return nullptr; - if (!LookupInterface(ValuePrintingInfo[WithAlloc], - MagicRuntimeInterface[WithAlloc])) - return nullptr; - if (!LookupInterface(ValuePrintingInfo[CopyArray], - MagicRuntimeInterface[CopyArray])) - return nullptr; - if (!LookupInterface(ValuePrintingInfo[NewTag], - MagicRuntimeInterface[NewTag])) - return nullptr; + if (Ctx.getLangOpts().CPlusPlus) { + if (!LookupInterface(ValuePrintingInfo[WithAlloc], + MagicRuntimeInterface[WithAlloc])) + return nullptr; + if (!LookupInterface(ValuePrintingInfo[CopyArray], + MagicRuntimeInterface[CopyArray])) + return nullptr; + if (!LookupInterface(ValuePrintingInfo[NewTag], + MagicRuntimeInterface[NewTag])) + return nullptr; + } return createInProcessRuntimeInterfaceBuilder(*this, Ctx, S); } @@ -855,69 +860,81 @@ __clang_Interpreter_SetValueWithAlloc(void *This, void *OutVal, return VRef.getPtr(); } -// Pointers, lvalue struct that can take as a reference. -REPL_EXTERNAL_VISIBILITY void -__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, - void *Val) { +extern "C" void REPL_EXTERNAL_VISIBILITY __clang_Interpreter_SetValueNoAlloc( + void *This, void *OutVal, void *OpaqueType, ...) { Value &VRef = *(Value *)OutVal; - VRef = Value(static_cast(This), OpaqueType); - VRef.setPtr(Val); -} + Interpreter *I = static_cast(This); + VRef = Value(I, OpaqueType); + if (VRef.isVoid()) + return; -REPL_EXTERNAL_VISIBILITY void -__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, - void *OpaqueType) { - Value &VRef = *(Value *)OutVal; - VRef = Value(static_cast(This), OpaqueType); -} + va_list args; + va_start(args, /*last named param*/ OpaqueType); -static void SetValueDataBasedOnQualType(Value &V, unsigned long long Data) { - QualType QT = V.getType(); - if (const auto *ET = QT->getAs()) - QT = ET->getDecl()->getIntegerType(); - - switch (QT->castAs()->getKind()) { - default: - llvm_unreachable("unknown type kind!"); -#define X(type, name) \ - case BuiltinType::name: \ - V.set##name(Data); \ - break; - REPL_BUILTIN_TYPES -#undef X + QualType QT = VRef.getType(); + if (VRef.getKind() == Value::K_PtrOrObj) { + VRef.setPtr(va_arg(args, void *)); + } else { + if (const auto *ET = QT->getAs()) + QT = ET->getDecl()->getIntegerType(); + switch (QT->castAs()->getKind()) { + default: + llvm_unreachable("unknown type kind!"); + break; + // Types shorter than int are resolved as int, else va_arg has UB. + case BuiltinType::Bool: + VRef.setBool(va_arg(args, int)); + break; + case BuiltinType::Char_S: + VRef.setChar_S(va_arg(args, int)); + break; + case BuiltinType::SChar: + VRef.setSChar(va_arg(args, int)); + break; + case BuiltinType::Char_U: + VRef.setChar_U(va_arg(args, unsigned)); + break; + case BuiltinType::UChar: + VRef.setUChar(va_arg(args, unsigned)); + break; + case BuiltinType::Short: + VRef.setShort(va_arg(args, int)); + break; + case BuiltinType::UShort: + VRef.setUShort(va_arg(args, unsigned)); + break; + case BuiltinType::Int: + VRef.setInt(va_arg(args, int)); + break; + case BuiltinType::UInt: + VRef.setUInt(va_arg(args, unsigned)); + break; + case BuiltinType::Long: + VRef.setLong(va_arg(args, long)); + break; + case BuiltinType::ULong: + VRef.setULong(va_arg(args, unsigned long)); + break; + case BuiltinType::LongLong: + VRef.setLongLong(va_arg(args, long long)); + break; + case BuiltinType::ULongLong: + VRef.setULongLong(va_arg(args, unsigned long long)); + break; + // Types shorter than double are resolved as double, else va_arg has UB. + case BuiltinType::Float: + VRef.setFloat(va_arg(args, double)); + break; + case BuiltinType::Double: + VRef.setDouble(va_arg(args, double)); + break; + case BuiltinType::LongDouble: + VRef.setLongDouble(va_arg(args, long double)); + break; + // See REPL_BUILTIN_TYPES. + } } -} - -REPL_EXTERNAL_VISIBILITY void -__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, - unsigned long long Val) { - Value &VRef = *(Value *)OutVal; - VRef = Value(static_cast(This), OpaqueType); - SetValueDataBasedOnQualType(VRef, Val); -} - -REPL_EXTERNAL_VISIBILITY void -__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, - float Val) { - Value &VRef = *(Value *)OutVal; - VRef = Value(static_cast(This), OpaqueType); - VRef.setFloat(Val); -} - -REPL_EXTERNAL_VISIBILITY void -__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, - double Val) { - Value &VRef = *(Value *)OutVal; - VRef = Value(static_cast(This), OpaqueType); - VRef.setDouble(Val); -} - -REPL_EXTERNAL_VISIBILITY void -__clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, - long double Val) { - Value &VRef = *(Value *)OutVal; - VRef = Value(static_cast(This), OpaqueType); - VRef.setLongDouble(Val); + va_end(args); } // A trampoline to work around the fact that operator placement new cannot diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp index c25203243ee4..16a5b7483ec1 100644 --- a/clang/lib/Parse/ParseStmt.cpp +++ b/clang/lib/Parse/ParseStmt.cpp @@ -571,11 +571,8 @@ StmtResult Parser::ParseExprStatement(ParsedStmtContext StmtCtx) { } Token *CurTok = nullptr; - // If the semicolon is missing at the end of REPL input, consider if - // we want to do value printing. Note this is only enabled in C++ mode - // since part of the implementation requires C++ language features. // Note we shouldn't eat the token since the callback needs it. - if (Tok.is(tok::annot_repl_input_end) && Actions.getLangOpts().CPlusPlus) + if (Tok.is(tok::annot_repl_input_end)) CurTok = &Tok; else // Otherwise, eat the semicolon. diff --git a/clang/test/Interpreter/pretty-print.c b/clang/test/Interpreter/pretty-print.c new file mode 100644 index 000000000000..f6158ad4ecc9 --- /dev/null +++ b/clang/test/Interpreter/pretty-print.c @@ -0,0 +1,8 @@ +// REQUIRES: host-supports-jit +// UNSUPPORTED: system-aix +// RUN: cat %s | clang-repl -Xcc -xc | FileCheck %s +// RUN: cat %s | clang-repl -Xcc -std=c++11 | FileCheck %s + +const char* c_str = "Hello, world!"; c_str + +// CHECK: Not implement yet. -- GitLab From d3f8eab0accfdddc897aea61763ef57dbc83ee30 Mon Sep 17 00:00:00 2001 From: Kai Luo Date: Thu, 6 Jun 2024 08:56:23 +0000 Subject: [PATCH 059/462] [PowerPC] Add test to show alignment of toc-data symbol is changed. NFC. After O3 opt pipeline, the alignment of toc-data symbol is changed which is unexpected. --- .../CodeGen/PowerPC/tocdata-firm-alignment.ll | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 llvm/test/CodeGen/PowerPC/tocdata-firm-alignment.ll diff --git a/llvm/test/CodeGen/PowerPC/tocdata-firm-alignment.ll b/llvm/test/CodeGen/PowerPC/tocdata-firm-alignment.ll new file mode 100644 index 000000000000..c982713d4f8d --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/tocdata-firm-alignment.ll @@ -0,0 +1,24 @@ +; RUN: opt -S -passes='default' < %s | FileCheck %s + +target datalayout = "E-m:a-p:32:32-Fi32-i64:64-n32" +target triple = "powerpc-ibm-aix7.2.0.0" + +%struct.widget = type { i8, i8, i8 } + +; CHECK: @global = {{.*}}constant %struct.widget { i8 4, i8 0, i8 0 }, align 8 #0 +@global = constant %struct.widget { i8 4, i8 0, i8 0 }, align 4 #0 + +define void @baz() #1 { +bb: + call void @snork(ptr @global) + ret void +} + +define void @snork(ptr byval(%struct.widget) %arg) #1 { +bb: + %load = load volatile ptr, ptr null, align 4 + ret void +} + +attributes #0 = { "toc-data" } +attributes #1 = { "target-cpu"="pwr7" "target-features"="+altivec,+bpermd,+extdiv,+isa-v206-instructions,+vsx,-aix-shared-lib-tls-model-opt,-aix-small-local-dynamic-tls,-aix-small-local-exec-tls,-crbits,-crypto,-direct-move,-htm,-isa-v207-instructions,-isa-v30-instructions,-power8-vector,-power9-vector,-privileged,-quadword-atomics,-rop-protect,-spe" } -- GitLab From a6cc363b2743a264eb06e46cac05c3c9c92e3ef7 Mon Sep 17 00:00:00 2001 From: Dmitry Vasilyev Date: Thu, 6 Jun 2024 13:02:27 +0400 Subject: [PATCH 060/462] [lldb] Disable TestPtyServer API test when remote testing (#94587) The local PTY is not available for the remotely executed lldb-server to pass the test. Also, in general, we cannot execute the local lldb-server instance because it could be compiled for the different system/cpu target. --- lldb/test/API/tools/lldb-server/TestPtyServer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lldb/test/API/tools/lldb-server/TestPtyServer.py b/lldb/test/API/tools/lldb-server/TestPtyServer.py index 4bfcf70bfa01..345f68f6d87d 100644 --- a/lldb/test/API/tools/lldb-server/TestPtyServer.py +++ b/lldb/test/API/tools/lldb-server/TestPtyServer.py @@ -7,6 +7,7 @@ from lldbgdbserverutils import * import xml.etree.ElementTree as ET +@skipIfRemote @skipIf(hostoslist=["windows"]) class PtyServerTestCase(gdbremote_testcase.GdbRemoteTestCaseBase): def setUp(self): -- GitLab From 0f38b4d1ca026f1c28e5a75adab00173852234dd Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 6 Jun 2024 10:03:40 +0100 Subject: [PATCH 061/462] [ARM] Add neon_vabd.ll based off aarch64 tests Test coverage for #94504 --- llvm/test/CodeGen/ARM/neon_vabd.ll | 890 +++++++++++++++++++++++++++++ 1 file changed, 890 insertions(+) create mode 100644 llvm/test/CodeGen/ARM/neon_vabd.ll diff --git a/llvm/test/CodeGen/ARM/neon_vabd.ll b/llvm/test/CodeGen/ARM/neon_vabd.ll new file mode 100644 index 000000000000..14ad1a108a72 --- /dev/null +++ b/llvm/test/CodeGen/ARM/neon_vabd.ll @@ -0,0 +1,890 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s + +; +; SABD +; + +define <8 x i8> @sabd_8b(<8 x i8> %a, <8 x i8> %b) { +; CHECK-LABEL: sabd_8b: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vsubl.s8 q8, d17, d16 +; CHECK-NEXT: vabs.s16 q8, q8 +; CHECK-NEXT: vmovn.i16 d16, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr + %a.sext = sext <8 x i8> %a to <8 x i16> + %b.sext = sext <8 x i8> %b to <8 x i16> + %sub = sub <8 x i16> %a.sext, %b.sext + %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true) + %trunc = trunc <8 x i16> %abs to <8 x i8> + ret <8 x i8> %trunc +} + +define <16 x i8> @sabd_16b(<16 x i8> %a, <16 x i8> %b) { +; CHECK-LABEL: sabd_16b: +; CHECK: @ %bb.0: +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d16, d17}, [r12] +; CHECK-NEXT: vmov d18, r2, r3 +; CHECK-NEXT: vmov d19, r0, r1 +; CHECK-NEXT: vsubl.s8 q10, d18, d17 +; CHECK-NEXT: vsubl.s8 q8, d19, d16 +; CHECK-NEXT: vabs.s16 q9, q10 +; CHECK-NEXT: vabs.s16 q8, q8 +; CHECK-NEXT: vmovn.i16 d19, q9 +; CHECK-NEXT: vmovn.i16 d18, q8 +; CHECK-NEXT: vmov r2, r3, d19 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: mov pc, lr + %a.sext = sext <16 x i8> %a to <16 x i16> + %b.sext = sext <16 x i8> %b to <16 x i16> + %sub = sub <16 x i16> %a.sext, %b.sext + %abs = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %sub, i1 true) + %trunc = trunc <16 x i16> %abs to <16 x i8> + ret <16 x i8> %trunc +} + +define <4 x i16> @sabd_4h(<4 x i16> %a, <4 x i16> %b) { +; CHECK-LABEL: sabd_4h: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vsubl.s16 q8, d17, d16 +; CHECK-NEXT: vabs.s32 q8, q8 +; CHECK-NEXT: vmovn.i32 d16, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr + %a.sext = sext <4 x i16> %a to <4 x i32> + %b.sext = sext <4 x i16> %b to <4 x i32> + %sub = sub <4 x i32> %a.sext, %b.sext + %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true) + %trunc = trunc <4 x i32> %abs to <4 x i16> + ret <4 x i16> %trunc +} + +define <4 x i16> @sabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) { +; CHECK-LABEL: sabd_4h_promoted_ops: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vshl.i16 d16, d16, #8 +; CHECK-NEXT: vshl.i16 d17, d17, #8 +; CHECK-NEXT: vshr.s16 d16, d16, #8 +; CHECK-NEXT: vshr.s16 d17, d17, #8 +; CHECK-NEXT: vsub.i16 d16, d17, d16 +; CHECK-NEXT: vabs.s16 d16, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr + %a.sext = sext <4 x i8> %a to <4 x i16> + %b.sext = sext <4 x i8> %b to <4 x i16> + %sub = sub <4 x i16> %a.sext, %b.sext + %abs = call <4 x i16> @llvm.abs.v4i16(<4 x i16> %sub, i1 true) + ret <4 x i16> %abs +} + +define <8 x i16> @sabd_8h(<8 x i16> %a, <8 x i16> %b) { +; CHECK-LABEL: sabd_8h: +; CHECK: @ %bb.0: +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d16, d17}, [r12] +; CHECK-NEXT: vmov d18, r2, r3 +; CHECK-NEXT: vmov d19, r0, r1 +; CHECK-NEXT: vsubl.s16 q10, d18, d17 +; CHECK-NEXT: vsubl.s16 q8, d19, d16 +; CHECK-NEXT: vabs.s32 q9, q10 +; CHECK-NEXT: vabs.s32 q8, q8 +; CHECK-NEXT: vmovn.i32 d19, q9 +; CHECK-NEXT: vmovn.i32 d18, q8 +; CHECK-NEXT: vmov r2, r3, d19 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: mov pc, lr + %a.sext = sext <8 x i16> %a to <8 x i32> + %b.sext = sext <8 x i16> %b to <8 x i32> + %sub = sub <8 x i32> %a.sext, %b.sext + %abs = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %sub, i1 true) + %trunc = trunc <8 x i32> %abs to <8 x i16> + ret <8 x i16> %trunc +} + +define <8 x i16> @sabd_8h_promoted_ops(<8 x i8> %a, <8 x i8> %b) { +; CHECK-LABEL: sabd_8h_promoted_ops: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vsubl.s8 q8, d17, d16 +; CHECK-NEXT: vabs.s16 q8, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a.sext = sext <8 x i8> %a to <8 x i16> + %b.sext = sext <8 x i8> %b to <8 x i16> + %sub = sub <8 x i16> %a.sext, %b.sext + %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true) + ret <8 x i16> %abs +} + +define <2 x i32> @sabd_2s(<2 x i32> %a, <2 x i32> %b) { +; CHECK-LABEL: sabd_2s: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vsubl.s32 q8, d17, d16 +; CHECK-NEXT: vshr.s64 q9, q8, #63 +; CHECK-NEXT: veor q8, q8, q9 +; CHECK-NEXT: vsub.i64 q8, q8, q9 +; CHECK-NEXT: vmovn.i64 d16, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr + %a.sext = sext <2 x i32> %a to <2 x i64> + %b.sext = sext <2 x i32> %b to <2 x i64> + %sub = sub <2 x i64> %a.sext, %b.sext + %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true) + %trunc = trunc <2 x i64> %abs to <2 x i32> + ret <2 x i32> %trunc +} + +define <2 x i32> @sabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) { +; CHECK-LABEL: sabd_2s_promoted_ops: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vshl.i32 d16, d16, #16 +; CHECK-NEXT: vshl.i32 d17, d17, #16 +; CHECK-NEXT: vshr.s32 d16, d16, #16 +; CHECK-NEXT: vshr.s32 d17, d17, #16 +; CHECK-NEXT: vsub.i32 d16, d17, d16 +; CHECK-NEXT: vabs.s32 d16, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr + %a.sext = sext <2 x i16> %a to <2 x i32> + %b.sext = sext <2 x i16> %b to <2 x i32> + %sub = sub <2 x i32> %a.sext, %b.sext + %abs = call <2 x i32> @llvm.abs.v2i32(<2 x i32> %sub, i1 true) + ret <2 x i32> %abs +} + +define <4 x i32> @sabd_4s(<4 x i32> %a, <4 x i32> %b) { +; CHECK-LABEL: sabd_4s: +; CHECK: @ %bb.0: +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d16, d17}, [r12] +; CHECK-NEXT: vmov d18, r2, r3 +; CHECK-NEXT: vmov d19, r0, r1 +; CHECK-NEXT: vsubl.s32 q10, d18, d17 +; CHECK-NEXT: vsubl.s32 q8, d19, d16 +; CHECK-NEXT: vshr.s64 q9, q10, #63 +; CHECK-NEXT: vshr.s64 q11, q8, #63 +; CHECK-NEXT: veor q10, q10, q9 +; CHECK-NEXT: veor q8, q8, q11 +; CHECK-NEXT: vsub.i64 q9, q10, q9 +; CHECK-NEXT: vsub.i64 q8, q8, q11 +; CHECK-NEXT: vmovn.i64 d19, q9 +; CHECK-NEXT: vmovn.i64 d18, q8 +; CHECK-NEXT: vmov r2, r3, d19 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: mov pc, lr + %a.sext = sext <4 x i32> %a to <4 x i64> + %b.sext = sext <4 x i32> %b to <4 x i64> + %sub = sub <4 x i64> %a.sext, %b.sext + %abs = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %sub, i1 true) + %trunc = trunc <4 x i64> %abs to <4 x i32> + ret <4 x i32> %trunc +} + +define <4 x i32> @sabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) { +; CHECK-LABEL: sabd_4s_promoted_ops: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vsubl.s16 q8, d17, d16 +; CHECK-NEXT: vabs.s32 q8, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a.sext = sext <4 x i16> %a to <4 x i32> + %b.sext = sext <4 x i16> %b to <4 x i32> + %sub = sub <4 x i32> %a.sext, %b.sext + %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true) + ret <4 x i32> %abs +} + +define <2 x i64> @sabd_2d(<2 x i64> %a, <2 x i64> %b) { +; CHECK-LABEL: sabd_2d: +; CHECK: @ %bb.0: +; CHECK-NEXT: .save {r4, r5, r6, r7, r11, lr} +; CHECK-NEXT: push {r4, r5, r6, r7, r11, lr} +; CHECK-NEXT: add r12, sp, #24 +; CHECK-NEXT: asr r6, r3, #31 +; CHECK-NEXT: vld1.64 {d16, d17}, [r12] +; CHECK-NEXT: vmov r12, lr, d17 +; CHECK-NEXT: vmov r7, r5, d16 +; CHECK-NEXT: subs r2, r2, r12 +; CHECK-NEXT: sbcs r3, r3, lr +; CHECK-NEXT: sbcs r4, r6, lr, asr #31 +; CHECK-NEXT: sbc r6, r6, lr, asr #31 +; CHECK-NEXT: eor r2, r2, r6, asr #31 +; CHECK-NEXT: eor r3, r3, r6, asr #31 +; CHECK-NEXT: subs r2, r2, r6, asr #31 +; CHECK-NEXT: sbc r3, r3, r6, asr #31 +; CHECK-NEXT: subs r0, r0, r7 +; CHECK-NEXT: asr r6, r1, #31 +; CHECK-NEXT: sbcs r1, r1, r5 +; CHECK-NEXT: sbcs r7, r6, r5, asr #31 +; CHECK-NEXT: vmov.32 d17[0], r2 +; CHECK-NEXT: sbc r7, r6, r5, asr #31 +; CHECK-NEXT: eor r0, r0, r7, asr #31 +; CHECK-NEXT: subs r0, r0, r7, asr #31 +; CHECK-NEXT: vmov.32 d16[0], r0 +; CHECK-NEXT: eor r0, r1, r7, asr #31 +; CHECK-NEXT: sbc r0, r0, r7, asr #31 +; CHECK-NEXT: vmov.32 d17[1], r3 +; CHECK-NEXT: vmov.32 d16[1], r0 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: pop {r4, r5, r6, r7, r11, lr} +; CHECK-NEXT: mov pc, lr + %a.sext = sext <2 x i64> %a to <2 x i128> + %b.sext = sext <2 x i64> %b to <2 x i128> + %sub = sub <2 x i128> %a.sext, %b.sext + %abs = call <2 x i128> @llvm.abs.v2i128(<2 x i128> %sub, i1 true) + %trunc = trunc <2 x i128> %abs to <2 x i64> + ret <2 x i64> %trunc +} + +define <2 x i64> @sabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) { +; CHECK-LABEL: sabd_2d_promoted_ops: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vsubl.s32 q8, d17, d16 +; CHECK-NEXT: vshr.s64 q9, q8, #63 +; CHECK-NEXT: veor q8, q8, q9 +; CHECK-NEXT: vsub.i64 q8, q8, q9 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a.sext = sext <2 x i32> %a to <2 x i64> + %b.sext = sext <2 x i32> %b to <2 x i64> + %sub = sub <2 x i64> %a.sext, %b.sext + %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true) + ret <2 x i64> %abs +} + +; +; UABD +; + +define <8 x i8> @uabd_8b(<8 x i8> %a, <8 x i8> %b) { +; CHECK-LABEL: uabd_8b: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vabdl.u8 q8, d17, d16 +; CHECK-NEXT: vmovn.i16 d16, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr + %a.zext = zext <8 x i8> %a to <8 x i16> + %b.zext = zext <8 x i8> %b to <8 x i16> + %sub = sub <8 x i16> %a.zext, %b.zext + %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true) + %trunc = trunc <8 x i16> %abs to <8 x i8> + ret <8 x i8> %trunc +} + +define <16 x i8> @uabd_16b(<16 x i8> %a, <16 x i8> %b) { +; CHECK-LABEL: uabd_16b: +; CHECK: @ %bb.0: +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d16, d17}, [r12] +; CHECK-NEXT: vmov d18, r2, r3 +; CHECK-NEXT: vmov d19, r0, r1 +; CHECK-NEXT: vabdl.u8 q10, d18, d17 +; CHECK-NEXT: vabdl.u8 q8, d19, d16 +; CHECK-NEXT: vmovn.i16 d19, q10 +; CHECK-NEXT: vmovn.i16 d18, q8 +; CHECK-NEXT: vmov r2, r3, d19 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: mov pc, lr + %a.zext = zext <16 x i8> %a to <16 x i16> + %b.zext = zext <16 x i8> %b to <16 x i16> + %sub = sub <16 x i16> %a.zext, %b.zext + %abs = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %sub, i1 true) + %trunc = trunc <16 x i16> %abs to <16 x i8> + ret <16 x i8> %trunc +} + +define <4 x i16> @uabd_4h(<4 x i16> %a, <4 x i16> %b) { +; CHECK-LABEL: uabd_4h: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vabdl.u16 q8, d17, d16 +; CHECK-NEXT: vmovn.i32 d16, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr + %a.zext = zext <4 x i16> %a to <4 x i32> + %b.zext = zext <4 x i16> %b to <4 x i32> + %sub = sub <4 x i32> %a.zext, %b.zext + %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true) + %trunc = trunc <4 x i32> %abs to <4 x i16> + ret <4 x i16> %trunc +} + +define <4 x i16> @uabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) { +; CHECK-LABEL: uabd_4h_promoted_ops: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vbic.i16 d16, #0xff00 +; CHECK-NEXT: vbic.i16 d17, #0xff00 +; CHECK-NEXT: vsub.i16 d16, d17, d16 +; CHECK-NEXT: vabs.s16 d16, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr + %a.zext = zext <4 x i8> %a to <4 x i16> + %b.zext = zext <4 x i8> %b to <4 x i16> + %sub = sub <4 x i16> %a.zext, %b.zext + %abs = call <4 x i16> @llvm.abs.v4i16(<4 x i16> %sub, i1 true) + ret <4 x i16> %abs +} + +define <8 x i16> @uabd_8h(<8 x i16> %a, <8 x i16> %b) { +; CHECK-LABEL: uabd_8h: +; CHECK: @ %bb.0: +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d16, d17}, [r12] +; CHECK-NEXT: vmov d18, r2, r3 +; CHECK-NEXT: vmov d19, r0, r1 +; CHECK-NEXT: vabdl.u16 q10, d18, d17 +; CHECK-NEXT: vabdl.u16 q8, d19, d16 +; CHECK-NEXT: vmovn.i32 d19, q10 +; CHECK-NEXT: vmovn.i32 d18, q8 +; CHECK-NEXT: vmov r2, r3, d19 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: mov pc, lr + %a.zext = zext <8 x i16> %a to <8 x i32> + %b.zext = zext <8 x i16> %b to <8 x i32> + %sub = sub <8 x i32> %a.zext, %b.zext + %abs = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %sub, i1 true) + %trunc = trunc <8 x i32> %abs to <8 x i16> + ret <8 x i16> %trunc +} + +define <8 x i16> @uabd_8h_promoted_ops(<8 x i8> %a, <8 x i8> %b) { +; CHECK-LABEL: uabd_8h_promoted_ops: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vabdl.u8 q8, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a.zext = zext <8 x i8> %a to <8 x i16> + %b.zext = zext <8 x i8> %b to <8 x i16> + %sub = sub <8 x i16> %a.zext, %b.zext + %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true) + ret <8 x i16> %abs +} + +define <2 x i32> @uabd_2s(<2 x i32> %a, <2 x i32> %b) { +; CHECK-LABEL: uabd_2s: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vsubl.u32 q8, d17, d16 +; CHECK-NEXT: vshr.s64 q9, q8, #63 +; CHECK-NEXT: veor q8, q8, q9 +; CHECK-NEXT: vsub.i64 q8, q8, q9 +; CHECK-NEXT: vmovn.i64 d16, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr + %a.zext = zext <2 x i32> %a to <2 x i64> + %b.zext = zext <2 x i32> %b to <2 x i64> + %sub = sub <2 x i64> %a.zext, %b.zext + %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true) + %trunc = trunc <2 x i64> %abs to <2 x i32> + ret <2 x i32> %trunc +} + +define <2 x i32> @uabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) { +; CHECK-LABEL: uabd_2s_promoted_ops: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov.i32 d16, #0xffff +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: vmov d18, r0, r1 +; CHECK-NEXT: vand d17, d17, d16 +; CHECK-NEXT: vand d16, d18, d16 +; CHECK-NEXT: vsub.i32 d16, d16, d17 +; CHECK-NEXT: vabs.s32 d16, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr + %a.zext = zext <2 x i16> %a to <2 x i32> + %b.zext = zext <2 x i16> %b to <2 x i32> + %sub = sub <2 x i32> %a.zext, %b.zext + %abs = call <2 x i32> @llvm.abs.v2i32(<2 x i32> %sub, i1 true) + ret <2 x i32> %abs +} + +define <4 x i32> @uabd_4s(<4 x i32> %a, <4 x i32> %b) { +; CHECK-LABEL: uabd_4s: +; CHECK: @ %bb.0: +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d16, d17}, [r12] +; CHECK-NEXT: vmov d18, r2, r3 +; CHECK-NEXT: vmov d19, r0, r1 +; CHECK-NEXT: vsubl.u32 q10, d18, d17 +; CHECK-NEXT: vsubl.u32 q8, d19, d16 +; CHECK-NEXT: vshr.s64 q9, q10, #63 +; CHECK-NEXT: vshr.s64 q11, q8, #63 +; CHECK-NEXT: veor q10, q10, q9 +; CHECK-NEXT: veor q8, q8, q11 +; CHECK-NEXT: vsub.i64 q9, q10, q9 +; CHECK-NEXT: vsub.i64 q8, q8, q11 +; CHECK-NEXT: vmovn.i64 d19, q9 +; CHECK-NEXT: vmovn.i64 d18, q8 +; CHECK-NEXT: vmov r2, r3, d19 +; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: mov pc, lr + %a.zext = zext <4 x i32> %a to <4 x i64> + %b.zext = zext <4 x i32> %b to <4 x i64> + %sub = sub <4 x i64> %a.zext, %b.zext + %abs = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %sub, i1 true) + %trunc = trunc <4 x i64> %abs to <4 x i32> + ret <4 x i32> %trunc +} + +define <4 x i32> @uabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) { +; CHECK-LABEL: uabd_4s_promoted_ops: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vabdl.u16 q8, d17, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a.zext = zext <4 x i16> %a to <4 x i32> + %b.zext = zext <4 x i16> %b to <4 x i32> + %sub = sub <4 x i32> %a.zext, %b.zext + %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true) + ret <4 x i32> %abs +} + +define <2 x i64> @uabd_2d(<2 x i64> %a, <2 x i64> %b) { +; CHECK-LABEL: uabd_2d: +; CHECK: @ %bb.0: +; CHECK-NEXT: .save {r4, r5, r6, r7, r11, lr} +; CHECK-NEXT: push {r4, r5, r6, r7, r11, lr} +; CHECK-NEXT: add r12, sp, #24 +; CHECK-NEXT: mov r6, #0 +; CHECK-NEXT: vld1.64 {d16, d17}, [r12] +; CHECK-NEXT: vmov r12, lr, d17 +; CHECK-NEXT: vmov r4, r7, d16 +; CHECK-NEXT: subs r2, r2, r12 +; CHECK-NEXT: sbcs r3, r3, lr +; CHECK-NEXT: sbcs r5, r6, #0 +; CHECK-NEXT: sbc r5, r6, #0 +; CHECK-NEXT: eor r2, r2, r5, asr #31 +; CHECK-NEXT: eor r3, r3, r5, asr #31 +; CHECK-NEXT: subs r2, r2, r5, asr #31 +; CHECK-NEXT: sbc r3, r3, r5, asr #31 +; CHECK-NEXT: subs r0, r0, r4 +; CHECK-NEXT: sbcs r1, r1, r7 +; CHECK-NEXT: vmov.32 d17[0], r2 +; CHECK-NEXT: sbcs r7, r6, #0 +; CHECK-NEXT: sbc r7, r6, #0 +; CHECK-NEXT: eor r0, r0, r7, asr #31 +; CHECK-NEXT: subs r0, r0, r7, asr #31 +; CHECK-NEXT: vmov.32 d16[0], r0 +; CHECK-NEXT: eor r0, r1, r7, asr #31 +; CHECK-NEXT: sbc r0, r0, r7, asr #31 +; CHECK-NEXT: vmov.32 d17[1], r3 +; CHECK-NEXT: vmov.32 d16[1], r0 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: pop {r4, r5, r6, r7, r11, lr} +; CHECK-NEXT: mov pc, lr + %a.zext = zext <2 x i64> %a to <2 x i128> + %b.zext = zext <2 x i64> %b to <2 x i128> + %sub = sub <2 x i128> %a.zext, %b.zext + %abs = call <2 x i128> @llvm.abs.v2i128(<2 x i128> %sub, i1 true) + %trunc = trunc <2 x i128> %abs to <2 x i64> + ret <2 x i64> %trunc +} + +define <2 x i64> @uabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) { +; CHECK-LABEL: uabd_2d_promoted_ops: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d16, r2, r3 +; CHECK-NEXT: vmov d17, r0, r1 +; CHECK-NEXT: vsubl.u32 q8, d17, d16 +; CHECK-NEXT: vshr.s64 q9, q8, #63 +; CHECK-NEXT: veor q8, q8, q9 +; CHECK-NEXT: vsub.i64 q8, q8, q9 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a.zext = zext <2 x i32> %a to <2 x i64> + %b.zext = zext <2 x i32> %b to <2 x i64> + %sub = sub <2 x i64> %a.zext, %b.zext + %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true) + ret <2 x i64> %abs +} + +define <16 x i8> @uabd_v16i8_nuw(<16 x i8> %a, <16 x i8> %b) { +; CHECK-LABEL: uabd_v16i8_nuw: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vsub.i8 q8, q8, q9 +; CHECK-NEXT: vabs.s8 q8, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %sub = sub nuw <16 x i8> %a, %b + %abs = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %sub, i1 true) + ret <16 x i8> %abs +} + +define <8 x i16> @uabd_v8i16_nuw(<8 x i16> %a, <8 x i16> %b) { +; CHECK-LABEL: uabd_v8i16_nuw: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vsub.i16 q8, q8, q9 +; CHECK-NEXT: vabs.s16 q8, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %sub = sub nuw <8 x i16> %a, %b + %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true) + ret <8 x i16> %abs +} + +define <4 x i32> @uabd_v4i32_nuw(<4 x i32> %a, <4 x i32> %b) { +; CHECK-LABEL: uabd_v4i32_nuw: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vsub.i32 q8, q8, q9 +; CHECK-NEXT: vabs.s32 q8, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %sub = sub nuw <4 x i32> %a, %b + %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true) + ret <4 x i32> %abs +} + +define <2 x i64> @uabd_v2i64_nuw(<2 x i64> %a, <2 x i64> %b) { +; CHECK-LABEL: uabd_v2i64_nuw: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vsub.i64 q8, q8, q9 +; CHECK-NEXT: vshr.s64 q9, q8, #63 +; CHECK-NEXT: veor q8, q8, q9 +; CHECK-NEXT: vsub.i64 q8, q8, q9 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %sub = sub nuw <2 x i64> %a, %b + %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true) + ret <2 x i64> %abs +} + +define <16 x i8> @sabd_v16i8_nsw(<16 x i8> %a, <16 x i8> %b) { +; CHECK-LABEL: sabd_v16i8_nsw: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vsub.i8 q8, q8, q9 +; CHECK-NEXT: vabs.s8 q8, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %sub = sub nsw <16 x i8> %a, %b + %abs = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %sub, i1 true) + ret <16 x i8> %abs +} + +define <8 x i16> @sabd_v8i16_nsw(<8 x i16> %a, <8 x i16> %b) { +; CHECK-LABEL: sabd_v8i16_nsw: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vsub.i16 q8, q8, q9 +; CHECK-NEXT: vabs.s16 q8, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %sub = sub nsw <8 x i16> %a, %b + %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true) + ret <8 x i16> %abs +} + +define <4 x i32> @sabd_v4i32_nsw(<4 x i32> %a, <4 x i32> %b) { +; CHECK-LABEL: sabd_v4i32_nsw: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vsub.i32 q8, q8, q9 +; CHECK-NEXT: vabs.s32 q8, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %sub = sub nsw <4 x i32> %a, %b + %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true) + ret <4 x i32> %abs +} + +define <2 x i64> @sabd_v2i64_nsw(<2 x i64> %a, <2 x i64> %b) { +; CHECK-LABEL: sabd_v2i64_nsw: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vsub.i64 q8, q8, q9 +; CHECK-NEXT: vshr.s64 q9, q8, #63 +; CHECK-NEXT: veor q8, q8, q9 +; CHECK-NEXT: vsub.i64 q8, q8, q9 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %sub = sub nsw <2 x i64> %a, %b + %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true) + ret <2 x i64> %abs +} + +define <16 x i8> @smaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) { +; CHECK-LABEL: smaxmin_v16i8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vmin.s8 q10, q8, q9 +; CHECK-NEXT: vmax.s8 q8, q8, q9 +; CHECK-NEXT: vsub.i8 q8, q8, q10 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a = tail call <16 x i8> @llvm.smax.v16i8(<16 x i8> %0, <16 x i8> %1) + %b = tail call <16 x i8> @llvm.smin.v16i8(<16 x i8> %0, <16 x i8> %1) + %sub = sub <16 x i8> %a, %b + ret <16 x i8> %sub +} + +define <8 x i16> @smaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) { +; CHECK-LABEL: smaxmin_v8i16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vmin.s16 q10, q8, q9 +; CHECK-NEXT: vmax.s16 q8, q8, q9 +; CHECK-NEXT: vsub.i16 q8, q8, q10 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a = tail call <8 x i16> @llvm.smax.v8i16(<8 x i16> %0, <8 x i16> %1) + %b = tail call <8 x i16> @llvm.smin.v8i16(<8 x i16> %0, <8 x i16> %1) + %sub = sub <8 x i16> %a, %b + ret <8 x i16> %sub +} + +define <4 x i32> @smaxmin_v4i32(<4 x i32> %0, <4 x i32> %1) { +; CHECK-LABEL: smaxmin_v4i32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vmin.s32 q10, q8, q9 +; CHECK-NEXT: vmax.s32 q8, q8, q9 +; CHECK-NEXT: vsub.i32 q8, q8, q10 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a = tail call <4 x i32> @llvm.smax.v4i32(<4 x i32> %0, <4 x i32> %1) + %b = tail call <4 x i32> @llvm.smin.v4i32(<4 x i32> %0, <4 x i32> %1) + %sub = sub <4 x i32> %a, %b + ret <4 x i32> %sub +} + +define <2 x i64> @smaxmin_v2i64(<2 x i64> %0, <2 x i64> %1) { +; CHECK-LABEL: smaxmin_v2i64: +; CHECK: @ %bb.0: +; CHECK-NEXT: .save {r4, r5, r6, r7, r8, lr} +; CHECK-NEXT: push {r4, r5, r6, r7, r8, lr} +; CHECK-NEXT: add r6, sp, #24 +; CHECK-NEXT: mov r8, #0 +; CHECK-NEXT: vld1.64 {d18, d19}, [r6] +; CHECK-NEXT: vmov r7, r12, d19 +; CHECK-NEXT: vmov r4, lr, d18 +; CHECK-NEXT: subs r5, r2, r7 +; CHECK-NEXT: sbcs r5, r3, r12 +; CHECK-NEXT: mov r6, r7 +; CHECK-NEXT: mov r5, #0 +; CHECK-NEXT: movlt r5, #1 +; CHECK-NEXT: cmp r5, #0 +; CHECK-NEXT: movne r6, r2 +; CHECK-NEXT: mov r5, r12 +; CHECK-NEXT: vmov.32 d17[0], r6 +; CHECK-NEXT: movne r5, r3 +; CHECK-NEXT: mov r6, r4 +; CHECK-NEXT: vmov.32 d17[1], r5 +; CHECK-NEXT: subs r5, r4, r0 +; CHECK-NEXT: sbcs r5, lr, r1 +; CHECK-NEXT: mov r5, #0 +; CHECK-NEXT: movlt r5, #1 +; CHECK-NEXT: cmp r5, #0 +; CHECK-NEXT: movne r6, r0 +; CHECK-NEXT: vmov.32 d18[0], r6 +; CHECK-NEXT: subs r6, r7, r2 +; CHECK-NEXT: sbcs r6, r12, r3 +; CHECK-NEXT: mov r6, #0 +; CHECK-NEXT: movlt r6, #1 +; CHECK-NEXT: cmp r6, #0 +; CHECK-NEXT: movne r7, r2 +; CHECK-NEXT: subs r2, r0, r4 +; CHECK-NEXT: sbcs r2, r1, lr +; CHECK-NEXT: vmov.32 d19[0], r7 +; CHECK-NEXT: movlt r8, #1 +; CHECK-NEXT: cmp r8, #0 +; CHECK-NEXT: movne r4, r0 +; CHECK-NEXT: mov r0, lr +; CHECK-NEXT: vmov.32 d16[0], r4 +; CHECK-NEXT: movne r0, r1 +; CHECK-NEXT: cmp r6, #0 +; CHECK-NEXT: movne r12, r3 +; CHECK-NEXT: cmp r5, #0 +; CHECK-NEXT: vmov.32 d16[1], r0 +; CHECK-NEXT: movne lr, r1 +; CHECK-NEXT: vmov.32 d19[1], r12 +; CHECK-NEXT: vmov.32 d18[1], lr +; CHECK-NEXT: vsub.i64 q8, q9, q8 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: pop {r4, r5, r6, r7, r8, lr} +; CHECK-NEXT: mov pc, lr + %a = tail call <2 x i64> @llvm.smax.v2i64(<2 x i64> %0, <2 x i64> %1) + %b = tail call <2 x i64> @llvm.smin.v2i64(<2 x i64> %0, <2 x i64> %1) + %sub = sub <2 x i64> %a, %b + ret <2 x i64> %sub +} + +define <16 x i8> @umaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) { +; CHECK-LABEL: umaxmin_v16i8: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vmin.u8 q10, q8, q9 +; CHECK-NEXT: vmax.u8 q8, q8, q9 +; CHECK-NEXT: vsub.i8 q8, q8, q10 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %0, <16 x i8> %1) + %b = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %0, <16 x i8> %1) + %sub = sub <16 x i8> %a, %b + ret <16 x i8> %sub +} + +define <8 x i16> @umaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) { +; CHECK-LABEL: umaxmin_v8i16: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vmin.u16 q10, q8, q9 +; CHECK-NEXT: vmax.u16 q8, q8, q9 +; CHECK-NEXT: vsub.i16 q8, q8, q10 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a = tail call <8 x i16> @llvm.umax.v8i16(<8 x i16> %0, <8 x i16> %1) + %b = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %0, <8 x i16> %1) + %sub = sub <8 x i16> %a, %b + ret <8 x i16> %sub +} + +define <4 x i32> @umaxmin_v4i32(<4 x i32> %0, <4 x i32> %1) { +; CHECK-LABEL: umaxmin_v4i32: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vmin.u32 q10, q8, q9 +; CHECK-NEXT: vmax.u32 q8, q8, q9 +; CHECK-NEXT: vsub.i32 q8, q8, q10 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a = tail call <4 x i32> @llvm.umax.v4i32(<4 x i32> %0, <4 x i32> %1) + %b = tail call <4 x i32> @llvm.umin.v4i32(<4 x i32> %0, <4 x i32> %1) + %sub = sub <4 x i32> %a, %b + ret <4 x i32> %sub +} + +define <2 x i64> @umaxmin_v2i64(<2 x i64> %0, <2 x i64> %1) { +; CHECK-LABEL: umaxmin_v2i64: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vqsub.u64 q10, q8, q9 +; CHECK-NEXT: vqsub.u64 q9, q9, q8 +; CHECK-NEXT: vsub.i64 q10, q10, q8 +; CHECK-NEXT: vadd.i64 q8, q8, q9 +; CHECK-NEXT: vadd.i64 q8, q8, q10 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a = tail call <2 x i64> @llvm.umax.v2i64(<2 x i64> %0, <2 x i64> %1) + %b = tail call <2 x i64> @llvm.umin.v2i64(<2 x i64> %0, <2 x i64> %1) + %sub = sub <2 x i64> %a, %b + ret <2 x i64> %sub +} + +define <16 x i8> @umaxmin_v16i8_com1(<16 x i8> %0, <16 x i8> %1) { +; CHECK-LABEL: umaxmin_v16i8_com1: +; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: mov r0, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r0] +; CHECK-NEXT: vmin.u8 q10, q9, q8 +; CHECK-NEXT: vmax.u8 q8, q8, q9 +; CHECK-NEXT: vsub.i8 q8, q8, q10 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 +; CHECK-NEXT: mov pc, lr + %a = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %0, <16 x i8> %1) + %b = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %1, <16 x i8> %0) + %sub = sub <16 x i8> %a, %b + ret <16 x i8> %sub +} -- GitLab From 03a2fe9a7574c0cff724666e713dead7009a9621 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 6 Jun 2024 10:06:47 +0100 Subject: [PATCH 062/462] [DAG] visitSUB - update the ABS matching code to use SDPatternMatch and hasOperation. Avoids the need to explicitly test both commuted variants and doesn't match custom lowering after legalization. Cleanup for #94504 --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 9a5359015439..02cd125eeff0 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -4041,17 +4041,11 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { return DAG.getNode(ISD::ADD, DL, VT, N0, SExt); } - // fold Y = sra (X, size(X)-1); sub (xor (X, Y), Y) -> (abs X) - if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { - if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) { - SDValue X0 = N0.getOperand(0), X1 = N0.getOperand(1); - SDValue S0 = N1.getOperand(0); - if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0)) - if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1))) - if (C->getAPIntValue() == (BitWidth - 1)) - return DAG.getNode(ISD::ABS, DL, VT, S0); - } - } + // fold B = sra (A, size(A)-1); sub (xor (A, B), B) -> (abs A) + if (hasOperation(ISD::ABS, VT) && + sd_match(N1, m_Sra(m_Value(A), m_SpecificInt(BitWidth - 1))) && + sd_match(N0, m_Xor(m_Specific(A), m_Specific(N1)))) + return DAG.getNode(ISD::ABS, DL, VT, A); // If the relocation model supports it, consider symbol offsets. if (GlobalAddressSDNode *GA = dyn_cast(N0)) -- GitLab From fdcdc3d759224f0ec0a3e472f7940d4b0f3a1b79 Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Thu, 6 Jun 2024 10:10:22 +0100 Subject: [PATCH 063/462] [flang][CodeGen][NFC] Reduce boilerplatre for ExternalNameConversion (#94474) Use tablegen to generate the pass constructor. I removed the duplicated pass option handling. I don't understand why the manual instantiation of the pass needs its own duplicate of the pass options in the (automatically generated) base class (even with the option to ignore the pass options in the base class). This pass doesn't need changes to support other top level operations. --- .../flang/Optimizer/Transforms/Passes.h | 3 --- .../flang/Optimizer/Transforms/Passes.td | 1 - flang/include/flang/Tools/CLOptions.inc | 5 ++-- .../Transforms/ExternalNameConversion.cpp | 23 ++++--------------- 4 files changed, 6 insertions(+), 26 deletions(-) diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h index a7ba704fdb39..2d43f4d4c55b 100644 --- a/flang/include/flang/Optimizer/Transforms/Passes.h +++ b/flang/include/flang/Optimizer/Transforms/Passes.h @@ -60,9 +60,6 @@ std::unique_ptr createAffineDemotionPass(); std::unique_ptr createArrayValueCopyPass(fir::ArrayValueCopyOptions options = {}); std::unique_ptr createCFGConversionPassWithNSW(); -std::unique_ptr createExternalNameConversionPass(); -std::unique_ptr -createExternalNameConversionPass(bool appendUnderscore); std::unique_ptr createMemDataFlowOptPass(); std::unique_ptr createPromoteToAffinePass(); std::unique_ptr diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td index 82638200e5e2..cac590a8da00 100644 --- a/flang/include/flang/Optimizer/Transforms/Passes.td +++ b/flang/include/flang/Optimizer/Transforms/Passes.td @@ -163,7 +163,6 @@ def ExternalNameConversion : Pass<"external-name-interop", "mlir::ModuleOp"> { let description = [{ Demangle FIR internal name and mangle them for external interoperability. }]; - let constructor = "::fir::createExternalNameConversionPass()"; let options = [ Option<"appendUnderscoreOpt", "append-underscore", "bool", /*default=*/"true", diff --git a/flang/include/flang/Tools/CLOptions.inc b/flang/include/flang/Tools/CLOptions.inc index c5c35e9a6a33..d0399d65f565 100644 --- a/flang/include/flang/Tools/CLOptions.inc +++ b/flang/include/flang/Tools/CLOptions.inc @@ -233,9 +233,8 @@ inline void addBoxedProcedurePass(mlir::PassManager &pm) { inline void addExternalNameConversionPass( mlir::PassManager &pm, bool appendUnderscore = true) { - addPassConditionally(pm, disableExternalNameConversion, [&]() { - return fir::createExternalNameConversionPass(appendUnderscore); - }); + addPassConditionally(pm, disableExternalNameConversion, + [&]() { return fir::createExternalNameConversion({appendUnderscore}); }); } // Use inliner extension point callback to register the default inliner pass. diff --git a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp index b265c74c33dd..648628fd1c9a 100644 --- a/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp +++ b/flang/lib/Optimizer/Transforms/ExternalNameConversion.cpp @@ -45,17 +45,11 @@ namespace { class ExternalNameConversionPass : public fir::impl::ExternalNameConversionBase { public: - ExternalNameConversionPass(bool appendUnderscoring) - : appendUnderscores(appendUnderscoring) {} - - ExternalNameConversionPass() { usePassOpt = true; } + using ExternalNameConversionBase< + ExternalNameConversionPass>::ExternalNameConversionBase; mlir::ModuleOp getModule() { return getOperation(); } void runOnOperation() override; - -private: - bool appendUnderscores; - bool usePassOpt = false; }; } // namespace @@ -63,7 +57,6 @@ void ExternalNameConversionPass::runOnOperation() { auto op = getOperation(); auto *context = &getContext(); - appendUnderscores = (usePassOpt) ? appendUnderscoreOpt : appendUnderscores; llvm::DenseMap remappings; // Update names of external Fortran functions and names of Common Block // globals. @@ -74,7 +67,8 @@ void ExternalNameConversionPass::runOnOperation() { mlir::SymbolTable::getSymbolAttrName()); auto deconstructedName = fir::NameUniquer::deconstruct(symName); if (fir::NameUniquer::isExternalFacingUniquedName(deconstructedName)) { - auto newName = mangleExternalName(deconstructedName, appendUnderscores); + auto newName = + mangleExternalName(deconstructedName, appendUnderscoreOpt); auto newAttr = mlir::StringAttr::get(context, newName); mlir::SymbolTable::setSymbolName(&funcOrGlobal, newAttr); auto newSymRef = mlir::FlatSymbolRefAttr::get(newAttr); @@ -101,12 +95,3 @@ void ExternalNameConversionPass::runOnOperation() { nestedOp->setAttr(update.first, update.second); }); } - -std::unique_ptr fir::createExternalNameConversionPass() { - return std::make_unique(); -} - -std::unique_ptr -fir::createExternalNameConversionPass(bool appendUnderscoring) { - return std::make_unique(appendUnderscoring); -} -- GitLab From bf02f81da743e60a5c51fc8f5ff43d57cf6db407 Mon Sep 17 00:00:00 2001 From: Kai Luo Date: Thu, 6 Jun 2024 17:15:53 +0800 Subject: [PATCH 064/462] [PowerPC] Adjust operand order of ADDItoc to be consistent with other ADDI* nodes (#93642) Simultaneously, the `ADDItoc` machineinstr is generated in `PPCISelDAGToDAG::Select` so the pattern is not used and can be removed. --- llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp | 5 +- llvm/lib/Target/PowerPC/PPCFastISel.cpp | 16 ++- llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp | 11 +- llvm/lib/Target/PowerPC/PPCInstr64Bit.td | 6 +- llvm/lib/Target/PowerPC/PPCInstrInfo.td | 6 +- llvm/test/CodeGen/PowerPC/toc-data-common.ll | 136 +++++++++++++------ llvm/test/CodeGen/PowerPC/toc-data.ll | 18 +-- 7 files changed, 129 insertions(+), 69 deletions(-) diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp index f4e84ade3b5a..bc0ae7a32c05 100644 --- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -1079,13 +1079,13 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { assert(IsAIX && TM.getCodeModel() == CodeModel::Small && "PseudoOp only valid for small code model AIX"); - // Transform %rN = ADDItoc/8 @op1, %r2. + // Transform %rN = ADDItoc/8 %r2, @op1. LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); // Change the opcode to load address. TmpInst.setOpcode((!IsPPC64) ? (PPC::LA) : (PPC::LA8)); - const MachineOperand &MO = MI->getOperand(1); + const MachineOperand &MO = MI->getOperand(2); assert(MO.isGlobal() && "Invalid operand for ADDItoc[8]."); // Map the operand to its corresponding MCSymbol. @@ -1094,7 +1094,6 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { const MCExpr *Exp = MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_None, OutContext); - TmpInst.getOperand(1) = TmpInst.getOperand(2); TmpInst.getOperand(2) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; diff --git a/llvm/lib/Target/PowerPC/PPCFastISel.cpp b/llvm/lib/Target/PowerPC/PPCFastISel.cpp index 735050641adf..a07954bd0d8b 100644 --- a/llvm/lib/Target/PowerPC/PPCFastISel.cpp +++ b/llvm/lib/Target/PowerPC/PPCFastISel.cpp @@ -2080,13 +2080,15 @@ unsigned PPCFastISel::PPCMaterializeGV(const GlobalValue *GV, MVT VT) { cast(GV)->hasAttribute("toc-data"); // For small code model, generate a simple TOC load. - if (CModel == CodeModel::Small) - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, - IsAIXTocData ? TII.get(PPC::ADDItoc8) : TII.get(PPC::LDtoc), - DestReg) - .addGlobalAddress(GV) - .addReg(PPC::X2); - else { + if (CModel == CodeModel::Small) { + auto MIB = BuildMI( + *FuncInfo.MBB, FuncInfo.InsertPt, MIMD, + IsAIXTocData ? TII.get(PPC::ADDItoc8) : TII.get(PPC::LDtoc), DestReg); + if (IsAIXTocData) + MIB.addReg(PPC::X2).addGlobalAddress(GV); + else + MIB.addGlobalAddress(GV).addReg(PPC::X2); + } else { // If the address is an externally defined symbol, a symbol with common // or externally available linkage, a non-local function address, or a // jump table address (not yet needed), or if we are generating code diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp index 275b3337a276..1a69d1e89313 100644 --- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp @@ -6102,8 +6102,15 @@ void PPCDAGToDAGISel::Select(SDNode *N) { EVT OperandTy) { SDValue GA = TocEntry->getOperand(0); SDValue TocBase = TocEntry->getOperand(1); - SDNode *MN = CurDAG->getMachineNode(OpCode, dl, OperandTy, GA, TocBase); - transferMemOperands(TocEntry, MN); + SDNode *MN = nullptr; + if (OpCode == PPC::ADDItoc || OpCode == PPC::ADDItoc8) + // toc-data access doesn't involve in loading from got, no need to + // keep memory operands. + MN = CurDAG->getMachineNode(OpCode, dl, OperandTy, TocBase, GA); + else { + MN = CurDAG->getMachineNode(OpCode, dl, OperandTy, GA, TocBase); + transferMemOperands(TocEntry, MN); + } ReplaceNode(TocEntry, MN); }; diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td index 9af8ada78376..eda5eb975e70 100644 --- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td +++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td @@ -1485,11 +1485,9 @@ def ADDItocL8: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, tocentry: } // Local Data Transform -def ADDItoc8 : PPCEmitTimePseudo<(outs g8rc:$rD), (ins tocentry:$disp, g8rc_nox0:$reg), +def ADDItoc8 : PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, tocentry:$disp), "#ADDItoc8", - [(set i64:$rD, - (PPCtoc_entry tglobaladdr:$disp, i64:$reg))]>, isPPC64; - + []>, isPPC64; let mayLoad = 1 in def LDtocL: PPCEmitTimePseudo<(outs g8rc:$rD), (ins tocentry:$disp, g8rc_nox0:$reg), "#LDtocL", []>, isPPC64; diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td index df6b2bf1a7b7..09f829943528 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -3345,10 +3345,8 @@ def LWZtocL : PPCEmitTimePseudo<(outs gprc:$rD), (ins tocentry32:$disp, gprc_nor def ADDIStocHA : PPCEmitTimePseudo<(outs gprc:$rD), (ins gprc_nor0:$reg, tocentry32:$disp), "#ADDIStocHA", []>; // TOC Data Transform on AIX -def ADDItoc : PPCEmitTimePseudo<(outs gprc:$rD), (ins tocentry32:$disp, gprc:$reg), - "#ADDItoc", - [(set i32:$rD, - (PPCtoc_entry tglobaladdr:$disp, i32:$reg))]>; +def ADDItoc : PPCEmitTimePseudo<(outs gprc:$rD), (ins gprc:$reg, tocentry32:$disp), + "#ADDItoc", []>; def ADDItocL : PPCEmitTimePseudo<(outs gprc:$rD), (ins gprc_nor0:$reg, tocentry32:$disp), "#ADDItocL", []>; diff --git a/llvm/test/CodeGen/PowerPC/toc-data-common.ll b/llvm/test/CodeGen/PowerPC/toc-data-common.ll index 7747f2eecc93..3b7ca44f5bbb 100644 --- a/llvm/test/CodeGen/PowerPC/toc-data-common.ll +++ b/llvm/test/CodeGen/PowerPC/toc-data-common.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple powerpc-ibm-aix-xcoff -verify-machineinstrs < %s | FileCheck %s -DINSTR=lwz --check-prefix=CHECK -; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -verify-machineinstrs < %s | FileCheck %s -DINSTR=ld --check-prefix=CHECK +; RUN: llc -mtriple powerpc-ibm-aix-xcoff -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK +; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-64 ; RUN: llc -filetype=obj -mtriple powerpc-ibm-aix-xcoff -verify-machineinstrs < %s -o %t32.o ; RUN: llvm-objdump -t --symbol-description %t32.o | FileCheck %s --check-prefix=OBJ32 @@ -15,16 +15,28 @@ define void @set(i32 noundef %_a) { ; CHECK-LABEL: set: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: la 4, a2[TD](2) -; CHECK-NEXT: la 5, a1[TD](2) -; CHECK-NEXT: stw 3, 0(4) -; CHECK-NEXT: [[INSTR]] 4, L..C0(2) # @a4 -; CHECK-NEXT: stw 3, 0(5) -; CHECK-NEXT: [[INSTR]] 5, L..C1(2) # @a3 -; CHECK-NEXT: stw 3, 0(4) -; CHECK-NEXT: stw 3, 0(5) -; CHECK-NEXT: blr +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: la 4, a2[TD](2) +; CHECK-NEXT: lwz 5, L..C0(2) # @a4 +; CHECK-NEXT: stw 3, 0(4) +; CHECK-NEXT: la 4, a1[TD](2) +; CHECK-NEXT: stw 3, 0(4) +; CHECK-NEXT: lwz 4, L..C1(2) # @a3 +; CHECK-NEXT: stw 3, 0(5) +; CHECK-NEXT: stw 3, 0(4) +; CHECK-NEXT: blr +; +; CHECK-64-LABEL: set: +; CHECK-64: # %bb.0: # %entry +; CHECK-64-NEXT: la 4, a2[TD](2) +; CHECK-64-NEXT: ld 5, L..C0(2) # @a4 +; CHECK-64-NEXT: stw 3, 0(4) +; CHECK-64-NEXT: la 4, a1[TD](2) +; CHECK-64-NEXT: stw 3, 0(4) +; CHECK-64-NEXT: ld 4, L..C1(2) # @a3 +; CHECK-64-NEXT: stw 3, 0(5) +; CHECK-64-NEXT: stw 3, 0(4) +; CHECK-64-NEXT: blr entry: store i32 %_a, ptr @a2, align 4 store i32 %_a, ptr @a1, align 4 @@ -35,10 +47,16 @@ ret void define i32 @get1() { ; CHECK-LABEL: get1: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: la 3, a2[TD](2) -; CHECK-NEXT: lwz 3, 0(3) -; CHECK-NEXT: blr +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: la 3, a2[TD](2) +; CHECK-NEXT: lwz 3, 0(3) +; CHECK-NEXT: blr +; +; CHECK-64-LABEL: get1: +; CHECK-64: # %bb.0: # %entry +; CHECK-64-NEXT: la 3, a2[TD](2) +; CHECK-64-NEXT: lwz 3, 0(3) +; CHECK-64-NEXT: blr entry: %0 = load i32, ptr @a2, align 4 ret i32 %0 @@ -46,10 +64,16 @@ ret i32 %0 define i32 @get2() { ; CHECK-LABEL: get2: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: la 3, a1[TD](2) -; CHECK-NEXT: lwz 3, 0(3) -; CHECK-NEXT: blr +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: la 3, a1[TD](2) +; CHECK-NEXT: lwz 3, 0(3) +; CHECK-NEXT: blr +; +; CHECK-64-LABEL: get2: +; CHECK-64: # %bb.0: # %entry +; CHECK-64-NEXT: la 3, a1[TD](2) +; CHECK-64-NEXT: lwz 3, 0(3) +; CHECK-64-NEXT: blr entry: %0 = load i32, ptr @a1, align 4 ret i32 %0 @@ -57,10 +81,16 @@ ret i32 %0 define i32 @get3() { ; CHECK-LABEL: get3: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: [[INSTR]] 3, L..C0(2) # @a4 -; CHECK-NEXT: lwz 3, 0(3) -; CHECK-NEXT: blr +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lwz 3, L..C0(2) # @a4 +; CHECK-NEXT: lwz 3, 0(3) +; CHECK-NEXT: blr +; +; CHECK-64-LABEL: get3: +; CHECK-64: # %bb.0: # %entry +; CHECK-64-NEXT: ld 3, L..C0(2) # @a4 +; CHECK-64-NEXT: lwz 3, 0(3) +; CHECK-64-NEXT: blr entry: %0 = load i32, ptr @a4, align 4 ret i32 %0 @@ -68,10 +98,16 @@ ret i32 %0 define i32 @get4() { ; CHECK-LABEL: get4: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: [[INSTR]] 3, L..C1(2) # @a3 -; CHECK-NEXT: lwz 3, 0(3) -; CHECK-NEXT: blr +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lwz 3, L..C1(2) # @a3 +; CHECK-NEXT: lwz 3, 0(3) +; CHECK-NEXT: blr +; +; CHECK-64-LABEL: get4: +; CHECK-64: # %bb.0: # %entry +; CHECK-64-NEXT: ld 3, L..C1(2) # @a3 +; CHECK-64-NEXT: lwz 3, 0(3) +; CHECK-64-NEXT: blr entry: %0 = load i32, ptr @a3, align 4 ret i32 %0 @@ -79,36 +115,56 @@ ret i32 %0 define nonnull ptr @escape1() { ; CHECK-LABEL: escape1: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: la 3, a2[TD](2) -; CHECK-NEXT: blr +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: la 3, a2[TD](2) +; CHECK-NEXT: blr +; +; CHECK-64-LABEL: escape1: +; CHECK-64: # %bb.0: # %entry +; CHECK-64-NEXT: la 3, a2[TD](2) +; CHECK-64-NEXT: blr entry: ret ptr @a2 } define nonnull ptr @escape2() { ; CHECK-LABEL: escape2: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: la 3, a1[TD](2) -; CHECK-NEXT: blr +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: la 3, a1[TD](2) +; CHECK-NEXT: blr +; +; CHECK-64-LABEL: escape2: +; CHECK-64: # %bb.0: # %entry +; CHECK-64-NEXT: la 3, a1[TD](2) +; CHECK-64-NEXT: blr entry: ret ptr @a1 } define nonnull ptr @escape3() { ; CHECK-LABEL: escape3: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: [[INSTR]] 3, L..C0(2) # @a4 -; CHECK-NEXT: blr +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lwz 3, L..C0(2) # @a4 +; CHECK-NEXT: blr +; +; CHECK-64-LABEL: escape3: +; CHECK-64: # %bb.0: # %entry +; CHECK-64-NEXT: ld 3, L..C0(2) # @a4 +; CHECK-64-NEXT: blr entry: ret ptr @a4 } define nonnull ptr @escape4() { ; CHECK-LABEL: escape4: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: [[INSTR]] 3, L..C1(2) # @a3 -; CHECK-NEXT: blr +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lwz 3, L..C1(2) # @a3 +; CHECK-NEXT: blr +; +; CHECK-64-LABEL: escape4: +; CHECK-64: # %bb.0: # %entry +; CHECK-64-NEXT: ld 3, L..C1(2) # @a3 +; CHECK-64-NEXT: blr entry: ret ptr @a3 } diff --git a/llvm/test/CodeGen/PowerPC/toc-data.ll b/llvm/test/CodeGen/PowerPC/toc-data.ll index 12286657488d..ee1dde190bb2 100644 --- a/llvm/test/CodeGen/PowerPC/toc-data.ll +++ b/llvm/test/CodeGen/PowerPC/toc-data.ll @@ -36,7 +36,7 @@ define dso_local void @write_int(i32 signext %in) { ret void } ; CHECK32: name: write_int -; CHECK32: %[[SCRATCH:[0-9]+]]:gprc_and_gprc_nor0 = ADDItoc @i, $r2 +; CHECK32: %[[SCRATCH:[0-9]+]]:gprc_and_gprc_nor0 = ADDItoc $r2, @i ; CHECK32-NEXT: STW %{{[0-9]+}}, 0, killed %[[SCRATCH]] :: (store (s32) into @i) ; TEST32: .write_int: @@ -44,12 +44,12 @@ define dso_local void @write_int(i32 signext %in) { ; TEST32-NEXT: stw 3, 0(4) ; CHECK64: name: write_int -; CHECK64: %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 @i, $x2 +; CHECK64: %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 $x2, @i ; CHECK64-NEXT: STW8 %{{[0-9]+}}, 0, killed %[[SCRATCH]] :: (store (s32) into @i) ; CHECK64-NOOPT: name: write_int ; CHECK64-NOOPT: %[[SUBREG:[0-9]+]]:gprc = COPY %{{[0-9]}}.sub_32 -; CHECK64-NOOPT: %[[ADDR:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 @i, $x2 +; CHECK64-NOOPT: %[[ADDR:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 $x2, @i ; CHECK64-NOOPT: STW %[[SUBREG]], 0, %[[ADDR]] ; TEST64: .write_int: @@ -128,7 +128,7 @@ define dso_local float @read_float() { ret float %0 } ; CHECK32: name: read_float -; CHECK32: %[[SCRATCH:[0-9]+]]:gprc_and_gprc_nor0 = ADDItoc @f, $r2 +; CHECK32: %[[SCRATCH:[0-9]+]]:gprc_and_gprc_nor0 = ADDItoc $r2, @f ; CHECK32: %{{[0-9]+}}:f4rc = LFS 0, killed %[[SCRATCH]] :: (dereferenceable load (s32) from @f) ; TEST32: .read_float: @@ -136,11 +136,11 @@ define dso_local float @read_float() { ; TEST32-NEXT: lfs 1, 0(3) ; CHECK64: name: read_float -; CHECK64: %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 @f, $x2 +; CHECK64: %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 $x2, @f ; CHECK64: %{{[0-9]+}}:f4rc = LFS 0, killed %[[SCRATCH]] :: (dereferenceable load (s32) from @f) ; CHECK64-NOOPT: name: read_float -; CHECK64-NOOPT: %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 @f, $x2 +; CHECK64-NOOPT: %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 $x2, @f ; CHECK64-NOOPT: %{{[0-9]+}}:f4rc = LFS 0, %[[SCRATCH]] ; TEST64: .read_float: @@ -217,18 +217,18 @@ define dso_local nonnull ptr @addr() { ret ptr @i } ; CHECK32: name: addr -; CHECK32: %[[SCRATCH:[0-9]+]]:gprc = ADDItoc @i, $r2 +; CHECK32: %[[SCRATCH:[0-9]+]]:gprc = ADDItoc $r2, @i ; CHECK32-NEXT: $r3 = COPY %[[SCRATCH]] ; TEST32: .addr ; TEST32: la 3, i[TD](2) ; CHECK64: name: addr -; CHECK64: %[[SCRATCH:[0-9]+]]:g8rc = ADDItoc8 @i, $x2 +; CHECK64: %[[SCRATCH:[0-9]+]]:g8rc = ADDItoc8 $x2, @i ; CHECK64-NEXT: $x3 = COPY %[[SCRATCH]] ; CHECK64-NOOPT: name: addr -; CHECK64-NOOPT: %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 @i, $x2 +; CHECK64-NOOPT: %[[SCRATCH:[0-9]+]]:g8rc_and_g8rc_nox0 = ADDItoc8 $x2, @i ; CHECK64-NOOPT: $x3 = COPY %[[SCRATCH]] ; TEST64: .addr -- GitLab From a86c1e7175d4acd8357326184bf4f88c8192676f Mon Sep 17 00:00:00 2001 From: Timm Baeder Date: Thu, 6 Jun 2024 11:17:48 +0200 Subject: [PATCH 065/462] [clang][Interp] Member Pointers (#91303) This adds a `MemberPointer` class along with a `PT_MemberPtr` primitive type. A `MemberPointer` has a `Pointer` Base as well as a `Decl*` (could be `ValueDecl*`?) decl it points to. For the actual logic, this mainly changes the way we handle `PtrMemOp`s in `VisitBinaryOperator`. --- clang/lib/AST/CMakeLists.txt | 1 + clang/lib/AST/Interp/ByteCodeExprGen.cpp | 100 ++++++++- clang/lib/AST/Interp/Context.cpp | 15 +- clang/lib/AST/Interp/Context.h | 2 + clang/lib/AST/Interp/Descriptor.cpp | 1 + clang/lib/AST/Interp/Disasm.cpp | 3 + clang/lib/AST/Interp/Interp.cpp | 30 ++- clang/lib/AST/Interp/Interp.h | 99 +++++++++ clang/lib/AST/Interp/InterpFrame.cpp | 1 + clang/lib/AST/Interp/InterpStack.cpp | 1 + clang/lib/AST/Interp/InterpStack.h | 3 + clang/lib/AST/Interp/MemberPointer.cpp | 76 +++++++ clang/lib/AST/Interp/MemberPointer.h | 112 ++++++++++ clang/lib/AST/Interp/Opcodes.td | 18 +- clang/lib/AST/Interp/Pointer.cpp | 1 + clang/lib/AST/Interp/Pointer.h | 1 + clang/lib/AST/Interp/PrimType.cpp | 1 + clang/lib/AST/Interp/PrimType.h | 8 +- clang/test/AST/Interp/eval-order.cpp | 4 +- clang/test/AST/Interp/literals.cpp | 7 +- clang/test/AST/Interp/memberpointers.cpp | 197 ++++++++++++++++++ .../CodeGenCXX/pointers-to-data-members.cpp | 1 + clang/test/SemaCXX/attr-weak.cpp | 1 + .../SemaCXX/nullptr_in_arithmetic_ops.cpp | 1 + clang/unittests/AST/Interp/toAPValue.cpp | 46 ++++ 25 files changed, 705 insertions(+), 25 deletions(-) create mode 100644 clang/lib/AST/Interp/MemberPointer.cpp create mode 100644 clang/lib/AST/Interp/MemberPointer.h create mode 100644 clang/test/AST/Interp/memberpointers.cpp diff --git a/clang/lib/AST/CMakeLists.txt b/clang/lib/AST/CMakeLists.txt index 3faefb54f599..a5d3dacfc1a8 100644 --- a/clang/lib/AST/CMakeLists.txt +++ b/clang/lib/AST/CMakeLists.txt @@ -87,6 +87,7 @@ add_clang_library(clangAST Interp/Record.cpp Interp/Source.cpp Interp/State.cpp + Interp/MemberPointer.cpp Interp/InterpShared.cpp ItaniumCXXABI.cpp ItaniumMangle.cpp diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index ea259639dace..d124248a3605 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -100,6 +100,35 @@ bool ByteCodeExprGen::VisitCastExpr(const CastExpr *CE) { return this->emitMemcpy(CE); } + case CK_DerivedToBaseMemberPointer: { + assert(classifyPrim(CE->getType()) == PT_MemberPtr); + assert(classifyPrim(SubExpr->getType()) == PT_MemberPtr); + const auto *FromMP = SubExpr->getType()->getAs(); + const auto *ToMP = CE->getType()->getAs(); + + unsigned DerivedOffset = collectBaseOffset(QualType(ToMP->getClass(), 0), + QualType(FromMP->getClass(), 0)); + + if (!this->visit(SubExpr)) + return false; + + return this->emitGetMemberPtrBasePop(DerivedOffset, CE); + } + + case CK_BaseToDerivedMemberPointer: { + assert(classifyPrim(CE) == PT_MemberPtr); + assert(classifyPrim(SubExpr) == PT_MemberPtr); + const auto *FromMP = SubExpr->getType()->getAs(); + const auto *ToMP = CE->getType()->getAs(); + + unsigned DerivedOffset = collectBaseOffset(QualType(FromMP->getClass(), 0), + QualType(ToMP->getClass(), 0)); + + if (!this->visit(SubExpr)) + return false; + return this->emitGetMemberPtrBasePop(-DerivedOffset, CE); + } + case CK_UncheckedDerivedToBase: case CK_DerivedToBase: { if (!this->visit(SubExpr)) @@ -187,7 +216,8 @@ bool ByteCodeExprGen::VisitCastExpr(const CastExpr *CE) { return this->emitCastFloatingIntegral(*ToT, CE); } - case CK_NullToPointer: { + case CK_NullToPointer: + case CK_NullToMemberPointer: { if (DiscardResult) return true; @@ -326,7 +356,8 @@ bool ByteCodeExprGen::VisitCastExpr(const CastExpr *CE) { return this->emitCast(*FromT, *ToT, CE); } - case CK_PointerToBoolean: { + case CK_PointerToBoolean: + case CK_MemberPointerToBoolean: { PrimType PtrT = classifyPrim(SubExpr->getType()); // Just emit p != nullptr for this. @@ -534,8 +565,23 @@ bool ByteCodeExprGen::VisitBinaryOperator(const BinaryOperator *BO) { BO->isComparisonOp()) return this->emitComplexComparison(LHS, RHS, BO); - if (BO->isPtrMemOp()) - return this->visit(RHS); + if (BO->isPtrMemOp()) { + if (!this->visit(LHS)) + return false; + + if (!this->visit(RHS)) + return false; + + if (!this->emitToMemberPtr(BO)) + return false; + + if (classifyPrim(BO) == PT_MemberPtr) + return true; + + if (!this->emitCastMemberPtrPtr(BO)) + return false; + return DiscardResult ? this->emitPopPtr(BO) : true; + } // Typecheck the args. std::optional LT = classify(LHS->getType()); @@ -2773,6 +2819,8 @@ bool ByteCodeExprGen::visitZeroInitializer(PrimType T, QualType QT, return this->emitNullPtr(nullptr, E); case PT_FnPtr: return this->emitNullFnPtr(nullptr, E); + case PT_MemberPtr: + return this->emitNullMemberPtr(nullptr, E); case PT_Float: { return this->emitConstFloat(APFloat::getZero(Ctx.getFloatSemantics(QT)), E); } @@ -2875,6 +2923,7 @@ bool ByteCodeExprGen::emitConst(T Value, PrimType Ty, const Expr *E) { return this->emitConstBool(Value, E); case PT_Ptr: case PT_FnPtr: + case PT_MemberPtr: case PT_Float: case PT_IntAP: case PT_IntAPS: @@ -3308,10 +3357,27 @@ bool ByteCodeExprGen::VisitCallExpr(const CallExpr *E) { } } + std::optional CalleeOffset; // Add the (optional, implicit) This pointer. if (const auto *MC = dyn_cast(E)) { - if (!this->visit(MC->getImplicitObjectArgument())) + if (!FuncDecl && classifyPrim(E->getCallee()) == PT_MemberPtr) { + // If we end up creating a CallPtr op for this, we need the base of the + // member pointer as the instance pointer, and later extract the function + // decl as the function pointer. + const Expr *Callee = E->getCallee(); + CalleeOffset = + this->allocateLocalPrimitive(Callee, PT_MemberPtr, true, false); + if (!this->visit(Callee)) + return false; + if (!this->emitSetLocal(PT_MemberPtr, *CalleeOffset, E)) + return false; + if (!this->emitGetLocal(PT_MemberPtr, *CalleeOffset, E)) + return false; + if (!this->emitGetMemberPtrBase(E)) + return false; + } else if (!this->visit(MC->getImplicitObjectArgument())) { return false; + } } llvm::BitVector NonNullArgs = collectNonNullArgs(FuncDecl, Args); @@ -3380,11 +3446,22 @@ bool ByteCodeExprGen::VisitCallExpr(const CallExpr *E) { for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I) ArgSize += align(primSize(classify(E->getArg(I)).value_or(PT_Ptr))); - if (!this->visit(E->getCallee())) - return false; + // Get the callee, either from a member pointer saved in CalleeOffset, + // or by just visiting the Callee expr. + if (CalleeOffset) { + if (!this->emitGetLocal(PT_MemberPtr, *CalleeOffset, E)) + return false; + if (!this->emitGetMemberPtrDecl(E)) + return false; + if (!this->emitCallPtr(ArgSize, E, E)) + return false; + } else { + if (!this->visit(E->getCallee())) + return false; - if (!this->emitCallPtr(ArgSize, E, E)) - return false; + if (!this->emitCallPtr(ArgSize, E, E)) + return false; + } } // Cleanup for discarded return values. @@ -3623,6 +3700,11 @@ bool ByteCodeExprGen::VisitUnaryOperator(const UnaryOperator *E) { return false; return DiscardResult ? this->emitPop(*T, E) : true; case UO_AddrOf: // &x + if (E->getType()->isMemberPointerType()) { + // C++11 [expr.unary.op]p3 has very strict rules on how the address of a + // member can be formed. + return this->emitGetMemberPtr(cast(SubExpr)->getDecl(), E); + } // We should already have a pointer when we get here. return this->delegate(SubExpr); case UO_Deref: // *x diff --git a/clang/lib/AST/Interp/Context.cpp b/clang/lib/AST/Interp/Context.cpp index b0b22b059b77..98d1837204eb 100644 --- a/clang/lib/AST/Interp/Context.cpp +++ b/clang/lib/AST/Interp/Context.cpp @@ -163,8 +163,12 @@ std::optional Context::classify(QualType T) const { if (T->isFloatingType()) return PT_Float; + if (T->isSpecificBuiltinType(BuiltinType::BoundMember) || + T->isMemberPointerType()) + return PT_MemberPtr; + if (T->isFunctionPointerType() || T->isFunctionReferenceType() || - T->isFunctionType() || T->isSpecificBuiltinType(BuiltinType::BoundMember)) + T->isFunctionType()) return PT_FnPtr; if (T->isReferenceType() || T->isPointerType() || @@ -177,9 +181,6 @@ std::optional Context::classify(QualType T) const { if (const auto *DT = dyn_cast(T)) return classify(DT->getUnderlyingType()); - if (const auto *DT = dyn_cast(T)) - return classify(DT->getPointeeType()); - return std::nullopt; } @@ -292,10 +293,12 @@ unsigned Context::collectBaseOffset(const RecordDecl *BaseDecl, } if (CurDecl == FinalDecl) break; - - // break; } assert(OffsetSum > 0); return OffsetSum; } + +const Record *Context::getRecord(const RecordDecl *D) const { + return P->getOrCreateRecord(D); +} diff --git a/clang/lib/AST/Interp/Context.h b/clang/lib/AST/Interp/Context.h index 360e9499d084..c78dc9a2a471 100644 --- a/clang/lib/AST/Interp/Context.h +++ b/clang/lib/AST/Interp/Context.h @@ -107,6 +107,8 @@ public: unsigned collectBaseOffset(const RecordDecl *BaseDecl, const RecordDecl *DerivedDecl) const; + const Record *getRecord(const RecordDecl *D) const; + private: /// Runs a function. bool Run(State &Parent, const Function *Func, APValue &Result); diff --git a/clang/lib/AST/Interp/Descriptor.cpp b/clang/lib/AST/Interp/Descriptor.cpp index 746b765ca421..d20ab1340c89 100644 --- a/clang/lib/AST/Interp/Descriptor.cpp +++ b/clang/lib/AST/Interp/Descriptor.cpp @@ -11,6 +11,7 @@ #include "Floating.h" #include "FunctionPointer.h" #include "IntegralAP.h" +#include "MemberPointer.h" #include "Pointer.h" #include "PrimType.h" #include "Record.h" diff --git a/clang/lib/AST/Interp/Disasm.cpp b/clang/lib/AST/Interp/Disasm.cpp index 3f8a92ed2f0b..0ab84d159c58 100644 --- a/clang/lib/AST/Interp/Disasm.cpp +++ b/clang/lib/AST/Interp/Disasm.cpp @@ -19,6 +19,7 @@ #include "Integral.h" #include "IntegralAP.h" #include "InterpFrame.h" +#include "MemberPointer.h" #include "Opcode.h" #include "PrimType.h" #include "Program.h" @@ -122,6 +123,8 @@ static const char *primTypeToString(PrimType T) { return "Ptr"; case PT_FnPtr: return "FnPtr"; + case PT_MemberPtr: + return "MemberPtr"; } llvm_unreachable("Unhandled PrimType"); } diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp index 145fa65791da..41bbaf83b11c 100644 --- a/clang/lib/AST/Interp/Interp.cpp +++ b/clang/lib/AST/Interp/Interp.cpp @@ -373,6 +373,26 @@ bool CheckSubobject(InterpState &S, CodePtr OpPC, const Pointer &Ptr, return false; } +bool CheckDowncast(InterpState &S, CodePtr OpPC, const Pointer &Ptr, + uint32_t Offset) { + uint32_t MinOffset = Ptr.getDeclDesc()->getMetadataSize(); + uint32_t PtrOffset = Ptr.getByteOffset(); + + // We subtract Offset from PtrOffset. The result must be at least + // MinOffset. + if (Offset < PtrOffset && (PtrOffset - Offset) >= MinOffset) + return true; + + const auto *E = cast(S.Current->getExpr(OpPC)); + QualType TargetQT = E->getType()->getPointeeType(); + QualType MostDerivedQT = Ptr.getDeclPtr().getType(); + + S.CCEDiag(E, diag::note_constexpr_invalid_downcast) + << MostDerivedQT << TargetQT; + + return false; +} + bool CheckConst(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { assert(Ptr.isLive() && "Pointer is not live"); if (!Ptr.isConst()) @@ -493,10 +513,12 @@ bool CheckStore(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { bool CheckInvoke(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { if (!CheckLive(S, OpPC, Ptr, AK_MemberCall)) return false; - if (!CheckExtern(S, OpPC, Ptr)) - return false; - if (!CheckRange(S, OpPC, Ptr, AK_MemberCall)) - return false; + if (!Ptr.isDummy()) { + if (!CheckExtern(S, OpPC, Ptr)) + return false; + if (!CheckRange(S, OpPC, Ptr, AK_MemberCall)) + return false; + } return true; } diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h index eca1792e6471..17f05548a190 100644 --- a/clang/lib/AST/Interp/Interp.h +++ b/clang/lib/AST/Interp/Interp.h @@ -20,6 +20,7 @@ #include "InterpFrame.h" #include "InterpStack.h" #include "InterpState.h" +#include "MemberPointer.h" #include "Opcode.h" #include "PrimType.h" #include "Program.h" @@ -75,6 +76,11 @@ bool CheckRange(InterpState &S, CodePtr OpPC, const Pointer &Ptr, bool CheckSubobject(InterpState &S, CodePtr OpPC, const Pointer &Ptr, CheckSubobjectKind CSK); +/// Checks if the dowcast using the given offset is possible with the given +/// pointer. +bool CheckDowncast(InterpState &S, CodePtr OpPC, const Pointer &Ptr, + uint32_t Offset); + /// Checks if a pointer points to const storage. bool CheckConst(InterpState &S, CodePtr OpPC, const Pointer &Ptr); @@ -725,6 +731,9 @@ using CompareFn = llvm::function_ref; template bool CmpHelper(InterpState &S, CodePtr OpPC, CompareFn Fn) { + assert((!std::is_same_v) && + "Non-equality comparisons on member pointer types should already be " + "rejected in Sema."); using BoolT = PrimConv::T; const T &RHS = S.Stk.pop(); const T &LHS = S.Stk.pop(); @@ -834,6 +843,47 @@ inline bool CmpHelperEQ(InterpState &S, CodePtr OpPC, CompareFn Fn) { } } +template <> +inline bool CmpHelperEQ(InterpState &S, CodePtr OpPC, + CompareFn Fn) { + const auto &RHS = S.Stk.pop(); + const auto &LHS = S.Stk.pop(); + + // If either operand is a pointer to a weak function, the comparison is not + // constant. + for (const auto &MP : {LHS, RHS}) { + if (const CXXMethodDecl *MD = MP.getMemberFunction(); MD && MD->isWeak()) { + const SourceInfo &Loc = S.Current->getSource(OpPC); + S.FFDiag(Loc, diag::note_constexpr_mem_pointer_weak_comparison) << MD; + return false; + } + } + + // C++11 [expr.eq]p2: + // If both operands are null, they compare equal. Otherwise if only one is + // null, they compare unequal. + if (LHS.isZero() && RHS.isZero()) { + S.Stk.push(Fn(ComparisonCategoryResult::Equal)); + return true; + } + if (LHS.isZero() || RHS.isZero()) { + S.Stk.push(Fn(ComparisonCategoryResult::Unordered)); + return true; + } + + // We cannot compare against virtual declarations at compile time. + for (const auto &MP : {LHS, RHS}) { + if (const CXXMethodDecl *MD = MP.getMemberFunction(); + MD && MD->isVirtual()) { + const SourceInfo &Loc = S.Current->getSource(OpPC); + S.CCEDiag(Loc, diag::note_constexpr_compare_virtual_mem_ptr) << MD; + } + } + + S.Stk.push(Boolean::from(Fn(LHS.compare(RHS)))); + return true; +} + template ::T> bool EQ(InterpState &S, CodePtr OpPC) { return CmpHelperEQ(S, OpPC, [](ComparisonCategoryResult R) { @@ -1300,6 +1350,9 @@ inline bool GetPtrDerivedPop(InterpState &S, CodePtr OpPC, uint32_t Off) { return false; if (!CheckSubobject(S, OpPC, Ptr, CSK_Derived)) return false; + if (!CheckDowncast(S, OpPC, Ptr, Off)) + return false; + S.Stk.push(Ptr.atFieldSub(Off)); return true; } @@ -1324,6 +1377,12 @@ inline bool GetPtrBasePop(InterpState &S, CodePtr OpPC, uint32_t Off) { return true; } +inline bool GetMemberPtrBasePop(InterpState &S, CodePtr OpPC, int32_t Off) { + const auto &Ptr = S.Stk.pop(); + S.Stk.push(Ptr.atInstanceBase(Off)); + return true; +} + inline bool GetPtrThisBase(InterpState &S, CodePtr OpPC, uint32_t Off) { if (S.checkingPotentialConstantExpression()) return false; @@ -1532,6 +1591,24 @@ inline bool Memcpy(InterpState &S, CodePtr OpPC) { return DoMemcpy(S, OpPC, Src, Dest); } +inline bool ToMemberPtr(InterpState &S, CodePtr OpPC) { + const auto &Member = S.Stk.pop(); + const auto &Base = S.Stk.pop(); + + S.Stk.push(Member.takeInstance(Base)); + return true; +} + +inline bool CastMemberPtrPtr(InterpState &S, CodePtr OpPC) { + const auto &MP = S.Stk.pop(); + + if (std::optional Ptr = MP.toPointer(S.Ctx)) { + S.Stk.push(*Ptr); + return true; + } + return false; +} + //===----------------------------------------------------------------------===// // AddOffset, SubOffset //===----------------------------------------------------------------------===// @@ -2329,6 +2406,28 @@ inline bool GetIntPtr(InterpState &S, CodePtr OpPC, const Descriptor *Desc) { return true; } +inline bool GetMemberPtr(InterpState &S, CodePtr OpPC, const Decl *D) { + S.Stk.push(D); + return true; +} + +inline bool GetMemberPtrBase(InterpState &S, CodePtr OpPC) { + const auto &MP = S.Stk.pop(); + + S.Stk.push(MP.getBase()); + return true; +} + +inline bool GetMemberPtrDecl(InterpState &S, CodePtr OpPC) { + const auto &MP = S.Stk.pop(); + + const auto *FD = cast(MP.getDecl()); + const auto *Func = S.getContext().getOrCreateFunction(FD); + + S.Stk.push(Func); + return true; +} + /// Just emit a diagnostic. The expression that caused emission of this /// op is not valid in a constant context. inline bool Invalid(InterpState &S, CodePtr OpPC) { diff --git a/clang/lib/AST/Interp/InterpFrame.cpp b/clang/lib/AST/Interp/InterpFrame.cpp index 51b0bd5c1551..54ccf9034c7a 100644 --- a/clang/lib/AST/Interp/InterpFrame.cpp +++ b/clang/lib/AST/Interp/InterpFrame.cpp @@ -12,6 +12,7 @@ #include "Function.h" #include "InterpStack.h" #include "InterpState.h" +#include "MemberPointer.h" #include "Pointer.h" #include "PrimType.h" #include "Program.h" diff --git a/clang/lib/AST/Interp/InterpStack.cpp b/clang/lib/AST/Interp/InterpStack.cpp index 91fe40feb767..c7024740d322 100644 --- a/clang/lib/AST/Interp/InterpStack.cpp +++ b/clang/lib/AST/Interp/InterpStack.cpp @@ -10,6 +10,7 @@ #include "Boolean.h" #include "Floating.h" #include "Integral.h" +#include "MemberPointer.h" #include "Pointer.h" #include #include diff --git a/clang/lib/AST/Interp/InterpStack.h b/clang/lib/AST/Interp/InterpStack.h index 3fd0f63c781f..9d85503b851b 100644 --- a/clang/lib/AST/Interp/InterpStack.h +++ b/clang/lib/AST/Interp/InterpStack.h @@ -15,6 +15,7 @@ #include "FunctionPointer.h" #include "IntegralAP.h" +#include "MemberPointer.h" #include "PrimType.h" #include #include @@ -188,6 +189,8 @@ private: return PT_IntAP; else if constexpr (std::is_same_v>) return PT_IntAP; + else if constexpr (std::is_same_v) + return PT_MemberPtr; llvm_unreachable("unknown type push()'ed into InterpStack"); } diff --git a/clang/lib/AST/Interp/MemberPointer.cpp b/clang/lib/AST/Interp/MemberPointer.cpp new file mode 100644 index 000000000000..96f63643e83c --- /dev/null +++ b/clang/lib/AST/Interp/MemberPointer.cpp @@ -0,0 +1,76 @@ +//===------------------------- MemberPointer.cpp ----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "MemberPointer.h" +#include "Context.h" +#include "FunctionPointer.h" +#include "Program.h" +#include "Record.h" + +namespace clang { +namespace interp { + +std::optional MemberPointer::toPointer(const Context &Ctx) const { + if (!Dcl || isa(Dcl)) + return Base; + const FieldDecl *FD = cast(Dcl); + assert(FD); + + if (!Base.isBlockPointer()) + return std::nullopt; + + Pointer CastedBase = + (PtrOffset < 0 ? Base.atField(-PtrOffset) : Base.atFieldSub(PtrOffset)); + + const Record *BaseRecord = CastedBase.getRecord(); + if (!BaseRecord) + return std::nullopt; + + assert(BaseRecord); + if (FD->getParent() == BaseRecord->getDecl()) + return CastedBase.atField(BaseRecord->getField(FD)->Offset); + + const RecordDecl *FieldParent = FD->getParent(); + const Record *FieldRecord = Ctx.getRecord(FieldParent); + + unsigned Offset = 0; + Offset += FieldRecord->getField(FD)->Offset; + Offset += CastedBase.block()->getDescriptor()->getMetadataSize(); + + if (Offset > CastedBase.block()->getSize()) + return std::nullopt; + + if (const RecordDecl *BaseDecl = Base.getDeclPtr().getRecord()->getDecl(); + BaseDecl != FieldParent) + Offset += Ctx.collectBaseOffset(FieldParent, BaseDecl); + + if (Offset > CastedBase.block()->getSize()) + return std::nullopt; + + assert(Offset <= CastedBase.block()->getSize()); + return Pointer(const_cast(Base.block()), Offset, Offset); +} + +FunctionPointer MemberPointer::toFunctionPointer(const Context &Ctx) const { + return FunctionPointer(Ctx.getProgram().getFunction(cast(Dcl))); +} + +APValue MemberPointer::toAPValue() const { + if (isZero()) + return APValue(static_cast(nullptr), /*IsDerivedMember=*/false, + /*Path=*/{}); + + if (hasBase()) + return Base.toAPValue(); + + return APValue(cast(getDecl()), /*IsDerivedMember=*/false, + /*Path=*/{}); +} + +} // namespace interp +} // namespace clang diff --git a/clang/lib/AST/Interp/MemberPointer.h b/clang/lib/AST/Interp/MemberPointer.h new file mode 100644 index 000000000000..5c61f6a43957 --- /dev/null +++ b/clang/lib/AST/Interp/MemberPointer.h @@ -0,0 +1,112 @@ +//===------------------------- MemberPointer.h ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_AST_INTERP_MEMBER_POINTER_H +#define LLVM_CLANG_AST_INTERP_MEMBER_POINTER_H + +#include "Pointer.h" +#include + +namespace clang { +class ASTContext; +namespace interp { + +class Context; +class FunctionPointer; + +class MemberPointer final { +private: + Pointer Base; + const Decl *Dcl = nullptr; + int32_t PtrOffset = 0; + + MemberPointer(Pointer Base, const Decl *Dcl, int32_t PtrOffset) + : Base(Base), Dcl(Dcl), PtrOffset(PtrOffset) {} + +public: + MemberPointer() = default; + MemberPointer(Pointer Base, const Decl *Dcl) : Base(Base), Dcl(Dcl) {} + MemberPointer(uint32_t Address, const Descriptor *D) { + // We only reach this for Address == 0, when creating a null member pointer. + assert(Address == 0); + } + + MemberPointer(const Decl *D) : Dcl(D) { + assert((isa(D))); + } + + uint64_t getIntegerRepresentation() const { + assert( + false && + "getIntegerRepresentation() shouldn't be reachable for MemberPointers"); + return 17; + } + + std::optional toPointer(const Context &Ctx) const; + + FunctionPointer toFunctionPointer(const Context &Ctx) const; + + Pointer getBase() const { + if (PtrOffset < 0) + return Base.atField(-PtrOffset); + return Base.atFieldSub(PtrOffset); + } + bool isMemberFunctionPointer() const { + return isa_and_nonnull(Dcl); + } + const CXXMethodDecl *getMemberFunction() const { + return dyn_cast_if_present(Dcl); + } + const FieldDecl *getField() const { + return dyn_cast_if_present(Dcl); + } + + bool hasDecl() const { return Dcl; } + const Decl *getDecl() const { return Dcl; } + + MemberPointer atInstanceBase(unsigned Offset) const { + if (Base.isZero()) + return MemberPointer(Base, Dcl, Offset); + return MemberPointer(this->Base, Dcl, Offset + PtrOffset); + } + + MemberPointer takeInstance(Pointer Instance) const { + assert(this->Base.isZero()); + return MemberPointer(Instance, this->Dcl, this->PtrOffset); + } + + APValue toAPValue() const; + + bool isZero() const { return Base.isZero() && !Dcl; } + bool hasBase() const { return !Base.isZero(); } + + void print(llvm::raw_ostream &OS) const { + OS << "MemberPtr(" << Base << " " << (void *)Dcl << " + " << PtrOffset + << ")"; + } + + std::string toDiagnosticString(const ASTContext &Ctx) const { + return "FIXME"; + } + + ComparisonCategoryResult compare(const MemberPointer &RHS) const { + if (this->Dcl == RHS.Dcl) + return ComparisonCategoryResult::Equal; + return ComparisonCategoryResult::Unordered; + } +}; + +inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, MemberPointer FP) { + FP.print(OS); + return OS; +} + +} // namespace interp +} // namespace clang + +#endif diff --git a/clang/lib/AST/Interp/Opcodes.td b/clang/lib/AST/Interp/Opcodes.td index cfbd7f93c32d..cb4f299c8d51 100644 --- a/clang/lib/AST/Interp/Opcodes.td +++ b/clang/lib/AST/Interp/Opcodes.td @@ -30,6 +30,7 @@ def IntAPS : Type; def Float : Type; def Ptr : Type; def FnPtr : Type; +def MemberPtr : Type; //===----------------------------------------------------------------------===// // Types transferred to the interpreter. @@ -61,6 +62,7 @@ def ArgOffsetOfExpr : ArgType { let Name = "const OffsetOfExpr *"; } def ArgDeclRef : ArgType { let Name = "const DeclRefExpr *"; } def ArgDesc : ArgType { let Name = "const Descriptor *"; } def ArgCCI : ArgType { let Name = "const ComparisonCategoryInfo *"; } +def ArgDecl : ArgType { let Name = "const Decl*"; } //===----------------------------------------------------------------------===// // Classes of types instructions operate on. @@ -93,7 +95,7 @@ def AluTypeClass : TypeClass { } def PtrTypeClass : TypeClass { - let Types = [Ptr, FnPtr]; + let Types = [Ptr, FnPtr, MemberPtr]; } def BoolTypeClass : TypeClass { @@ -208,7 +210,6 @@ def CallBI : Opcode { def CallPtr : Opcode { let Args = [ArgUint32, ArgCallExpr]; - let Types = []; } def CallVar : Opcode { @@ -327,6 +328,11 @@ def GetPtrBasePop : Opcode { // Offset of field, which is a base. let Args = [ArgUint32]; } +def GetMemberPtrBasePop : Opcode { + // Offset of field, which is a base. + let Args = [ArgSint32]; +} + def FinishInitPop : Opcode; def FinishInit : Opcode; @@ -751,6 +757,14 @@ def CheckNonNullArg : Opcode { def Memcpy : Opcode; +def ToMemberPtr : Opcode; +def CastMemberPtrPtr : Opcode; +def GetMemberPtr : Opcode { + let Args = [ArgDecl]; +} +def GetMemberPtrBase : Opcode; +def GetMemberPtrDecl : Opcode; + //===----------------------------------------------------------------------===// // Debugging. //===----------------------------------------------------------------------===// diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp index 252f7ea46086..a60b4d28b438 100644 --- a/clang/lib/AST/Interp/Pointer.cpp +++ b/clang/lib/AST/Interp/Pointer.cpp @@ -13,6 +13,7 @@ #include "Function.h" #include "Integral.h" #include "InterpBlock.h" +#include "MemberPointer.h" #include "PrimType.h" #include "Record.h" diff --git a/clang/lib/AST/Interp/Pointer.h b/clang/lib/AST/Interp/Pointer.h index 93ca754d04a6..c6e4f4d0b4ab 100644 --- a/clang/lib/AST/Interp/Pointer.h +++ b/clang/lib/AST/Interp/Pointer.h @@ -620,6 +620,7 @@ public: private: friend class Block; friend class DeadBlock; + friend class MemberPointer; friend struct InitMap; Pointer(Block *Pointee, unsigned Base, uint64_t Offset); diff --git a/clang/lib/AST/Interp/PrimType.cpp b/clang/lib/AST/Interp/PrimType.cpp index 9b96dcfe6a27..3054e67d5c49 100644 --- a/clang/lib/AST/Interp/PrimType.cpp +++ b/clang/lib/AST/Interp/PrimType.cpp @@ -11,6 +11,7 @@ #include "Floating.h" #include "FunctionPointer.h" #include "IntegralAP.h" +#include "MemberPointer.h" #include "Pointer.h" using namespace clang; diff --git a/clang/lib/AST/Interp/PrimType.h b/clang/lib/AST/Interp/PrimType.h index 604fb5dfde1e..20fb5e81774d 100644 --- a/clang/lib/AST/Interp/PrimType.h +++ b/clang/lib/AST/Interp/PrimType.h @@ -25,6 +25,7 @@ class Pointer; class Boolean; class Floating; class FunctionPointer; +class MemberPointer; template class IntegralAP; template class Integral; @@ -44,10 +45,11 @@ enum PrimType : unsigned { PT_Float = 11, PT_Ptr = 12, PT_FnPtr = 13, + PT_MemberPtr = 14, }; inline constexpr bool isPtrType(PrimType T) { - return T == PT_Ptr || T == PT_FnPtr; + return T == PT_Ptr || T == PT_FnPtr || T == PT_MemberPtr; } enum class CastKind : uint8_t { @@ -91,6 +93,9 @@ template <> struct PrimConv { using T = Pointer; }; template <> struct PrimConv { using T = FunctionPointer; }; +template <> struct PrimConv { + using T = MemberPointer; +}; /// Returns the size of a primitive type in bytes. size_t primSize(PrimType Type); @@ -131,6 +136,7 @@ static inline bool aligned(const void *P) { TYPE_SWITCH_CASE(PT_Bool, B) \ TYPE_SWITCH_CASE(PT_Ptr, B) \ TYPE_SWITCH_CASE(PT_FnPtr, B) \ + TYPE_SWITCH_CASE(PT_MemberPtr, B) \ } \ } while (0) diff --git a/clang/test/AST/Interp/eval-order.cpp b/clang/test/AST/Interp/eval-order.cpp index aaf2b74510bb..7a7ce6a71460 100644 --- a/clang/test/AST/Interp/eval-order.cpp +++ b/clang/test/AST/Interp/eval-order.cpp @@ -71,8 +71,8 @@ namespace EvalOrder { // Rules 1 and 2 have no effect ('b' is not an expression). // Rule 3: a->*b - // SEQ(A(ud).*B(&UserDefined::n)); FIXME - // SEQ(A(&ud)->*B(&UserDefined::n)); FIXME + SEQ(A(ud).*B(&UserDefined::n)); + SEQ(A(&ud)->*B(&UserDefined::n)); // Rule 4: a(b1, b2, b3) SEQ(A(f)(B(1), B(2), B(3))); // expected-error {{not an integral constant expression}} FIXME \ diff --git a/clang/test/AST/Interp/literals.cpp b/clang/test/AST/Interp/literals.cpp index c160be06dd24..27b2af7db0da 100644 --- a/clang/test/AST/Interp/literals.cpp +++ b/clang/test/AST/Interp/literals.cpp @@ -66,7 +66,12 @@ namespace ScalarTypes { First = 0, }; static_assert(getScalar() == First, ""); - /// FIXME: Member pointers. + + struct S { + int v; + }; + constexpr int S::* MemberPtr = &S::v; + static_assert(getScalar() == nullptr, ""); #if __cplusplus >= 201402L constexpr void Void(int n) { diff --git a/clang/test/AST/Interp/memberpointers.cpp b/clang/test/AST/Interp/memberpointers.cpp new file mode 100644 index 000000000000..54d73fe86ca1 --- /dev/null +++ b/clang/test/AST/Interp/memberpointers.cpp @@ -0,0 +1,197 @@ +// RUN: %clang_cc1 -std=c++14 -fexperimental-new-constant-interpreter -verify=expected,both %s +// RUN: %clang_cc1 -std=c++14 -verify=ref,both %s + +namespace MemberPointers { + struct A { + constexpr A(int n) : n(n) {} + int n; + constexpr int f() const { return n + 3; } + }; + + constexpr A a(7); + static_assert(A(5).*&A::n == 5, ""); + static_assert((&a)->*&A::n == 7, ""); + static_assert((A(8).*&A::f)() == 11, ""); + static_assert(((&a)->*&A::f)() == 10, ""); + + struct B : A { + constexpr B(int n, int m) : A(n), m(m) {} + int m; + constexpr int g() const { return n + m + 1; } + }; + constexpr B b(9, 13); + static_assert(B(4, 11).*&A::n == 4, ""); + static_assert(B(4, 11).*&B::m == 11, ""); + static_assert(B(4, 11).m == 11, ""); + static_assert(B(4, 11).*(int(A::*))&B::m == 11, ""); + static_assert(B(4, 11).*&B::m == 11, ""); + static_assert((&b)->*&A::n == 9, ""); + static_assert((&b)->*&B::m == 13, ""); + static_assert((&b)->*(int(A::*))&B::m == 13, ""); + static_assert((B(4, 11).*&A::f)() == 7, ""); + static_assert((B(4, 11).*&B::g)() == 16, ""); + + static_assert((B(4, 11).*(int(A::*)() const)&B::g)() == 16, ""); + + static_assert(((&b)->*&A::f)() == 12, ""); + static_assert(((&b)->*&B::g)() == 23, ""); + static_assert(((&b)->*(int(A::*)()const)&B::g)() == 23, ""); + + + struct S { + constexpr S(int m, int n, int (S::*pf)() const, int S::*pn) : + m(m), n(n), pf(pf), pn(pn) {} + constexpr S() : m(), n(), pf(&S::f), pn(&S::n) {} + + constexpr int f() const { return this->*pn; } + virtual int g() const; + + int m, n; + int (S::*pf)() const; + int S::*pn; + }; + + constexpr int S::*pm = &S::m; + constexpr int S::*pn = &S::n; + + constexpr int (S::*pf)() const = &S::f; + constexpr int (S::*pg)() const = &S::g; + + constexpr S s(2, 5, &S::f, &S::m); + + static_assert((s.*&S::f)() == 2, ""); + static_assert((s.*s.pf)() == 2, ""); + + static_assert(pf == &S::f, ""); + + static_assert(pf == s.*&S::pf, ""); + + static_assert(pm == &S::m, ""); + static_assert(pm != pn, ""); + static_assert(s.pn != pn, ""); + static_assert(s.pn == pm, ""); + static_assert(pg != nullptr, ""); + static_assert(pf != nullptr, ""); + static_assert((int S::*)nullptr == nullptr, ""); + static_assert(pg == pg, ""); // both-error {{constant expression}} \ + // both-note {{comparison of pointer to virtual member function 'g' has unspecified value}} + static_assert(pf != pg, ""); // both-error {{constant expression}} \ + // both-note {{comparison of pointer to virtual member function 'g' has unspecified value}} + + template struct T : T { const int X = n;}; + template<> struct T<0> { int n; char k;}; + template<> struct T<30> : T<29> { int m; }; + + T<17> t17; + T<30> t30; + + constexpr int (T<15>::*deepm) = (int(T<10>::*))&T<30>::m; + constexpr int (T<10>::*deepn) = &T<0>::n; + constexpr char (T<10>::*deepk) = &T<0>::k; + + static_assert(&(t17.*deepn) == &t17.n, ""); + static_assert(&(t17.*deepk) == &t17.k, ""); + static_assert(deepn == &T<2>::n, ""); + + constexpr int *pgood = &(t30.*deepm); + constexpr int *pbad = &(t17.*deepm); // both-error {{constant expression}} + static_assert(&(t30.*deepm) == &t30.m, ""); + + static_assert(deepm == &T<50>::m, ""); + static_assert(deepm != deepn, ""); + + constexpr T<5> *p17_5 = &t17; + constexpr T<13> *p17_13 = (T<13>*)p17_5; + constexpr T<23> *p17_23 = (T<23>*)p17_13; // both-error {{constant expression}} \ + // both-note {{cannot cast object of dynamic type 'T<17>' to type 'T<23>'}} + constexpr T<18> *p17_18 = (T<18>*)p17_13; // both-error {{constant expression}} \ + // both-note {{cannot cast object of dynamic type 'T<17>' to type 'T<18>'}} + static_assert(&(p17_5->*(int(T<0>::*))deepn) == &t17.n, ""); + static_assert(&(p17_5->*(int(T<0>::*))deepn), ""); + + + static_assert(&(p17_13->*deepn) == &t17.n, ""); + constexpr int *pbad2 = &(p17_13->*(int(T<9>::*))deepm); // both-error {{constant expression}} + + constexpr T<5> *p30_5 = &t30; + constexpr T<23> *p30_23 = (T<23>*)p30_5; + constexpr T<13> *p30_13 = p30_23; + static_assert(&(p30_13->*deepn) == &t30.n, ""); + static_assert(&(p30_23->*deepn) == &t30.n, ""); + static_assert(&(p30_5->*(int(T<3>::*))deepn) == &t30.n, ""); + + static_assert(&(p30_5->*(int(T<2>::*))deepm) == &t30.m, ""); + static_assert(&(((T<17>*)p30_13)->*deepm) == &t30.m, ""); + static_assert(&(p30_23->*deepm) == &t30.m, ""); + + + /// Added tests not from constant-expression-cxx11.cpp + static_assert(pm, ""); + static_assert(!((int S::*)nullptr), ""); + constexpr int S::*pk = nullptr; + static_assert(!pk, ""); +} + +namespace test3 { + struct nsCSSRect { + }; + static int nsCSSRect::* sides; + nsCSSRect dimenX; + void ParseBoxCornerRadii(int y) { + switch (y) { + } + int& x = dimenX.*sides; + } +} + +void foo() { + class X; + void (X::*d) (); + d = nullptr; /// This calls in the constant interpreter. +} + +namespace { + struct A { int n; }; + struct B { int n; }; + struct C : A, B {}; + struct D { double d; C c; }; + const int &&u = static_cast(0, ((D&&)D{}).*&D::c).n; // both-warning {{left operand of comma operator has no effect}} +} + +/// From SemaTemplate/instantiate-member-pointers.cpp +namespace { + struct Y { + int x; + }; + + template + struct X3 { + X3 &operator=(const T& value) { + return *this; + } + }; + + typedef int Y::*IntMember; + template + struct X4 { + X3 member; + int &getMember(Y& y) { return y.*Member; } + }; + + int &get_X4(X4<&Y::x> x4, Y& y) { + return x4.getMember(y); + } +} + +/// From test/CXX/basic/basic.def.odr/p2.cpp +namespace { + void use(int); + struct S { int x; int f() const; }; + constexpr S *ps = nullptr; + S *const &psr = ps; + + void test() { + use(ps->*&S::x); + use(psr->*&S::x); + } +} diff --git a/clang/test/CodeGenCXX/pointers-to-data-members.cpp b/clang/test/CodeGenCXX/pointers-to-data-members.cpp index 29f1c3f48e3a..cf1d6c018409 100644 --- a/clang/test/CodeGenCXX/pointers-to-data-members.cpp +++ b/clang/test/CodeGenCXX/pointers-to-data-members.cpp @@ -1,4 +1,5 @@ // RUN: %clang_cc1 %s -emit-llvm -o %t.ll -triple=x86_64-apple-darwin10 +// RUN: %clang_cc1 %s -emit-llvm -o %t.ll -triple=x86_64-apple-darwin10 -fexperimental-new-constant-interpreter // RUN: FileCheck %s < %t.ll // RUN: FileCheck -check-prefix=CHECK-GLOBAL %s < %t.ll diff --git a/clang/test/SemaCXX/attr-weak.cpp b/clang/test/SemaCXX/attr-weak.cpp index f065bfd9483f..0f9a2975e5f6 100644 --- a/clang/test/SemaCXX/attr-weak.cpp +++ b/clang/test/SemaCXX/attr-weak.cpp @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fsyntax-only -verify -std=c++11 %s +// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fsyntax-only -verify -std=c++11 %s -fexperimental-new-constant-interpreter static int test0 __attribute__((weak)); // expected-error {{weak declaration cannot have internal linkage}} static void test1() __attribute__((weak)); // expected-error {{weak declaration cannot have internal linkage}} diff --git a/clang/test/SemaCXX/nullptr_in_arithmetic_ops.cpp b/clang/test/SemaCXX/nullptr_in_arithmetic_ops.cpp index 6273d9c42e0b..98bec184164b 100644 --- a/clang/test/SemaCXX/nullptr_in_arithmetic_ops.cpp +++ b/clang/test/SemaCXX/nullptr_in_arithmetic_ops.cpp @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -fsyntax-only -Wno-tautological-pointer-compare -fblocks -std=c++11 -verify %s +// RUN: %clang_cc1 -fsyntax-only -Wno-tautological-pointer-compare -fblocks -std=c++11 -verify %s -fexperimental-new-constant-interpreter void foo() { int a; diff --git a/clang/unittests/AST/Interp/toAPValue.cpp b/clang/unittests/AST/Interp/toAPValue.cpp index e56453aba2c5..d6879d6e0bca 100644 --- a/clang/unittests/AST/Interp/toAPValue.cpp +++ b/clang/unittests/AST/Interp/toAPValue.cpp @@ -186,3 +186,49 @@ TEST(ToAPValue, FunctionPointersC) { ASSERT_EQ(I, 17); } } + +TEST(ToAPValue, MemberPointers) { + constexpr char Code[] = "struct S {\n" + " int m, n;\n" + "};\n" + "constexpr int S::*pm = &S::m;\n" + "constexpr int S::*nn = nullptr;\n"; + + auto AST = tooling::buildASTFromCodeWithArgs( + Code, {"-fexperimental-new-constant-interpreter"}); + + auto &Ctx = AST->getASTContext().getInterpContext(); + Program &Prog = Ctx.getProgram(); + + auto getDecl = [&](const char *Name) -> const ValueDecl * { + auto Nodes = + match(valueDecl(hasName(Name)).bind("var"), AST->getASTContext()); + assert(Nodes.size() == 1); + const auto *D = Nodes[0].getNodeAs("var"); + assert(D); + return D; + }; + + auto getGlobalPtr = [&](const char *Name) -> Pointer { + const VarDecl *D = cast(getDecl(Name)); + return Prog.getPtrGlobal(*Prog.getGlobal(D)); + }; + + { + const Pointer &GP = getGlobalPtr("pm"); + ASSERT_TRUE(GP.isLive()); + const MemberPointer &FP = GP.deref(); + APValue A = FP.toAPValue(); + ASSERT_EQ(A.getMemberPointerDecl(), getDecl("m")); + ASSERT_EQ(A.getKind(), APValue::MemberPointer); + } + + { + const Pointer &GP = getGlobalPtr("nn"); + ASSERT_TRUE(GP.isLive()); + const MemberPointer &NP = GP.deref(); + ASSERT_TRUE(NP.isZero()); + APValue A = NP.toAPValue(); + ASSERT_EQ(A.getKind(), APValue::MemberPointer); + } +} -- GitLab From f5d8c0ebfe9f382f404607db404353af5467cec1 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Thu, 6 Jun 2024 09:18:40 +0000 Subject: [PATCH 066/462] [gn build] Port a86c1e7175d4 --- llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn index 0cf99256c9bd..d3a3ee755808 100644 --- a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn @@ -110,6 +110,7 @@ static_library("AST") { "Interp/InterpShared.cpp", "Interp/InterpStack.cpp", "Interp/InterpState.cpp", + "Interp/MemberPointer.cpp", "Interp/Pointer.cpp", "Interp/PrimType.cpp", "Interp/Program.cpp", -- GitLab From 4c6dd70ec4fb7491eb59e3efa6d3080499a15d4d Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Thu, 6 Jun 2024 10:29:55 +0100 Subject: [PATCH 067/462] [AMDGPU] Move INIT_EXEC lowering from SILowerControlFlow to SIWholeQuadMode (#94452) NFCI; this just preserves SI_INIT_EXEC and SI_INIT_EXEC_FROM_INPUT instructions a little longer so that we can reliably identify them in SIWholeQuadMode. --- llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp | 103 ------------------ llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp | 103 +++++++++++++++++- 2 files changed, 102 insertions(+), 104 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp index f178324dbbe2..5dc3457b5bfa 100644 --- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp +++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp @@ -103,8 +103,6 @@ private: MachineBasicBlock *emitEndCf(MachineInstr &MI); - void lowerInitExec(MachineBasicBlock *MBB, MachineInstr &MI); - void findMaskOperands(MachineInstr &MI, unsigned OpNo, SmallVectorImpl &Src) const; @@ -709,95 +707,6 @@ MachineBasicBlock *SILowerControlFlow::process(MachineInstr &MI) { return SplitBB; } -void SILowerControlFlow::lowerInitExec(MachineBasicBlock *MBB, - MachineInstr &MI) { - MachineFunction &MF = *MBB->getParent(); - const GCNSubtarget &ST = MF.getSubtarget(); - bool IsWave32 = ST.isWave32(); - - if (MI.getOpcode() == AMDGPU::SI_INIT_EXEC) { - // This should be before all vector instructions. - MachineInstr *InitMI = BuildMI(*MBB, MBB->begin(), MI.getDebugLoc(), - TII->get(IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64), Exec) - .addImm(MI.getOperand(0).getImm()); - if (LIS) { - LIS->RemoveMachineInstrFromMaps(MI); - LIS->InsertMachineInstrInMaps(*InitMI); - } - MI.eraseFromParent(); - return; - } - - // Extract the thread count from an SGPR input and set EXEC accordingly. - // Since BFM can't shift by 64, handle that case with CMP + CMOV. - // - // S_BFE_U32 count, input, {shift, 7} - // S_BFM_B64 exec, count, 0 - // S_CMP_EQ_U32 count, 64 - // S_CMOV_B64 exec, -1 - Register InputReg = MI.getOperand(0).getReg(); - MachineInstr *FirstMI = &*MBB->begin(); - if (InputReg.isVirtual()) { - MachineInstr *DefInstr = MRI->getVRegDef(InputReg); - assert(DefInstr && DefInstr->isCopy()); - if (DefInstr->getParent() == MBB) { - if (DefInstr != FirstMI) { - // If the `InputReg` is defined in current block, we also need to - // move that instruction to the beginning of the block. - DefInstr->removeFromParent(); - MBB->insert(FirstMI, DefInstr); - if (LIS) - LIS->handleMove(*DefInstr); - } else { - // If first instruction is definition then move pointer after it. - FirstMI = &*std::next(FirstMI->getIterator()); - } - } - } - - // Insert instruction sequence at block beginning (before vector operations). - const DebugLoc DL = MI.getDebugLoc(); - const unsigned WavefrontSize = ST.getWavefrontSize(); - const unsigned Mask = (WavefrontSize << 1) - 1; - Register CountReg = MRI->createVirtualRegister(&AMDGPU::SGPR_32RegClass); - auto BfeMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_BFE_U32), CountReg) - .addReg(InputReg) - .addImm((MI.getOperand(1).getImm() & Mask) | 0x70000); - if (LV) - LV->recomputeForSingleDefVirtReg(InputReg); - auto BfmMI = - BuildMI(*MBB, FirstMI, DL, - TII->get(IsWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64), Exec) - .addReg(CountReg) - .addImm(0); - auto CmpMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_CMP_EQ_U32)) - .addReg(CountReg, RegState::Kill) - .addImm(WavefrontSize); - if (LV) - LV->getVarInfo(CountReg).Kills.push_back(CmpMI); - auto CmovMI = - BuildMI(*MBB, FirstMI, DL, - TII->get(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64), - Exec) - .addImm(-1); - - if (!LIS) { - MI.eraseFromParent(); - return; - } - - LIS->RemoveMachineInstrFromMaps(MI); - MI.eraseFromParent(); - - LIS->InsertMachineInstrInMaps(*BfeMI); - LIS->InsertMachineInstrInMaps(*BfmMI); - LIS->InsertMachineInstrInMaps(*CmpMI); - LIS->InsertMachineInstrInMaps(*CmovMI); - - RecomputeRegs.insert(InputReg); - LIS->createAndComputeVirtRegInterval(CountReg); -} - bool SILowerControlFlow::removeMBBifRedundant(MachineBasicBlock &MBB) { for (auto &I : MBB.instrs()) { if (!I.isDebugInstr() && !I.isUnconditionalBranch()) @@ -927,18 +836,6 @@ bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) { SplitMBB = process(MI); Changed = true; break; - - // FIXME: find a better place for this - case AMDGPU::SI_INIT_EXEC: - case AMDGPU::SI_INIT_EXEC_FROM_INPUT: - lowerInitExec(MBB, MI); - if (LIS) - LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC); - Changed = true; - break; - - default: - break; } if (SplitMBB != MBB) { diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp index 09dc1c781e2f..5b4c44302fa6 100644 --- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp +++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp @@ -177,6 +177,7 @@ private: SmallVector LowerToMovInstrs; SmallVector LowerToCopyInstrs; SmallVector KillInstrs; + SmallVector InitExecInstrs; void printInfo(); @@ -223,6 +224,8 @@ private: void lowerLiveMaskQueries(); void lowerCopyInstrs(); void lowerKillInstrs(bool IsWQM); + void lowerInitExec(MachineInstr &MI); + void lowerInitExecInstrs(); public: static char ID; @@ -580,6 +583,9 @@ char SIWholeQuadMode::scanInstructions(MachineFunction &MF, Opcode == AMDGPU::SI_DEMOTE_I1) { KillInstrs.push_back(&MI); BBI.NeedsLowering = true; + } else if (Opcode == AMDGPU::SI_INIT_EXEC || + Opcode == AMDGPU::SI_INIT_EXEC_FROM_INPUT) { + InitExecInstrs.push_back(&MI); } else if (WQMOutputs) { // The function is in machine SSA form, which means that physical // VGPRs correspond to shader inputs and outputs. Inputs are @@ -1556,6 +1562,97 @@ void SIWholeQuadMode::lowerKillInstrs(bool IsWQM) { } } +void SIWholeQuadMode::lowerInitExec(MachineInstr &MI) { + MachineBasicBlock *MBB = MI.getParent(); + bool IsWave32 = ST->isWave32(); + + if (MI.getOpcode() == AMDGPU::SI_INIT_EXEC) { + // This should be before all vector instructions. + MachineInstr *InitMI = + BuildMI(*MBB, MBB->begin(), MI.getDebugLoc(), + TII->get(IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64), + Exec) + .addImm(MI.getOperand(0).getImm()); + if (LIS) { + LIS->RemoveMachineInstrFromMaps(MI); + LIS->InsertMachineInstrInMaps(*InitMI); + } + MI.eraseFromParent(); + return; + } + + // Extract the thread count from an SGPR input and set EXEC accordingly. + // Since BFM can't shift by 64, handle that case with CMP + CMOV. + // + // S_BFE_U32 count, input, {shift, 7} + // S_BFM_B64 exec, count, 0 + // S_CMP_EQ_U32 count, 64 + // S_CMOV_B64 exec, -1 + Register InputReg = MI.getOperand(0).getReg(); + MachineInstr *FirstMI = &*MBB->begin(); + if (InputReg.isVirtual()) { + MachineInstr *DefInstr = MRI->getVRegDef(InputReg); + assert(DefInstr && DefInstr->isCopy()); + if (DefInstr->getParent() == MBB) { + if (DefInstr != FirstMI) { + // If the `InputReg` is defined in current block, we also need to + // move that instruction to the beginning of the block. + DefInstr->removeFromParent(); + MBB->insert(FirstMI, DefInstr); + if (LIS) + LIS->handleMove(*DefInstr); + } else { + // If first instruction is definition then move pointer after it. + FirstMI = &*std::next(FirstMI->getIterator()); + } + } + } + + // Insert instruction sequence at block beginning (before vector operations). + const DebugLoc DL = MI.getDebugLoc(); + const unsigned WavefrontSize = ST->getWavefrontSize(); + const unsigned Mask = (WavefrontSize << 1) - 1; + Register CountReg = MRI->createVirtualRegister(&AMDGPU::SGPR_32RegClass); + auto BfeMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_BFE_U32), CountReg) + .addReg(InputReg) + .addImm((MI.getOperand(1).getImm() & Mask) | 0x70000); + auto BfmMI = + BuildMI(*MBB, FirstMI, DL, + TII->get(IsWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64), Exec) + .addReg(CountReg) + .addImm(0); + auto CmpMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_CMP_EQ_U32)) + .addReg(CountReg, RegState::Kill) + .addImm(WavefrontSize); + auto CmovMI = + BuildMI(*MBB, FirstMI, DL, + TII->get(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64), + Exec) + .addImm(-1); + + if (!LIS) { + MI.eraseFromParent(); + return; + } + + LIS->RemoveMachineInstrFromMaps(MI); + MI.eraseFromParent(); + + LIS->InsertMachineInstrInMaps(*BfeMI); + LIS->InsertMachineInstrInMaps(*BfmMI); + LIS->InsertMachineInstrInMaps(*CmpMI); + LIS->InsertMachineInstrInMaps(*CmovMI); + + LIS->removeInterval(InputReg); + LIS->createAndComputeVirtRegInterval(InputReg); + LIS->createAndComputeVirtRegInterval(CountReg); +} + +void SIWholeQuadMode::lowerInitExecInstrs() { + for (MachineInstr *MI : InitExecInstrs) + lowerInitExec(*MI); +} + bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { LLVM_DEBUG(dbgs() << "SI Whole Quad Mode on " << MF.getName() << " ------------- \n"); @@ -1567,6 +1664,7 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { LowerToCopyInstrs.clear(); LowerToMovInstrs.clear(); KillInstrs.clear(); + InitExecInstrs.clear(); StateTransition.clear(); ST = &MF.getSubtarget(); @@ -1606,10 +1704,13 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { // Shader is simple does not need any state changes or any complex lowering if (!(GlobalFlags & (StateWQM | StateStrict)) && LowerToCopyInstrs.empty() && LowerToMovInstrs.empty() && KillInstrs.empty()) { + lowerInitExecInstrs(); lowerLiveMaskQueries(); - return !LiveMaskQueries.empty(); + return !InitExecInstrs.empty() || !LiveMaskQueries.empty(); } + lowerInitExecInstrs(); + MachineBasicBlock &Entry = MF.front(); MachineBasicBlock::iterator EntryMI = Entry.getFirstNonPHI(); -- GitLab From 8516f54e6a984a79d1a988ec66a414b5cfbd7ec7 Mon Sep 17 00:00:00 2001 From: Jon Chesterfield Date: Thu, 6 Jun 2024 10:44:53 +0100 Subject: [PATCH 068/462] [AMDGPU] Implement variadic functions by IR lowering (#93362) This is a mostly-target-independent variadic function optimisation and lowering pass. It is only enabled for AMDGPU in this initial commit. The purpose is to make C style variadic functions a zero cost abstraction. They are lowered to equivalent IR which is then amenable to other optimisations. This is inherently slightly target specific but much less so than one might expect - the C varargs interface heavily constrains the ABI design divergence. The pass is primarily tested from webassembly. This is because wasm has a straightforward variadic lowering strategy which coincides exactly with what this pass transforms code into and a struct passing convention with few cases to check. Adding further targets conventions is straightforward and elided from this patch primarily to simplify the review. Implemented in other branches are Linux X86, AMD64, AArch64 and NVPTX. Testing for targets that have existing lowering for va_arg from clang is most efficiently done by checking that clang | opt completely elides the variadic syntax from test cases. The lowering produces a struct for each call site which can be inspected to check the various alignment and indirections are correct. AMDGPU presently has no variadic support other than some ad hoc printf handling. Combined with the pass being inactive on all other targets landing this represents strict increase in capability with zero risk. Testing and refining will continue post commit. In addition to the compiler tests included here, a self contained x64 clang/musl toolchain was constructed using the "lowering" instead of the systemv ABI and used to build various C programs like lua and libxml2. --- clang/lib/CodeGen/Targets/AMDGPU.cpp | 6 +- clang/test/CodeGen/voidptr-vaarg.c | 478 ++++++++ .../CodeGenCXX/inline-then-fold-variadics.cpp | 181 +++ libc/config/gpu/entrypoints.txt | 4 + libc/test/src/__support/CMakeLists.txt | 4 +- llvm/include/llvm/IR/InstrTypes.h | 9 + llvm/include/llvm/InitializePasses.h | 1 + .../llvm/Transforms/IPO/ExpandVariadics.h | 40 + llvm/lib/Passes/PassBuilder.cpp | 1 + llvm/lib/Passes/PassRegistry.def | 1 + .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 5 + llvm/lib/Transforms/IPO/CMakeLists.txt | 1 + llvm/lib/Transforms/IPO/ExpandVariadics.cpp | 1013 +++++++++++++++++ .../CodeGen/AMDGPU/expand-variadic-call.ll | 545 +++++++++ llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 5 + llvm/test/CodeGen/AMDGPU/unsupported-calls.ll | 19 - .../WebAssembly/expand-variadic-call.ll | 484 ++++++++ llvm/test/CodeGen/WebAssembly/vararg-frame.ll | 526 +++++++++ .../expand-va-intrinsic-split-linkage.ll | 232 ++++ .../expand-va-intrinsic-split-simple.ll | 214 ++++ .../ExpandVariadics/indirect-calls.ll | 59 + .../Transforms/ExpandVariadics/intrinsics.ll | 120 ++ .../test/Transforms/ExpandVariadics/invoke.ll | 89 ++ .../ExpandVariadics/pass-byval-byref.ll | 153 +++ .../ExpandVariadics/pass-indirect.ll | 59 + .../ExpandVariadics/pass-integers.ll | 345 ++++++ 26 files changed, 4572 insertions(+), 22 deletions(-) create mode 100644 clang/test/CodeGen/voidptr-vaarg.c create mode 100644 clang/test/CodeGenCXX/inline-then-fold-variadics.cpp create mode 100644 llvm/include/llvm/Transforms/IPO/ExpandVariadics.h create mode 100644 llvm/lib/Transforms/IPO/ExpandVariadics.cpp create mode 100644 llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll create mode 100644 llvm/test/CodeGen/WebAssembly/expand-variadic-call.ll create mode 100644 llvm/test/CodeGen/WebAssembly/vararg-frame.ll create mode 100644 llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-linkage.ll create mode 100644 llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-simple.ll create mode 100644 llvm/test/Transforms/ExpandVariadics/indirect-calls.ll create mode 100644 llvm/test/Transforms/ExpandVariadics/intrinsics.ll create mode 100644 llvm/test/Transforms/ExpandVariadics/invoke.ll create mode 100644 llvm/test/Transforms/ExpandVariadics/pass-byval-byref.ll create mode 100644 llvm/test/Transforms/ExpandVariadics/pass-indirect.ll create mode 100644 llvm/test/Transforms/ExpandVariadics/pass-integers.ll diff --git a/clang/lib/CodeGen/Targets/AMDGPU.cpp b/clang/lib/CodeGen/Targets/AMDGPU.cpp index d1ff8b4b62f1..057f6ef40c51 100644 --- a/clang/lib/CodeGen/Targets/AMDGPU.cpp +++ b/clang/lib/CodeGen/Targets/AMDGPU.cpp @@ -120,7 +120,11 @@ void AMDGPUABIInfo::computeInfo(CGFunctionInfo &FI) const { Address AMDGPUABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty) const { - llvm_unreachable("AMDGPU does not support varargs"); + const bool IsIndirect = false; + const bool AllowHigherAlign = false; + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, + getContext().getTypeInfoInChars(Ty), + CharUnits::fromQuantity(4), AllowHigherAlign); } ABIArgInfo AMDGPUABIInfo::classifyReturnType(QualType RetTy) const { diff --git a/clang/test/CodeGen/voidptr-vaarg.c b/clang/test/CodeGen/voidptr-vaarg.c new file mode 100644 index 000000000000..d023ddf0fb5d --- /dev/null +++ b/clang/test/CodeGen/voidptr-vaarg.c @@ -0,0 +1,478 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: webassembly-registered-target +// RUN: %clang_cc1 -triple wasm32-unknown-unknown -emit-llvm -o - %s | FileCheck %s + +// Multiple targets use emitVoidPtrVAArg to lower va_arg instructions in clang +// PPC is complicated, excluding from this case analysis +// ForceRightAdjust is false for all non-PPC targets +// AllowHigherAlign is only false for two Microsoft targets, both of which +// pass most things by reference. +// +// Address emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr, +// QualType ValueTy, bool IsIndirect, +// TypeInfoChars ValueInfo, CharUnits SlotSizeAndAlign, +// bool AllowHigherAlign, bool ForceRightAdjust = +// false); +// +// Target IsIndirect SlotSize AllowHigher ForceRightAdjust +// ARC false four true false +// ARM varies four true false +// Mips false 4 or 8 true false +// RISCV varies register true false +// PPC elided +// LoongArch varies register true false +// NVPTX WIP +// AMDGPU WIP +// X86_32 false four true false +// X86_64 MS varies eight false false +// CSKY false four true false +// Webassembly varies four true false +// AArch64 false eight true false +// AArch64 MS false eight false false +// +// Webassembly passes indirectly iff it's an aggregate of multiple values +// Choosing this as a representative architecture to check IR generation +// partly because it has a relatively simple variadic calling convention. + +// Int, by itself and packed in structs +// CHECK-LABEL: @raw_int( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARGP_CUR]], align 4 +// CHECK-NEXT: ret i32 [[TMP0]] +// +int raw_int(__builtin_va_list list) { return __builtin_va_arg(list, int); } + +typedef struct { + int x; +} one_int_t; + +// CHECK-LABEL: @one_int( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[RETVAL:%.*]] = alloca [[STRUCT_ONE_INT_T:%.*]], align 4 +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[RETVAL]], ptr align 4 [[ARGP_CUR]], i32 4, i1 false) +// CHECK-NEXT: [[COERCE_DIVE:%.*]] = getelementptr inbounds [[STRUCT_ONE_INT_T]], ptr [[RETVAL]], i32 0, i32 0 +// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[COERCE_DIVE]], align 4 +// CHECK-NEXT: ret i32 [[TMP0]] +// +one_int_t one_int(__builtin_va_list list) { + return __builtin_va_arg(list, one_int_t); +} + +typedef struct { + int x; + int y; +} two_int_t; + +// CHECK-LABEL: @two_int( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ARGP_CUR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[AGG_RESULT:%.*]], ptr align 4 [[TMP0]], i32 8, i1 false) +// CHECK-NEXT: ret void +// +two_int_t two_int(__builtin_va_list list) { + return __builtin_va_arg(list, two_int_t); +} + +// Double, by itself and packed in structs +// CHECK-LABEL: @raw_double( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 7 +// CHECK-NEXT: [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP0]], i32 -8) +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR_ALIGNED]], i32 8 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[ARGP_CUR_ALIGNED]], align 8 +// CHECK-NEXT: ret double [[TMP1]] +// +double raw_double(__builtin_va_list list) { + return __builtin_va_arg(list, double); +} + +typedef struct { + double x; +} one_double_t; + +// CHECK-LABEL: @one_double( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[RETVAL:%.*]] = alloca [[STRUCT_ONE_DOUBLE_T:%.*]], align 8 +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 7 +// CHECK-NEXT: [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP0]], i32 -8) +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR_ALIGNED]], i32 8 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[RETVAL]], ptr align 8 [[ARGP_CUR_ALIGNED]], i32 8, i1 false) +// CHECK-NEXT: [[COERCE_DIVE:%.*]] = getelementptr inbounds [[STRUCT_ONE_DOUBLE_T]], ptr [[RETVAL]], i32 0, i32 0 +// CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[COERCE_DIVE]], align 8 +// CHECK-NEXT: ret double [[TMP1]] +// +one_double_t one_double(__builtin_va_list list) { + return __builtin_va_arg(list, one_double_t); +} + +typedef struct { + double x; + double y; +} two_double_t; + +// CHECK-LABEL: @two_double( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ARGP_CUR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[AGG_RESULT:%.*]], ptr align 8 [[TMP0]], i32 16, i1 false) +// CHECK-NEXT: ret void +// +two_double_t two_double(__builtin_va_list list) { + return __builtin_va_arg(list, two_double_t); +} + +// Scalar smaller than the slot size (C would promote a short to int) +typedef struct { + char x; +} one_char_t; + +// CHECK-LABEL: @one_char( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[RETVAL:%.*]] = alloca [[STRUCT_ONE_CHAR_T:%.*]], align 1 +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[RETVAL]], ptr align 4 [[ARGP_CUR]], i32 1, i1 false) +// CHECK-NEXT: [[COERCE_DIVE:%.*]] = getelementptr inbounds [[STRUCT_ONE_CHAR_T]], ptr [[RETVAL]], i32 0, i32 0 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[COERCE_DIVE]], align 1 +// CHECK-NEXT: ret i8 [[TMP0]] +// +one_char_t one_char(__builtin_va_list list) { + return __builtin_va_arg(list, one_char_t); +} + +typedef struct { + short x; +} one_short_t; + +// CHECK-LABEL: @one_short( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[RETVAL:%.*]] = alloca [[STRUCT_ONE_SHORT_T:%.*]], align 2 +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 2 [[RETVAL]], ptr align 4 [[ARGP_CUR]], i32 2, i1 false) +// CHECK-NEXT: [[COERCE_DIVE:%.*]] = getelementptr inbounds [[STRUCT_ONE_SHORT_T]], ptr [[RETVAL]], i32 0, i32 0 +// CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr [[COERCE_DIVE]], align 2 +// CHECK-NEXT: ret i16 [[TMP0]] +// +one_short_t one_short(__builtin_va_list list) { + return __builtin_va_arg(list, one_short_t); +} + +// Composite smaller than the slot size +typedef struct { + _Alignas(2) char x; + char y; +} char_pair_t; + +// CHECK-LABEL: @char_pair( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ARGP_CUR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 2 [[AGG_RESULT:%.*]], ptr align 2 [[TMP0]], i32 2, i1 false) +// CHECK-NEXT: ret void +// +char_pair_t char_pair(__builtin_va_list list) { + return __builtin_va_arg(list, char_pair_t); +} + +// Empty struct +typedef struct { +} empty_t; + +// CHECK-LABEL: @empty( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[RETVAL:%.*]] = alloca [[STRUCT_EMPTY_T:%.*]], align 1 +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 0 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[RETVAL]], ptr align 4 [[ARGP_CUR]], i32 0, i1 false) +// CHECK-NEXT: ret void +// +empty_t empty(__builtin_va_list list) { + return __builtin_va_arg(list, empty_t); +} + +typedef struct { + empty_t x; + int y; +} empty_int_t; + +// CHECK-LABEL: @empty_int( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[RETVAL:%.*]] = alloca [[STRUCT_EMPTY_INT_T:%.*]], align 4 +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[RETVAL]], ptr align 4 [[ARGP_CUR]], i32 4, i1 false) +// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[RETVAL]], align 4 +// CHECK-NEXT: ret i32 [[TMP0]] +// +empty_int_t empty_int(__builtin_va_list list) { + return __builtin_va_arg(list, empty_int_t); +} + +typedef struct { + int x; + empty_t y; +} int_empty_t; + +// CHECK-LABEL: @int_empty( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[RETVAL:%.*]] = alloca [[STRUCT_INT_EMPTY_T:%.*]], align 4 +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[RETVAL]], ptr align 4 [[ARGP_CUR]], i32 4, i1 false) +// CHECK-NEXT: [[COERCE_DIVE:%.*]] = getelementptr inbounds [[STRUCT_INT_EMPTY_T]], ptr [[RETVAL]], i32 0, i32 0 +// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[COERCE_DIVE]], align 4 +// CHECK-NEXT: ret i32 [[TMP0]] +// +int_empty_t int_empty(__builtin_va_list list) { + return __builtin_va_arg(list, int_empty_t); +} + +// Need multiple va_arg instructions to check the postincrement +// Using types that are passed directly as the indirect handling +// is independent of the alignment handling in emitVoidPtrDirectVAArg. + +// CHECK-LABEL: @multiple_int( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT0_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT1_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT2_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT0:%.*]], ptr [[OUT0_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT1:%.*]], ptr [[OUT1_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT2:%.*]], ptr [[OUT2_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARGP_CUR]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[OUT0_ADDR]], align 4 +// CHECK-NEXT: store i32 [[TMP0]], ptr [[TMP1]], align 4 +// CHECK-NEXT: [[ARGP_CUR1:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT2:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR1]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT2]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARGP_CUR1]], align 4 +// CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[OUT1_ADDR]], align 4 +// CHECK-NEXT: store i32 [[TMP2]], ptr [[TMP3]], align 4 +// CHECK-NEXT: [[ARGP_CUR3:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT4:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR3]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT4]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARGP_CUR3]], align 4 +// CHECK-NEXT: [[TMP5:%.*]] = load ptr, ptr [[OUT2_ADDR]], align 4 +// CHECK-NEXT: store i32 [[TMP4]], ptr [[TMP5]], align 4 +// CHECK-NEXT: ret void +// +void multiple_int(__builtin_va_list list, int *out0, int *out1, int *out2) { + *out0 = __builtin_va_arg(list, int); + *out1 = __builtin_va_arg(list, int); + *out2 = __builtin_va_arg(list, int); +} + +// Scalars in structs are an easy way of specifying alignment from C +// CHECK-LABEL: @increasing_alignment( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT0_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT1_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT2_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT3_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT0:%.*]], ptr [[OUT0_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT1:%.*]], ptr [[OUT1_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT2:%.*]], ptr [[OUT2_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT3:%.*]], ptr [[OUT3_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[OUT0_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[TMP0]], ptr align 4 [[ARGP_CUR]], i32 1, i1 false) +// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[OUT1_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR1:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT2:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR1]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT2]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 2 [[TMP1]], ptr align 4 [[ARGP_CUR1]], i32 2, i1 false) +// CHECK-NEXT: [[ARGP_CUR3:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT4:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR3]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT4]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARGP_CUR3]], align 4 +// CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[OUT2_ADDR]], align 4 +// CHECK-NEXT: store i32 [[TMP2]], ptr [[TMP3]], align 4 +// CHECK-NEXT: [[ARGP_CUR5:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR5]], i32 7 +// CHECK-NEXT: [[ARGP_CUR5_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP4]], i32 -8) +// CHECK-NEXT: [[ARGP_NEXT6:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR5_ALIGNED]], i32 8 +// CHECK-NEXT: store ptr [[ARGP_NEXT6]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP5:%.*]] = load double, ptr [[ARGP_CUR5_ALIGNED]], align 8 +// CHECK-NEXT: [[TMP6:%.*]] = load ptr, ptr [[OUT3_ADDR]], align 4 +// CHECK-NEXT: store double [[TMP5]], ptr [[TMP6]], align 8 +// CHECK-NEXT: ret void +// +void increasing_alignment(__builtin_va_list list, one_char_t *out0, + one_short_t *out1, int *out2, double *out3) { + *out0 = __builtin_va_arg(list, one_char_t); + *out1 = __builtin_va_arg(list, one_short_t); + *out2 = __builtin_va_arg(list, int); + *out3 = __builtin_va_arg(list, double); +} + +// CHECK-LABEL: @decreasing_alignment( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT0_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT1_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT2_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT3_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT0:%.*]], ptr [[OUT0_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT1:%.*]], ptr [[OUT1_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT2:%.*]], ptr [[OUT2_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT3:%.*]], ptr [[OUT3_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 7 +// CHECK-NEXT: [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP0]], i32 -8) +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR_ALIGNED]], i32 8 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[ARGP_CUR_ALIGNED]], align 8 +// CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[OUT0_ADDR]], align 4 +// CHECK-NEXT: store double [[TMP1]], ptr [[TMP2]], align 8 +// CHECK-NEXT: [[ARGP_CUR1:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT2:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR1]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT2]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARGP_CUR1]], align 4 +// CHECK-NEXT: [[TMP4:%.*]] = load ptr, ptr [[OUT1_ADDR]], align 4 +// CHECK-NEXT: store i32 [[TMP3]], ptr [[TMP4]], align 4 +// CHECK-NEXT: [[TMP5:%.*]] = load ptr, ptr [[OUT2_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR3:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT4:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR3]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT4]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 2 [[TMP5]], ptr align 4 [[ARGP_CUR3]], i32 2, i1 false) +// CHECK-NEXT: [[TMP6:%.*]] = load ptr, ptr [[OUT3_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR5:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT6:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR5]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT6]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[TMP6]], ptr align 4 [[ARGP_CUR5]], i32 1, i1 false) +// CHECK-NEXT: ret void +// +void decreasing_alignment(__builtin_va_list list, double *out0, int *out1, + one_short_t *out2, one_char_t *out3) { + *out0 = __builtin_va_arg(list, double); + *out1 = __builtin_va_arg(list, int); + *out2 = __builtin_va_arg(list, one_short_t); + *out3 = __builtin_va_arg(list, one_char_t); +} + +// Typical edge cases, none hit special handling in VAArg lowering. +typedef struct { + int x[16]; + double y[8]; +} large_value_t; + +// CHECK-LABEL: @large_value( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT:%.*]], ptr [[OUT_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[OUT_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[ARGP_CUR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[TMP0]], ptr align 8 [[TMP1]], i32 128, i1 false) +// CHECK-NEXT: ret void +// +void large_value(__builtin_va_list list, large_value_t *out) { + *out = __builtin_va_arg(list, large_value_t); +} + +typedef int v128_t __attribute__((__vector_size__(16), __aligned__(16))); +// CHECK-LABEL: @vector( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT:%.*]], ptr [[OUT_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 15 +// CHECK-NEXT: [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP0]], i32 -16) +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR_ALIGNED]], i32 16 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[ARGP_CUR_ALIGNED]], align 16 +// CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[OUT_ADDR]], align 4 +// CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 16 +// CHECK-NEXT: ret void +// +void vector(__builtin_va_list list, v128_t *out) { + *out = __builtin_va_arg(list, v128_t); +} + +typedef struct BF { + float not_an_i32[2]; + int A : 1; + char B; + int C : 13; +} BF; + +// CHECK-LABEL: @bitfield( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[LIST_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: store ptr [[LIST:%.*]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: store ptr [[OUT:%.*]], ptr [[OUT_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[OUT_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_CUR:%.*]] = load ptr, ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4 +// CHECK-NEXT: store ptr [[ARGP_NEXT]], ptr [[LIST_ADDR]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[ARGP_CUR]], align 4 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[TMP0]], ptr align 4 [[TMP1]], i32 12, i1 false) +// CHECK-NEXT: ret void +// +void bitfield(__builtin_va_list list, BF *out) { + *out = __builtin_va_arg(list, BF); +} diff --git a/clang/test/CodeGenCXX/inline-then-fold-variadics.cpp b/clang/test/CodeGenCXX/inline-then-fold-variadics.cpp new file mode 100644 index 000000000000..a0673b96626d --- /dev/null +++ b/clang/test/CodeGenCXX/inline-then-fold-variadics.cpp @@ -0,0 +1,181 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature +// REQUIRES: webassembly-registered-target + +// Simple calls to known variadic functions that are completely elided when +// optimisations are on This is a functional check that the expand-variadic pass +// is consistent with clang's va_arg handling + +// When expand-variadics is added to the default pipeline, clang -O1 will +// suffice here -Wno-varargs avoids warning second argument to 'va_start' is not +// the last named parameter + +// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -Wno-varargs -O1 -emit-llvm -o - | opt - -S --passes='module(expand-variadics,default)' --expand-variadics-override=optimize -o - | FileCheck %s + +#include +#include + +template static X first(...) { + va_list va; + __builtin_va_start(va, 0); + X r = va_arg(va, X); + va_end(va); + return r; +} + +template static Y second(...) { + va_list va; + __builtin_va_start(va, 0); + va_arg(va, X); + Y r = va_arg(va, Y); + va_end(va); + return r; +} + +extern "C" { + +// CHECK-LABEL: define {{[^@]+}}@first_pair_i32 +// CHECK-SAME: (i32 noundef returned [[X:%.*]], i32 noundef [[Y:%.*]]) +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 [[X]] +// +int first_pair_i32(int x, int y) { return first(x, y); } + +// CHECK-LABEL: define {{[^@]+}}@second_pair_i32 +// CHECK-SAME: (i32 noundef [[X:%.*]], i32 noundef returned [[Y:%.*]]) +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 [[Y]] +// +int second_pair_i32(int x, int y) { return second(x, y); } + +// CHECK-LABEL: define {{[^@]+}}@first_pair_f64 +// CHECK-SAME: (double noundef returned [[X:%.*]], double noundef [[Y:%.*]]) +// CHECK-NEXT: entry: +// CHECK-NEXT: ret double [[X]] +// +double first_pair_f64(double x, double y) { + return first(x, y); +} + +// CHECK-LABEL: define {{[^@]+}}@second_pair_f64 +// CHECK-SAME: (double noundef [[X:%.*]], double noundef returned [[Y:%.*]]) +// CHECK-NEXT: entry: +// CHECK-NEXT: ret double [[Y]] +// +double second_pair_f64(double x, double y) { + return second(x, y); +} +} + +extern "C" { + +// CHECK-LABEL: define {{[^@]+}}@first_i32_f64 +// CHECK-SAME: (i32 noundef returned [[X:%.*]], double noundef [[Y:%.*]]) +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 [[X]] +// +int first_i32_f64(int x, double y) { return first(x, y); } + +// CHECK-LABEL: define {{[^@]+}}@second_i32_f64 +// CHECK-SAME: (i32 noundef [[X:%.*]], double noundef returned [[Y:%.*]]) +// CHECK-NEXT: entry: +// CHECK-NEXT: ret double [[Y]] +// +double second_i32_f64(int x, double y) { return second(x, y); } + +// CHECK-LABEL: define {{[^@]+}}@first_f64_i32 +// CHECK-SAME: (double noundef returned [[X:%.*]], i32 noundef [[Y:%.*]]) +// CHECK-NEXT: entry: +// CHECK-NEXT: ret double [[X]] +// +double first_f64_i32(double x, int y) { return first(x, y); } + +// CHECK-LABEL: define {{[^@]+}}@second_f64_i32 +// CHECK-SAME: (double noundef [[X:%.*]], i32 noundef returned [[Y:%.*]]) +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 [[Y]] +// +int second_f64_i32(double x, int y) { return second(x, y); } +} + +extern "C" { +typedef uint64_t ulong2 __attribute__((__vector_size__(16), __aligned__(16))); + +// CHECK-LABEL: define {{[^@]+}}@first_i32_ulong2 +// CHECK-SAME: (i32 noundef returned [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 [[X]] +// +int first_i32_ulong2(int x, ulong2 *y) { return first(x, *y); } + +// CHECK-LABEL: define {{[^@]+}}@second_i32_ulong2 +// CHECK-SAME: (i32 noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]], ptr nocapture noundef writeonly [[R:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load <2 x i64>, ptr [[Y]], align 16, !tbaa [[TBAA2:![0-9]+]] +// CHECK-NEXT: store <2 x i64> [[TMP0]], ptr [[R]], align 16, !tbaa [[TBAA2]] +// CHECK-NEXT: ret void +// +void second_i32_ulong2(int x, ulong2 *y, ulong2 *r) { + *r = second(x, *y); +} + +// CHECK-LABEL: define {{[^@]+}}@first_ulong2_i32 +// CHECK-SAME: (ptr nocapture noundef readonly [[X:%.*]], i32 noundef [[Y:%.*]], ptr nocapture noundef writeonly [[R:%.*]]) local_unnamed_addr #[[ATTR1]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = load <2 x i64>, ptr [[X]], align 16, !tbaa [[TBAA2]] +// CHECK-NEXT: store <2 x i64> [[TMP0]], ptr [[R]], align 16, !tbaa [[TBAA2]] +// CHECK-NEXT: ret void +// +void first_ulong2_i32(ulong2 *x, int y, ulong2 *r) { + *r = first(*x, y); +} + +// CHECK-LABEL: define {{[^@]+}}@second_ulong2_i32 +// CHECK-SAME: (ptr nocapture noundef readonly [[X:%.*]], i32 noundef returned [[Y:%.*]]) +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 [[Y]] +// +int second_ulong2_i32(ulong2 *x, int y) { return second(*x, y); } +} + +// ascending alignment +typedef struct { + char c; + short s; + int i; + long l; + float f; + double d; +} asc; + +extern "C" { + +// CHECK-LABEL: define {{[^@]+}}@first_i32_asc +// CHECK-SAME: (i32 noundef returned [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]]) +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 [[X]] +// +int first_i32_asc(int x, asc *y) { return first(x, *y); } + +// CHECK-LABEL: define {{[^@]+}}@second_i32_asc +// CHECK-SAME: (i32 noundef [[X:%.*]], ptr nocapture noundef readonly [[Y:%.*]], ptr nocapture noundef writeonly [[R:%.*]]) local_unnamed_addr #[[ATTR1]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.memmove.p0.p0.i32(ptr noundef nonnull align 8 dereferenceable(24) [[R]], ptr noundef nonnull align 1 dereferenceable(24) [[Y]], i32 24, i1 false) +// CHECK-NEXT: ret void +// +void second_i32_asc(int x, asc *y, asc *r) { *r = second(x, *y); } + +// CHECK-LABEL: define {{[^@]+}}@first_asc_i32 +// CHECK-SAME: (ptr nocapture noundef readonly [[X:%.*]], i32 noundef [[Y:%.*]], ptr nocapture noundef writeonly [[R:%.*]]) local_unnamed_addr #[[ATTR1]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.memmove.p0.p0.i32(ptr noundef nonnull align 8 dereferenceable(24) [[R]], ptr noundef nonnull align 1 dereferenceable(24) [[X]], i32 24, i1 false) +// CHECK-NEXT: ret void +// +void first_asc_i32(asc *x, int y, asc *r) { *r = first(*x, y); } + +// CHECK-LABEL: define {{[^@]+}}@second_asc_i32 +// CHECK-SAME: (ptr nocapture noundef readonly [[X:%.*]], i32 noundef returned [[Y:%.*]]) +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 [[Y]] +// +int second_asc_i32(asc *x, int y) { return second(*x, y); } +} diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt index b678350e9fcb..7b73b8a22421 100644 --- a/libc/config/gpu/entrypoints.txt +++ b/libc/config/gpu/entrypoints.txt @@ -181,6 +181,10 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdio.fflush libc.src.stdio.ftell libc.src.stdio.clearerr + libc.src.stdio.sprintf + libc.src.stdio.snprintf + libc.src.stdio.vsprintf + libc.src.stdio.vsnprintf libc.src.stdio.puts libc.src.stdio.fopen libc.src.stdio.fclose diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt index 663aa2bb82ca..5afc4173f61a 100644 --- a/libc/test/src/__support/CMakeLists.txt +++ b/libc/test/src/__support/CMakeLists.txt @@ -86,8 +86,8 @@ add_libc_test( libc.src.__support.uint128 ) -# The GPU does not support varargs currently. -if(NOT LIBC_TARGET_OS_IS_GPU) +# NVPTX does not support varargs currently. +if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX) add_libc_test( arg_list_test SUITE diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h index 441e6a1e7984..ad649b53761a 100644 --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -2128,6 +2128,15 @@ public: return Attrs.getParamStackAlignment(ArgNo); } + /// Extract the byref type for a call or parameter. + Type *getParamByRefType(unsigned ArgNo) const { + if (auto *Ty = Attrs.getParamByRefType(ArgNo)) + return Ty; + if (const Function *F = getCalledFunction()) + return F->getAttributes().getParamByRefType(ArgNo); + return nullptr; + } + /// Extract the byval type for a call or parameter. Type *getParamByValType(unsigned ArgNo) const { if (auto *Ty = Attrs.getParamByValType(ArgNo)) diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index c4c1825bbf09..8803ef5a90e6 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -106,6 +106,7 @@ void initializeExpandLargeDivRemLegacyPassPass(PassRegistry&); void initializeExpandMemCmpLegacyPassPass(PassRegistry &); void initializeExpandPostRAPass(PassRegistry&); void initializeExpandReductionsPass(PassRegistry&); +void initializeExpandVariadicsPass(PassRegistry &); void initializeExpandVectorPredicationPass(PassRegistry &); void initializeExternalAAWrapperPassPass(PassRegistry&); void initializeFEntryInserterPass(PassRegistry&); diff --git a/llvm/include/llvm/Transforms/IPO/ExpandVariadics.h b/llvm/include/llvm/Transforms/IPO/ExpandVariadics.h new file mode 100644 index 000000000000..4c5a1b61e2d4 --- /dev/null +++ b/llvm/include/llvm/Transforms/IPO/ExpandVariadics.h @@ -0,0 +1,40 @@ +//===- ExpandVariadics.h - expand variadic functions ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TRANSFORMS_IPO_EXPANDVARIADICS_H +#define LLVM_TRANSFORMS_IPO_EXPANDVARIADICS_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class Module; +class ModulePass; +class OptimizationLevel; + +enum class ExpandVariadicsMode { + Unspecified, // Use the implementation defaults + Disable, // Disable the pass entirely + Optimize, // Optimise without changing ABI + Lowering, // Change variadic calling convention +}; + +class ExpandVariadicsPass : public PassInfoMixin { + const ExpandVariadicsMode Mode; + +public: + // Operates under passed mode unless overridden on commandline + ExpandVariadicsPass(ExpandVariadicsMode Mode); + + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + +ModulePass *createExpandVariadicsPass(ExpandVariadicsMode); + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_IPO_EXPANDVARIADICS_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 316d05bf1dc3..8dd060d0151a 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -139,6 +139,7 @@ #include "llvm/Transforms/IPO/DeadArgumentElimination.h" #include "llvm/Transforms/IPO/ElimAvailExtern.h" #include "llvm/Transforms/IPO/EmbedBitcodePass.h" +#include "llvm/Transforms/IPO/ExpandVariadics.h" #include "llvm/Transforms/IPO/ForceFunctionAttrs.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/IPO/FunctionImport.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 50682ca4970f..dad97146a9f6 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -59,6 +59,7 @@ MODULE_PASS("dot-callgraph", CallGraphDOTPrinterPass()) MODULE_PASS("dxil-upgrade", DXILUpgradePass()) MODULE_PASS("elim-avail-extern", EliminateAvailableExternallyPass()) MODULE_PASS("extract-blocks", BlockExtractorPass({}, false)) +MODULE_PASS("expand-variadics", ExpandVariadicsPass(ExpandVariadicsMode::Disable)) MODULE_PASS("forceattrs", ForceFunctionAttrsPass()) MODULE_PASS("function-import", FunctionImportPass()) MODULE_PASS("globalopt", GlobalOptPass()) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index 9c94ca1e4708..17c961578382 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -57,6 +57,7 @@ #include "llvm/Transforms/HipStdPar/HipStdPar.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" +#include "llvm/Transforms/IPO/ExpandVariadics.h" #include "llvm/Transforms/IPO/GlobalDCE.h" #include "llvm/Transforms/IPO/Internalize.h" #include "llvm/Transforms/Scalar.h" @@ -992,6 +993,10 @@ void AMDGPUPassConfig::addIRPasses() { if (isPassEnabled(EnableImageIntrinsicOptimizer)) addPass(createAMDGPUImageIntrinsicOptimizerPass(&TM)); + // This can be disabled by passing ::Disable here or on the command line + // with --expand-variadics-override=disable. + addPass(createExpandVariadicsPass(ExpandVariadicsMode::Lowering)); + // Function calls are not supported, so make sure we inline everything. addPass(createAMDGPUAlwaysInlinePass()); addPass(createAlwaysInlinerLegacyPass()); diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt index 5fbdbc3a014f..92a9697720ef 100644 --- a/llvm/lib/Transforms/IPO/CMakeLists.txt +++ b/llvm/lib/Transforms/IPO/CMakeLists.txt @@ -12,6 +12,7 @@ add_llvm_component_library(LLVMipo DeadArgumentElimination.cpp ElimAvailExtern.cpp EmbedBitcodePass.cpp + ExpandVariadics.cpp ExtractGV.cpp ForceFunctionAttrs.cpp FunctionAttrs.cpp diff --git a/llvm/lib/Transforms/IPO/ExpandVariadics.cpp b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp new file mode 100644 index 000000000000..9a4f39948a30 --- /dev/null +++ b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp @@ -0,0 +1,1013 @@ +//===-- ExpandVariadicsPass.cpp --------------------------------*- C++ -*-=// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This is an optimization pass for variadic functions. If called from codegen, +// it can serve as the implementation of variadic functions for a given target. +// +// The strategy is to turn the ... part of a variadic function into a va_list +// and fix up the call sites. The majority of the pass is target independent. +// The exceptions are the va_list type itself and the rules for where to store +// variables in memory such that va_arg can iterate over them given a va_list. +// +// The majority of the plumbing is splitting the variadic function into a +// single basic block that packs the variadic arguments into a va_list and +// a second function that does the work of the original. That packing is +// exactly what is done by va_start. Further, the transform from ... to va_list +// replaced va_start with an operation to copy a va_list from the new argument, +// which is exactly a va_copy. This is useful for reducing target-dependence. +// +// A va_list instance is a forward iterator, where the primary operation va_arg +// is dereference-then-increment. This interface forces significant convergent +// evolution between target specific implementations. The variation in runtime +// data layout is limited to that representable by the iterator, parameterised +// by the type passed to the va_arg instruction. +// +// Therefore the majority of the target specific subtlety is packing arguments +// into a stack allocated buffer such that a va_list can be initialised with it +// and the va_arg expansion for the target will find the arguments at runtime. +// +// The aggregate effect is to unblock other transforms, most critically the +// general purpose inliner. Known calls to variadic functions become zero cost. +// +// Consistency with clang is primarily tested by emitting va_arg using clang +// then expanding the variadic functions using this pass, followed by trying +// to constant fold the functions to no-ops. +// +// Target specific behaviour is tested in IR - mainly checking that values are +// put into positions in call frames that make sense for that particular target. +// +// There is one "clever" invariant in use. va_start intrinsics that are not +// within a varidic functions are an error in the IR verifier. When this +// transform moves blocks from a variadic function into a fixed arity one, it +// moves va_start intrinsics along with everything else. That means that the +// va_start intrinsics that need to be rewritten to use the trailing argument +// are exactly those that are in non-variadic functions so no further state +// is needed to distinguish those that need to be rewritten. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/IPO/ExpandVariadics.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" +#include "llvm/Passes/OptimizationLevel.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/TargetParser/Triple.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" + +#define DEBUG_TYPE "expand-variadics" + +using namespace llvm; + +namespace { + +cl::opt ExpandVariadicsModeOption( + DEBUG_TYPE "-override", cl::desc("Override the behaviour of " DEBUG_TYPE), + cl::init(ExpandVariadicsMode::Unspecified), + cl::values(clEnumValN(ExpandVariadicsMode::Unspecified, "unspecified", + "Use the implementation defaults"), + clEnumValN(ExpandVariadicsMode::Disable, "disable", + "Disable the pass entirely"), + clEnumValN(ExpandVariadicsMode::Optimize, "optimize", + "Optimise without changing ABI"), + clEnumValN(ExpandVariadicsMode::Lowering, "lowering", + "Change variadic calling convention"))); + +bool commandLineOverride() { + return ExpandVariadicsModeOption != ExpandVariadicsMode::Unspecified; +} + +// Instances of this class encapsulate the target-dependant behaviour as a +// function of triple. Implementing a new ABI is adding a case to the switch +// in create(llvm::Triple) at the end of this file. +// This class may end up instantiated in TargetMachine instances, keeping it +// here for now until enough targets are implemented for the API to evolve. +class VariadicABIInfo { +protected: + VariadicABIInfo() = default; + +public: + static std::unique_ptr create(const Triple &T); + + // Allow overriding whether the pass runs on a per-target basis + virtual bool enableForTarget() = 0; + + // Whether a valist instance is passed by value or by address + // I.e. does it need to be alloca'ed and stored into, or can + // it be passed directly in a SSA register + virtual bool vaListPassedInSSARegister() = 0; + + // The type of a va_list iterator object + virtual Type *vaListType(LLVMContext &Ctx) = 0; + + // The type of a va_list as a function argument as lowered by C + virtual Type *vaListParameterType(Module &M) = 0; + + // Initialize an allocated va_list object to point to an already + // initialized contiguous memory region. + // Return the value to pass as the va_list argument + virtual Value *initializeVaList(Module &M, LLVMContext &Ctx, + IRBuilder<> &Builder, AllocaInst *VaList, + Value *Buffer) = 0; + + struct VAArgSlotInfo { + Align DataAlign; // With respect to the call frame + bool Indirect; // Passed via a pointer + }; + virtual VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) = 0; + + // Targets implemented so far all have the same trivial lowering for these + bool vaEndIsNop() { return true; } + bool vaCopyIsMemcpy() { return true; } + + virtual ~VariadicABIInfo() = default; +}; + +// Module implements getFunction() which returns nullptr on missing declaration +// and getOrInsertFunction which creates one when absent. Intrinsics.h only +// implements getDeclaration which creates one when missing. Checking whether +// an intrinsic exists thus inserts it in the module and it then needs to be +// deleted again to clean up. +// The right name for the two functions on intrinsics would match Module::, +// but doing that in a single change would introduce nullptr dereferences +// where currently there are none. The minimal collateral damage approach +// would split the change over a release to help downstream branches. As it +// is unclear what approach will be preferred, implementing the trivial +// function here in the meantime to decouple from that discussion. +Function *getPreexistingDeclaration(Module *M, Intrinsic::ID Id, + ArrayRef Tys = {}) { + auto *FT = Intrinsic::getType(M->getContext(), Id, Tys); + return M->getFunction(Tys.empty() ? Intrinsic::getName(Id) + : Intrinsic::getName(Id, Tys, M, FT)); +} + +class ExpandVariadics : public ModulePass { + + // The pass construction sets the default to optimize when called from middle + // end and lowering when called from the backend. The command line variable + // overrides that. This is useful for testing and debugging. It also allows + // building an applications with variadic functions wholly removed if one + // has sufficient control over the dependencies, e.g. a statically linked + // clang that has no variadic function calls remaining in the binary. + +public: + static char ID; + const ExpandVariadicsMode Mode; + std::unique_ptr ABI; + + ExpandVariadics(ExpandVariadicsMode Mode) + : ModulePass(ID), + Mode(commandLineOverride() ? ExpandVariadicsModeOption : Mode) {} + + StringRef getPassName() const override { return "Expand variadic functions"; } + + bool rewriteABI() { return Mode == ExpandVariadicsMode::Lowering; } + + bool runOnModule(Module &M) override; + + bool runOnFunction(Module &M, IRBuilder<> &Builder, Function *F); + + Function *replaceAllUsesWithNewDeclaration(Module &M, + Function *OriginalFunction); + + Function *deriveFixedArityReplacement(Module &M, IRBuilder<> &Builder, + Function *OriginalFunction); + + Function *defineVariadicWrapper(Module &M, IRBuilder<> &Builder, + Function *VariadicWrapper, + Function *FixedArityReplacement); + + bool expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB, FunctionType *, + Function *NF); + + // The intrinsic functions va_copy and va_end are removed unconditionally. + // They correspond to a memcpy and a no-op on all implemented targets. + // The va_start intrinsic is removed from basic blocks that were not created + // by this pass, some may remain if needed to maintain the external ABI. + + template + bool expandIntrinsicUsers(Module &M, IRBuilder<> &Builder, + PointerType *IntrinsicArgType) { + bool Changed = false; + const DataLayout &DL = M.getDataLayout(); + if (Function *Intrinsic = + getPreexistingDeclaration(&M, ID, {IntrinsicArgType})) { + for (User *U : make_early_inc_range(Intrinsic->users())) + if (auto *I = dyn_cast(U)) + Changed |= expandVAIntrinsicCall(Builder, DL, I); + + if (Intrinsic->use_empty()) + Intrinsic->eraseFromParent(); + } + return Changed; + } + + bool expandVAIntrinsicUsersWithAddrspace(Module &M, IRBuilder<> &Builder, + unsigned Addrspace) { + auto &Ctx = M.getContext(); + PointerType *IntrinsicArgType = PointerType::get(Ctx, Addrspace); + bool Changed = false; + + // expand vastart before vacopy as vastart may introduce a vacopy + Changed |= expandIntrinsicUsers( + M, Builder, IntrinsicArgType); + Changed |= expandIntrinsicUsers( + M, Builder, IntrinsicArgType); + Changed |= expandIntrinsicUsers( + M, Builder, IntrinsicArgType); + return Changed; + } + + bool expandVAIntrinsicCall(IRBuilder<> &Builder, const DataLayout &DL, + VAStartInst *Inst); + + bool expandVAIntrinsicCall(IRBuilder<> &, const DataLayout &, + VAEndInst *Inst); + + bool expandVAIntrinsicCall(IRBuilder<> &Builder, const DataLayout &DL, + VACopyInst *Inst); + + FunctionType *inlinableVariadicFunctionType(Module &M, FunctionType *FTy) { + // The type of "FTy" with the ... removed and a va_list appended + SmallVector ArgTypes(FTy->param_begin(), FTy->param_end()); + ArgTypes.push_back(ABI->vaListParameterType(M)); + return FunctionType::get(FTy->getReturnType(), ArgTypes, + /*IsVarArgs=*/false); + } + + static ConstantInt *sizeOfAlloca(LLVMContext &Ctx, const DataLayout &DL, + AllocaInst *Alloced) { + std::optional AllocaTypeSize = Alloced->getAllocationSize(DL); + uint64_t AsInt = AllocaTypeSize ? AllocaTypeSize->getFixedValue() : 0; + return ConstantInt::get(Type::getInt64Ty(Ctx), AsInt); + } + + bool expansionApplicableToFunction(Module &M, Function *F) { + if (F->isIntrinsic() || !F->isVarArg() || + F->hasFnAttribute(Attribute::Naked)) + return false; + + if (F->getCallingConv() != CallingConv::C) + return false; + + if (rewriteABI()) + return true; + + if (!F->hasExactDefinition()) + return false; + + return true; + } + + bool expansionApplicableToFunctionCall(CallBase *CB) { + if (CallInst *CI = dyn_cast(CB)) { + if (CI->isMustTailCall()) { + // Cannot expand musttail calls + return false; + } + + if (CI->getCallingConv() != CallingConv::C) + return false; + + return true; + } + + if (isa(CB)) { + // Invoke not implemented in initial implementation of pass + return false; + } + + // Other unimplemented derivative of CallBase + return false; + } + + class ExpandedCallFrame { + // Helper for constructing an alloca instance containing the arguments bound + // to the variadic ... parameter, rearranged to allow indexing through a + // va_list iterator + enum { N = 4 }; + SmallVector FieldTypes; + enum Tag { Store, Memcpy, Padding }; + SmallVector, N> Source; + + template void append(Type *FieldType, Value *V, uint64_t Bytes) { + FieldTypes.push_back(FieldType); + Source.push_back({V, Bytes, tag}); + } + + public: + void store(LLVMContext &Ctx, Type *T, Value *V) { append(T, V, 0); } + + void memcpy(LLVMContext &Ctx, Type *T, Value *V, uint64_t Bytes) { + append(T, V, Bytes); + } + + void padding(LLVMContext &Ctx, uint64_t By) { + append(ArrayType::get(Type::getInt8Ty(Ctx), By), nullptr, 0); + } + + size_t size() const { return FieldTypes.size(); } + bool empty() const { return FieldTypes.empty(); } + + StructType *asStruct(LLVMContext &Ctx, StringRef Name) { + const bool IsPacked = true; + return StructType::create(Ctx, FieldTypes, + (Twine(Name) + ".vararg").str(), IsPacked); + } + + void initializeStructAlloca(const DataLayout &DL, IRBuilder<> &Builder, + AllocaInst *Alloced) { + + StructType *VarargsTy = cast(Alloced->getAllocatedType()); + + for (size_t I = 0; I < size(); I++) { + + auto [V, bytes, tag] = Source[I]; + + if (tag == Padding) { + assert(V == nullptr); + continue; + } + + auto Dst = Builder.CreateStructGEP(VarargsTy, Alloced, I); + + assert(V != nullptr); + + if (tag == Store) + Builder.CreateStore(V, Dst); + + if (tag == Memcpy) + Builder.CreateMemCpy(Dst, {}, V, {}, bytes); + } + } + }; +}; + +bool ExpandVariadics::runOnModule(Module &M) { + bool Changed = false; + if (Mode == ExpandVariadicsMode::Disable) + return Changed; + + Triple TT(M.getTargetTriple()); + ABI = VariadicABIInfo::create(TT); + if (!ABI) + return Changed; + + if (!ABI->enableForTarget()) + return Changed; + + auto &Ctx = M.getContext(); + const DataLayout &DL = M.getDataLayout(); + IRBuilder<> Builder(Ctx); + + // Lowering needs to run on all functions exactly once. + // Optimize could run on functions containing va_start exactly once. + for (Function &F : make_early_inc_range(M)) + Changed |= runOnFunction(M, Builder, &F); + + // After runOnFunction, all known calls to known variadic functions have been + // replaced. va_start intrinsics are presently (and invalidly!) only present + // in functions that used to be variadic and have now been replaced to take a + // va_list instead. If lowering as opposed to optimising, calls to unknown + // variadic functions have also been replaced. + + { + // 0 and AllocaAddrSpace are sufficient for the targets implemented so far + unsigned Addrspace = 0; + Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace); + + Addrspace = DL.getAllocaAddrSpace(); + if (Addrspace != 0) + Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace); + } + + if (Mode != ExpandVariadicsMode::Lowering) + return Changed; + + for (Function &F : make_early_inc_range(M)) { + if (F.isDeclaration()) + continue; + + // Now need to track down indirect calls. Can't find those + // by walking uses of variadic functions, need to crawl the instruction + // stream. Fortunately this is only necessary for the ABI rewrite case. + for (BasicBlock &BB : F) { + for (Instruction &I : make_early_inc_range(BB)) { + if (CallBase *CB = dyn_cast(&I)) { + if (CB->isIndirectCall()) { + FunctionType *FTy = CB->getFunctionType(); + if (FTy->isVarArg()) + Changed |= expandCall(M, Builder, CB, FTy, 0); + } + } + } + } + } + + return Changed; +} + +bool ExpandVariadics::runOnFunction(Module &M, IRBuilder<> &Builder, + Function *OriginalFunction) { + bool Changed = false; + + if (!expansionApplicableToFunction(M, OriginalFunction)) + return Changed; + + const bool OriginalFunctionIsDeclaration = OriginalFunction->isDeclaration(); + assert(rewriteABI() || !OriginalFunctionIsDeclaration); + + // Declare a new function and redirect every use to that new function + Function *VariadicWrapper = + replaceAllUsesWithNewDeclaration(M, OriginalFunction); + assert(VariadicWrapper->isDeclaration()); + assert(OriginalFunction->use_empty()); + + // Create a new function taking va_list containing the implementation of the + // original + Function *FixedArityReplacement = + deriveFixedArityReplacement(M, Builder, OriginalFunction); + assert(OriginalFunction->isDeclaration()); + assert(FixedArityReplacement->isDeclaration() == + OriginalFunctionIsDeclaration); + assert(VariadicWrapper->isDeclaration()); + + // Create a single block forwarding wrapper that turns a ... into a va_list + Function *VariadicWrapperDefine = + defineVariadicWrapper(M, Builder, VariadicWrapper, FixedArityReplacement); + assert(VariadicWrapperDefine == VariadicWrapper); + assert(!VariadicWrapper->isDeclaration()); + + // We now have: + // 1. the original function, now as a declaration with no uses + // 2. a variadic function that unconditionally calls a fixed arity replacement + // 3. a fixed arity function equivalent to the original function + + // Replace known calls to the variadic with calls to the va_list equivalent + for (User *U : make_early_inc_range(VariadicWrapper->users())) { + if (CallBase *CB = dyn_cast(U)) { + Value *calledOperand = CB->getCalledOperand(); + if (VariadicWrapper == calledOperand) + Changed |= + expandCall(M, Builder, CB, VariadicWrapper->getFunctionType(), + FixedArityReplacement); + } + } + + // The original function will be erased. + // One of the two new functions will become a replacement for the original. + // When preserving the ABI, the other is an internal implementation detail. + // When rewriting the ABI, RAUW then the variadic one. + Function *const ExternallyAccessible = + rewriteABI() ? FixedArityReplacement : VariadicWrapper; + Function *const InternalOnly = + rewriteABI() ? VariadicWrapper : FixedArityReplacement; + + // The external function is the replacement for the original + ExternallyAccessible->setLinkage(OriginalFunction->getLinkage()); + ExternallyAccessible->setVisibility(OriginalFunction->getVisibility()); + ExternallyAccessible->setComdat(OriginalFunction->getComdat()); + ExternallyAccessible->takeName(OriginalFunction); + + // Annotate the internal one as internal + InternalOnly->setVisibility(GlobalValue::DefaultVisibility); + InternalOnly->setLinkage(GlobalValue::InternalLinkage); + + // The original is unused and obsolete + OriginalFunction->eraseFromParent(); + + InternalOnly->removeDeadConstantUsers(); + + if (rewriteABI()) { + // All known calls to the function have been removed by expandCall + // Resolve everything else by replaceAllUsesWith + VariadicWrapper->replaceAllUsesWith(FixedArityReplacement); + VariadicWrapper->eraseFromParent(); + } + + return Changed; +} + +Function * +ExpandVariadics::replaceAllUsesWithNewDeclaration(Module &M, + Function *OriginalFunction) { + auto &Ctx = M.getContext(); + Function &F = *OriginalFunction; + FunctionType *FTy = F.getFunctionType(); + Function *NF = Function::Create(FTy, F.getLinkage(), F.getAddressSpace()); + + NF->setName(F.getName() + ".varargs"); + NF->IsNewDbgInfoFormat = F.IsNewDbgInfoFormat; + + F.getParent()->getFunctionList().insert(F.getIterator(), NF); + + AttrBuilder ParamAttrs(Ctx); + AttributeList Attrs = NF->getAttributes(); + Attrs = Attrs.addParamAttributes(Ctx, FTy->getNumParams(), ParamAttrs); + NF->setAttributes(Attrs); + + OriginalFunction->replaceAllUsesWith(NF); + return NF; +} + +Function * +ExpandVariadics::deriveFixedArityReplacement(Module &M, IRBuilder<> &Builder, + Function *OriginalFunction) { + Function &F = *OriginalFunction; + // The purpose here is split the variadic function F into two functions + // One is a variadic function that bundles the passed argument into a va_list + // and passes it to the second function. The second function does whatever + // the original F does, except that it takes a va_list instead of the ... + + assert(expansionApplicableToFunction(M, &F)); + + auto &Ctx = M.getContext(); + + // Returned value isDeclaration() is equal to F.isDeclaration() + // but that property is not invariant throughout this function + const bool FunctionIsDefinition = !F.isDeclaration(); + + FunctionType *FTy = F.getFunctionType(); + SmallVector ArgTypes(FTy->param_begin(), FTy->param_end()); + ArgTypes.push_back(ABI->vaListParameterType(M)); + + FunctionType *NFTy = inlinableVariadicFunctionType(M, FTy); + Function *NF = Function::Create(NFTy, F.getLinkage(), F.getAddressSpace()); + + // Note - same attribute handling as DeadArgumentElimination + NF->copyAttributesFrom(&F); + NF->setComdat(F.getComdat()); + F.getParent()->getFunctionList().insert(F.getIterator(), NF); + NF->setName(F.getName() + ".valist"); + NF->IsNewDbgInfoFormat = F.IsNewDbgInfoFormat; + + AttrBuilder ParamAttrs(Ctx); + + AttributeList Attrs = NF->getAttributes(); + Attrs = Attrs.addParamAttributes(Ctx, NFTy->getNumParams() - 1, ParamAttrs); + NF->setAttributes(Attrs); + + // Splice the implementation into the new function with minimal changes + if (FunctionIsDefinition) { + NF->splice(NF->begin(), &F); + + auto NewArg = NF->arg_begin(); + for (Argument &Arg : F.args()) { + Arg.replaceAllUsesWith(NewArg); + NewArg->setName(Arg.getName()); // takeName without killing the old one + ++NewArg; + } + NewArg->setName("varargs"); + } + + SmallVector, 1> MDs; + F.getAllMetadata(MDs); + for (auto [KindID, Node] : MDs) + NF->addMetadata(KindID, *Node); + F.clearMetadata(); + + return NF; +} + +Function * +ExpandVariadics::defineVariadicWrapper(Module &M, IRBuilder<> &Builder, + Function *VariadicWrapper, + Function *FixedArityReplacement) { + auto &Ctx = Builder.getContext(); + const DataLayout &DL = M.getDataLayout(); + assert(VariadicWrapper->isDeclaration()); + Function &F = *VariadicWrapper; + + assert(F.isDeclaration()); + Type *VaListTy = ABI->vaListType(Ctx); + + auto *BB = BasicBlock::Create(Ctx, "entry", &F); + Builder.SetInsertPoint(BB); + + AllocaInst *VaListInstance = + Builder.CreateAlloca(VaListTy, nullptr, "va_start"); + + Builder.CreateLifetimeStart(VaListInstance, + sizeOfAlloca(Ctx, DL, VaListInstance)); + + Builder.CreateIntrinsic(Intrinsic::vastart, {DL.getAllocaPtrType(Ctx)}, + {VaListInstance}); + + SmallVector Args; + for (Argument &A : F.args()) + Args.push_back(&A); + + Type *ParameterType = ABI->vaListParameterType(M); + if (ABI->vaListPassedInSSARegister()) + Args.push_back(Builder.CreateLoad(ParameterType, VaListInstance)); + else + Args.push_back(Builder.CreateAddrSpaceCast(VaListInstance, ParameterType)); + + CallInst *Result = Builder.CreateCall(FixedArityReplacement, Args); + + Builder.CreateIntrinsic(Intrinsic::vaend, {DL.getAllocaPtrType(Ctx)}, + {VaListInstance}); + Builder.CreateLifetimeEnd(VaListInstance, + sizeOfAlloca(Ctx, DL, VaListInstance)); + + if (Result->getType()->isVoidTy()) + Builder.CreateRetVoid(); + else + Builder.CreateRet(Result); + + return VariadicWrapper; +} + +bool ExpandVariadics::expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB, + FunctionType *VarargFunctionType, + Function *NF) { + bool Changed = false; + const DataLayout &DL = M.getDataLayout(); + + if (!expansionApplicableToFunctionCall(CB)) { + if (rewriteABI()) + report_fatal_error("Cannot lower callbase instruction"); + return Changed; + } + + // This is tricky. The call instruction's function type might not match + // the type of the caller. When optimising, can leave it unchanged. + // Webassembly detects that inconsistency and repairs it. + FunctionType *FuncType = CB->getFunctionType(); + if (FuncType != VarargFunctionType) { + if (!rewriteABI()) + return Changed; + FuncType = VarargFunctionType; + } + + auto &Ctx = CB->getContext(); + + Align MaxFieldAlign(1); + + // The strategy is to allocate a call frame containing the variadic + // arguments laid out such that a target specific va_list can be initialized + // with it, such that target specific va_arg instructions will correctly + // iterate over it. This means getting the alignment right and sometimes + // embedding a pointer to the value instead of embedding the value itself. + + Function *CBF = CB->getParent()->getParent(); + + ExpandedCallFrame Frame; + + uint64_t CurrentOffset = 0; + + for (unsigned I = FuncType->getNumParams(), E = CB->arg_size(); I < E; ++I) { + Value *ArgVal = CB->getArgOperand(I); + const bool IsByVal = CB->paramHasAttr(I, Attribute::ByVal); + const bool IsByRef = CB->paramHasAttr(I, Attribute::ByRef); + + // The type of the value being passed, decoded from byval/byref metadata if + // required + Type *const UnderlyingType = IsByVal ? CB->getParamByValType(I) + : IsByRef ? CB->getParamByRefType(I) + : ArgVal->getType(); + const uint64_t UnderlyingSize = + DL.getTypeAllocSize(UnderlyingType).getFixedValue(); + + // The type to be written into the call frame + Type *FrameFieldType = UnderlyingType; + + // The value to copy from when initialising the frame alloca + Value *SourceValue = ArgVal; + + VariadicABIInfo::VAArgSlotInfo SlotInfo = ABI->slotInfo(DL, UnderlyingType); + + if (SlotInfo.Indirect) { + // The va_arg lowering loads through a pointer. Set up an alloca to aim + // that pointer at. + Builder.SetInsertPointPastAllocas(CBF); + Builder.SetCurrentDebugLocation(CB->getStableDebugLoc()); + Value *CallerCopy = + Builder.CreateAlloca(UnderlyingType, nullptr, "IndirectAlloca"); + + Builder.SetInsertPoint(CB); + if (IsByVal) + Builder.CreateMemCpy(CallerCopy, {}, ArgVal, {}, UnderlyingSize); + else + Builder.CreateStore(ArgVal, CallerCopy); + + // Indirection now handled, pass the alloca ptr by value + FrameFieldType = DL.getAllocaPtrType(Ctx); + SourceValue = CallerCopy; + } + + // Alignment of the value within the frame + // This probably needs to be controllable as a function of type + Align DataAlign = SlotInfo.DataAlign; + + MaxFieldAlign = std::max(MaxFieldAlign, DataAlign); + + uint64_t DataAlignV = DataAlign.value(); + if (uint64_t Rem = CurrentOffset % DataAlignV) { + // Inject explicit padding to deal with alignment requirements + uint64_t Padding = DataAlignV - Rem; + Frame.padding(Ctx, Padding); + CurrentOffset += Padding; + } + + if (SlotInfo.Indirect) { + Frame.store(Ctx, FrameFieldType, SourceValue); + } else { + if (IsByVal) + Frame.memcpy(Ctx, FrameFieldType, SourceValue, UnderlyingSize); + else + Frame.store(Ctx, FrameFieldType, SourceValue); + } + + CurrentOffset += DL.getTypeAllocSize(FrameFieldType).getFixedValue(); + } + + if (Frame.empty()) { + // Not passing any arguments, hopefully va_arg won't try to read any + // Creating a single byte frame containing nothing to point the va_list + // instance as that is less special-casey in the compiler and probably + // easier to interpret in a debugger. + Frame.padding(Ctx, 1); + } + + StructType *VarargsTy = Frame.asStruct(Ctx, CBF->getName()); + + // The struct instance needs to be at least MaxFieldAlign for the alignment of + // the fields to be correct at runtime. Use the native stack alignment instead + // if that's greater as that tends to give better codegen. + // This is an awkward way to guess whether there is a known stack alignment + // without hitting an assert in DL.getStackAlignment, 1024 is an arbitrary + // number likely to be greater than the natural stack alignment. + // TODO: DL.getStackAlignment could return a MaybeAlign instead of assert + Align AllocaAlign = MaxFieldAlign; + if (DL.exceedsNaturalStackAlignment(Align(1024))) + AllocaAlign = std::max(AllocaAlign, DL.getStackAlignment()); + + // Put the alloca to hold the variadic args in the entry basic block. + Builder.SetInsertPointPastAllocas(CBF); + + // SetCurrentDebugLocation when the builder SetInsertPoint method does not + Builder.SetCurrentDebugLocation(CB->getStableDebugLoc()); + + // The awkward construction here is to set the alignment on the instance + AllocaInst *Alloced = Builder.Insert( + new AllocaInst(VarargsTy, DL.getAllocaAddrSpace(), nullptr, AllocaAlign), + "vararg_buffer"); + Changed = true; + assert(Alloced->getAllocatedType() == VarargsTy); + + // Initialize the fields in the struct + Builder.SetInsertPoint(CB); + Builder.CreateLifetimeStart(Alloced, sizeOfAlloca(Ctx, DL, Alloced)); + Frame.initializeStructAlloca(DL, Builder, Alloced); + + const unsigned NumArgs = FuncType->getNumParams(); + SmallVector Args(CB->arg_begin(), CB->arg_begin() + NumArgs); + + // Initialize a va_list pointing to that struct and pass it as the last + // argument + AllocaInst *VaList = nullptr; + { + if (!ABI->vaListPassedInSSARegister()) { + Type *VaListTy = ABI->vaListType(Ctx); + Builder.SetInsertPointPastAllocas(CBF); + Builder.SetCurrentDebugLocation(CB->getStableDebugLoc()); + VaList = Builder.CreateAlloca(VaListTy, nullptr, "va_argument"); + Builder.SetInsertPoint(CB); + Builder.CreateLifetimeStart(VaList, sizeOfAlloca(Ctx, DL, VaList)); + } + Builder.SetInsertPoint(CB); + Args.push_back(ABI->initializeVaList(M, Ctx, Builder, VaList, Alloced)); + } + + // Attributes excluding any on the vararg arguments + AttributeList PAL = CB->getAttributes(); + if (!PAL.isEmpty()) { + SmallVector ArgAttrs; + for (unsigned ArgNo = 0; ArgNo < NumArgs; ArgNo++) + ArgAttrs.push_back(PAL.getParamAttrs(ArgNo)); + PAL = + AttributeList::get(Ctx, PAL.getFnAttrs(), PAL.getRetAttrs(), ArgAttrs); + } + + SmallVector OpBundles; + CB->getOperandBundlesAsDefs(OpBundles); + + CallBase *NewCB = nullptr; + + if (CallInst *CI = dyn_cast(CB)) { + Value *Dst = NF ? NF : CI->getCalledOperand(); + FunctionType *NFTy = inlinableVariadicFunctionType(M, VarargFunctionType); + + NewCB = CallInst::Create(NFTy, Dst, Args, OpBundles, "", CI); + + CallInst::TailCallKind TCK = CI->getTailCallKind(); + assert(TCK != CallInst::TCK_MustTail); + + // Can't tail call a function that is being passed a pointer to an alloca + if (TCK == CallInst::TCK_Tail) + TCK = CallInst::TCK_None; + CI->setTailCallKind(TCK); + + } else { + llvm_unreachable("Unreachable when !expansionApplicableToFunctionCall()"); + } + + if (VaList) + Builder.CreateLifetimeEnd(VaList, sizeOfAlloca(Ctx, DL, VaList)); + + Builder.CreateLifetimeEnd(Alloced, sizeOfAlloca(Ctx, DL, Alloced)); + + NewCB->setAttributes(PAL); + NewCB->takeName(CB); + NewCB->setCallingConv(CB->getCallingConv()); + NewCB->setDebugLoc(DebugLoc()); + + // DeadArgElim and ArgPromotion copy exactly this metadata + NewCB->copyMetadata(*CB, {LLVMContext::MD_prof, LLVMContext::MD_dbg}); + + CB->replaceAllUsesWith(NewCB); + CB->eraseFromParent(); + return Changed; +} + +bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &Builder, + const DataLayout &DL, + VAStartInst *Inst) { + // Only removing va_start instructions that are not in variadic functions. + // Those would be rejected by the IR verifier before this pass. + // After splicing basic blocks from a variadic function into a fixed arity + // one the va_start that used to refer to the ... parameter still exist. + // There are also variadic functions that this pass did not change and + // va_start instances in the created single block wrapper functions. + // Replace exactly the instances in non-variadic functions as those are + // the ones to be fixed up to use the va_list passed as the final argument. + + Function *ContainingFunction = Inst->getFunction(); + if (ContainingFunction->isVarArg()) { + return false; + } + + // The last argument is a vaListParameterType, either a va_list + // or a pointer to one depending on the target. + bool PassedByValue = ABI->vaListPassedInSSARegister(); + Argument *PassedVaList = + ContainingFunction->getArg(ContainingFunction->arg_size() - 1); + + // va_start takes a pointer to a va_list, e.g. one on the stack + Value *VaStartArg = Inst->getArgList(); + + Builder.SetInsertPoint(Inst); + + if (PassedByValue) { + // The general thing to do is create an alloca, store the va_list argument + // to it, then create a va_copy. When vaCopyIsMemcpy(), this optimises to a + // store to the VaStartArg. + assert(ABI->vaCopyIsMemcpy()); + Builder.CreateStore(PassedVaList, VaStartArg); + } else { + + // Otherwise emit a vacopy to pick up target-specific handling if any + auto &Ctx = Builder.getContext(); + + Builder.CreateIntrinsic(Intrinsic::vacopy, {DL.getAllocaPtrType(Ctx)}, + {VaStartArg, PassedVaList}); + } + + Inst->eraseFromParent(); + return true; +} + +bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &, const DataLayout &, + VAEndInst *Inst) { + assert(ABI->vaEndIsNop()); + Inst->eraseFromParent(); + return true; +} + +bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &Builder, + const DataLayout &DL, + VACopyInst *Inst) { + assert(ABI->vaCopyIsMemcpy()); + Builder.SetInsertPoint(Inst); + + auto &Ctx = Builder.getContext(); + Type *VaListTy = ABI->vaListType(Ctx); + uint64_t Size = DL.getTypeAllocSize(VaListTy).getFixedValue(); + + Builder.CreateMemCpy(Inst->getDest(), {}, Inst->getSrc(), {}, + Builder.getInt32(Size)); + + Inst->eraseFromParent(); + return true; +} + +struct Amdgpu final : public VariadicABIInfo { + + bool enableForTarget() override { return true; } + + bool vaListPassedInSSARegister() override { return true; } + + Type *vaListType(LLVMContext &Ctx) override { + return PointerType::getUnqual(Ctx); + } + + Type *vaListParameterType(Module &M) override { + return PointerType::getUnqual(M.getContext()); + } + + Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder, + AllocaInst * /*va_list*/, Value *Buffer) override { + // Given Buffer, which is an AllocInst of vararg_buffer + // need to return something usable as parameter type + return Builder.CreateAddrSpaceCast(Buffer, vaListParameterType(M)); + } + + VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override { + return {Align(4), false}; + } +}; + +struct Wasm final : public VariadicABIInfo { + + bool enableForTarget() override { + // Currently wasm is only used for testing. + return commandLineOverride(); + } + + bool vaListPassedInSSARegister() override { return true; } + + Type *vaListType(LLVMContext &Ctx) override { + return PointerType::getUnqual(Ctx); + } + + Type *vaListParameterType(Module &M) override { + return PointerType::getUnqual(M.getContext()); + } + + Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder, + AllocaInst * /*va_list*/, Value *Buffer) override { + return Buffer; + } + + VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override { + LLVMContext &Ctx = Parameter->getContext(); + const unsigned MinAlign = 4; + Align A = DL.getABITypeAlign(Parameter); + if (A < MinAlign) + A = Align(MinAlign); + + if (auto s = dyn_cast(Parameter)) { + if (s->getNumElements() > 1) { + return {DL.getABITypeAlign(PointerType::getUnqual(Ctx)), true}; + } + } + + return {A, false}; + } +}; + +std::unique_ptr VariadicABIInfo::create(const Triple &T) { + switch (T.getArch()) { + case Triple::r600: + case Triple::amdgcn: { + return std::make_unique(); + } + + case Triple::wasm32: { + return std::make_unique(); + } + + default: + return {}; + } +} + +} // namespace + +char ExpandVariadics::ID = 0; + +INITIALIZE_PASS(ExpandVariadics, DEBUG_TYPE, "Expand variadic functions", false, + false) + +ModulePass *llvm::createExpandVariadicsPass(ExpandVariadicsMode M) { + return new ExpandVariadics(M); +} + +PreservedAnalyses ExpandVariadicsPass::run(Module &M, ModuleAnalysisManager &) { + return ExpandVariadics(Mode).runOnModule(M) ? PreservedAnalyses::none() + : PreservedAnalyses::all(); +} + +ExpandVariadicsPass::ExpandVariadicsPass(ExpandVariadicsMode M) : Mode(M) {} diff --git a/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll b/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll new file mode 100644 index 000000000000..ce55558dabaf --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll @@ -0,0 +1,545 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --function-signature +; RUN: opt -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s +; REQUIRES: amdgpu-registered-target +target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9" +target triple = "amdgcn-amd-amdhsa" + +; Check the variables are lowered to the locations this target expects + +; The types show the call frames +; CHECK: %single_i32.vararg = type <{ i32 }> +; CHECK: %single_double.vararg = type <{ double }> +; CHECK: %single_v4f32.vararg = type <{ <4 x float> }> +; CHECK: %single_v8f32.vararg = type <{ <8 x float> }> +; CHECK: %single_v16f32.vararg = type <{ <16 x float> }> +; CHECK: %single_v32f32.vararg = type <{ <32 x float> }> +; CHECK: %i32_double.vararg = type <{ i32, double }> +; CHECK: %double_i32.vararg = type <{ double, i32 }> +; CHECK: %i32_libcS.vararg = type <{ i32, %struct.libcS }> +; CHECK: %libcS_i32.vararg = type <{ %struct.libcS, i32 }> +; CHECK: %i32_v4f32.vararg = type <{ i32, <4 x float> }> +; CHECK: %v4f32_i32.vararg = type <{ <4 x float>, i32 }> +; CHECK: %i32_v8f32.vararg = type <{ i32, <8 x float> }> +; CHECK: %v8f32_i32.vararg = type <{ <8 x float>, i32 }> +; CHECK: %i32_v16f32.vararg = type <{ i32, <16 x float> }> +; CHECK: %v16f32_i32.vararg = type <{ <16 x float>, i32 }> +; CHECK: %i32_v32f32.vararg = type <{ i32, <32 x float> }> +; CHECK: %v32f32_i32.vararg = type <{ <32 x float>, i32 }> +; CHECK: %fptr_single_i32.vararg = type <{ i32 }> +; CHECK: %fptr_libcS.vararg = type <{ %struct.libcS }> + +%struct.libcS = type { i8, i16, i32, i64, float, double } + +@vararg_ptr = hidden addrspace(1) global ptr @vararg, align 8 + +define hidden void @copy(ptr noundef %va) { +; CHECK-LABEL: define {{[^@]+}}@copy(ptr noundef %va) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %va.addr = alloca ptr, align 8, addrspace(5) +; CHECK-NEXT: %cp = alloca ptr, align 8, addrspace(5) +; CHECK-NEXT: %va.addr.ascast = addrspacecast ptr addrspace(5) %va.addr to ptr +; CHECK-NEXT: %cp.ascast = addrspacecast ptr addrspace(5) %cp to ptr +; CHECK-NEXT: store ptr %va, ptr addrspace(5) %va.addr, align 8 +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %cp) +; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr %cp.ascast, ptr %va.addr.ascast, i32 8, i1 false) +; CHECK-NEXT: %0 = load ptr, ptr addrspace(5) %cp, align 8 +; CHECK-NEXT: call void @valist(ptr noundef %0) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %cp) +; CHECK-NEXT: ret void +; +entry: + %va.addr = alloca ptr, align 8, addrspace(5) + %cp = alloca ptr, align 8, addrspace(5) + %va.addr.ascast = addrspacecast ptr addrspace(5) %va.addr to ptr + %cp.ascast = addrspacecast ptr addrspace(5) %cp to ptr + store ptr %va, ptr addrspace(5) %va.addr, align 8 + call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %cp) + call void @llvm.va_copy.p0(ptr %cp.ascast, ptr nonnull %va.addr.ascast) + %0 = load ptr, ptr addrspace(5) %cp, align 8 + call void @valist(ptr noundef %0) + call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %cp) + ret void +} + +declare void @llvm.lifetime.start.p5(i64 immarg, ptr addrspace(5) nocapture) + +declare void @llvm.va_copy.p0(ptr, ptr) + +declare hidden void @valist(ptr noundef) + +declare void @llvm.lifetime.end.p5(i64 immarg, ptr addrspace(5) nocapture) + +define hidden void @start_once(...) { +; CHECK-LABEL: define {{[^@]+}}@start_once(ptr %varargs) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %s = alloca ptr, align 8, addrspace(5) +; CHECK-NEXT: %s.ascast = addrspacecast ptr addrspace(5) %s to ptr +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s) +; CHECK-NEXT: store ptr %varargs, ptr %s.ascast, align 8 +; CHECK-NEXT: %0 = load ptr, ptr addrspace(5) %s, align 8 +; CHECK-NEXT: call void @valist(ptr noundef %0) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s) +; CHECK-NEXT: ret void +; +entry: + %s = alloca ptr, align 8, addrspace(5) + %s.ascast = addrspacecast ptr addrspace(5) %s to ptr + call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s) + call void @llvm.va_start.p0(ptr %s.ascast) + %0 = load ptr, ptr addrspace(5) %s, align 8 + call void @valist(ptr noundef %0) + call void @llvm.va_end.p0(ptr %s.ascast) + call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s) + ret void +} + +declare void @llvm.va_start.p0(ptr) + +declare void @llvm.va_end.p0(ptr) + +define hidden void @start_twice(...) { +; CHECK-LABEL: define {{[^@]+}}@start_twice(ptr %varargs) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %s0 = alloca ptr, align 8, addrspace(5) +; CHECK-NEXT: %s1 = alloca ptr, align 8, addrspace(5) +; CHECK-NEXT: %s0.ascast = addrspacecast ptr addrspace(5) %s0 to ptr +; CHECK-NEXT: %s1.ascast = addrspacecast ptr addrspace(5) %s1 to ptr +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s0) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s1) +; CHECK-NEXT: store ptr %varargs, ptr %s0.ascast, align 8 +; CHECK-NEXT: %0 = load ptr, ptr addrspace(5) %s0, align 8 +; CHECK-NEXT: call void @valist(ptr noundef %0) +; CHECK-NEXT: store ptr %varargs, ptr %s1.ascast, align 8 +; CHECK-NEXT: %1 = load ptr, ptr addrspace(5) %s1, align 8 +; CHECK-NEXT: call void @valist(ptr noundef %1) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s1) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s0) +; CHECK-NEXT: ret void +; +entry: + %s0 = alloca ptr, align 8, addrspace(5) + %s1 = alloca ptr, align 8, addrspace(5) + %s0.ascast = addrspacecast ptr addrspace(5) %s0 to ptr + %s1.ascast = addrspacecast ptr addrspace(5) %s1 to ptr + call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s0) + call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s1) + call void @llvm.va_start.p0(ptr %s0.ascast) + %0 = load ptr, ptr addrspace(5) %s0, align 8 + call void @valist(ptr noundef %0) + call void @llvm.va_end.p0(ptr %s0.ascast) + call void @llvm.va_start.p0(ptr %s1.ascast) + %1 = load ptr, ptr addrspace(5) %s1, align 8 + call void @valist(ptr noundef %1) + call void @llvm.va_end.p0(ptr %s1.ascast) + call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s1) + call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s0) + ret void +} + +define hidden void @single_i32(i32 noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@single_i32(i32 noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %single_i32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %single_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr addrspace(5) %0, align 4 +; CHECK-NEXT: %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %1) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(i32 noundef %x) + ret void +} + +declare hidden void @vararg(...) + +define hidden void @single_double(double noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@single_double(double noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %single_double.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %single_double.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store double %x, ptr addrspace(5) %0, align 8 +; CHECK-NEXT: %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %1) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(double noundef %x) + ret void +} + +define hidden void @single_v4f32(<4 x float> noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@single_v4f32(<4 x float> noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %single_v4f32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 16, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %single_v4f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <4 x float> %x, ptr addrspace(5) %0, align 16 +; CHECK-NEXT: %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %1) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 16, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<4 x float> noundef %x) + ret void +} + +define hidden void @single_v8f32(<8 x float> noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@single_v8f32(<8 x float> noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %single_v8f32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 32, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %single_v8f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <8 x float> %x, ptr addrspace(5) %0, align 32 +; CHECK-NEXT: %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %1) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 32, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<8 x float> noundef %x) + ret void +} + +define hidden void @single_v16f32(<16 x float> noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@single_v16f32(<16 x float> noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %single_v16f32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 64, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %single_v16f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <16 x float> %x, ptr addrspace(5) %0, align 64 +; CHECK-NEXT: %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %1) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 64, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<16 x float> noundef %x) + ret void +} + +define hidden void @single_v32f32(<32 x float> noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@single_v32f32(<32 x float> noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %single_v32f32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 128, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %single_v32f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <32 x float> %x, ptr addrspace(5) %0, align 128 +; CHECK-NEXT: %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %1) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 128, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<32 x float> noundef %x) + ret void +} + +define hidden void @i32_double(i32 noundef %x, double noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@i32_double(i32 noundef %x, double noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %i32_double.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 12, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %i32_double.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr addrspace(5) %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %i32_double.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store double %y, ptr addrspace(5) %1, align 8 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 12, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(i32 noundef %x, double noundef %y) + ret void +} + +define hidden void @double_i32(double noundef %x, i32 noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@double_i32(double noundef %x, i32 noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %double_i32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 12, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %double_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store double %x, ptr addrspace(5) %0, align 8 +; CHECK-NEXT: %1 = getelementptr inbounds %double_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store i32 %y, ptr addrspace(5) %1, align 4 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 12, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(double noundef %x, i32 noundef %y) + ret void +} + +define hidden void @i32_libcS(i32 noundef %x, i8 %y.coerce0, i16 %y.coerce1, i32 %y.coerce2, i64 %y.coerce3, float %y.coerce4, double %y.coerce5) { +; CHECK-LABEL: define {{[^@]+}}@i32_libcS(i32 noundef %x, i8 %y.coerce0, i16 %y.coerce1, i32 %y.coerce2, i64 %y.coerce3, float %y.coerce4, double %y.coerce5) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %i32_libcS.vararg, align 4, addrspace(5) +; CHECK-NEXT: %.fca.0.insert = insertvalue %struct.libcS poison, i8 %y.coerce0, 0 +; CHECK-NEXT: %.fca.1.insert = insertvalue %struct.libcS %.fca.0.insert, i16 %y.coerce1, 1 +; CHECK-NEXT: %.fca.2.insert = insertvalue %struct.libcS %.fca.1.insert, i32 %y.coerce2, 2 +; CHECK-NEXT: %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %y.coerce3, 3 +; CHECK-NEXT: %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %y.coerce4, 4 +; CHECK-NEXT: %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %y.coerce5, 5 +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 36, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %i32_libcS.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr addrspace(5) %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %i32_libcS.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store %struct.libcS %.fca.5.insert, ptr addrspace(5) %1, align 8 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 36, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + %.fca.0.insert = insertvalue %struct.libcS poison, i8 %y.coerce0, 0 + %.fca.1.insert = insertvalue %struct.libcS %.fca.0.insert, i16 %y.coerce1, 1 + %.fca.2.insert = insertvalue %struct.libcS %.fca.1.insert, i32 %y.coerce2, 2 + %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %y.coerce3, 3 + %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %y.coerce4, 4 + %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %y.coerce5, 5 + tail call void (...) @vararg(i32 noundef %x, %struct.libcS %.fca.5.insert) + ret void +} + +define hidden void @libcS_i32(i8 %x.coerce0, i16 %x.coerce1, i32 %x.coerce2, i64 %x.coerce3, float %x.coerce4, double %x.coerce5, i32 noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@libcS_i32(i8 %x.coerce0, i16 %x.coerce1, i32 %x.coerce2, i64 %x.coerce3, float %x.coerce4, double %x.coerce5, i32 noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %libcS_i32.vararg, align 4, addrspace(5) +; CHECK-NEXT: %.fca.0.insert = insertvalue %struct.libcS poison, i8 %x.coerce0, 0 +; CHECK-NEXT: %.fca.1.insert = insertvalue %struct.libcS %.fca.0.insert, i16 %x.coerce1, 1 +; CHECK-NEXT: %.fca.2.insert = insertvalue %struct.libcS %.fca.1.insert, i32 %x.coerce2, 2 +; CHECK-NEXT: %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %x.coerce3, 3 +; CHECK-NEXT: %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %x.coerce4, 4 +; CHECK-NEXT: %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %x.coerce5, 5 +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 36, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %libcS_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store %struct.libcS %.fca.5.insert, ptr addrspace(5) %0, align 8 +; CHECK-NEXT: %1 = getelementptr inbounds %libcS_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store i32 %y, ptr addrspace(5) %1, align 4 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 36, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + %.fca.0.insert = insertvalue %struct.libcS poison, i8 %x.coerce0, 0 + %.fca.1.insert = insertvalue %struct.libcS %.fca.0.insert, i16 %x.coerce1, 1 + %.fca.2.insert = insertvalue %struct.libcS %.fca.1.insert, i32 %x.coerce2, 2 + %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %x.coerce3, 3 + %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %x.coerce4, 4 + %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %x.coerce5, 5 + tail call void (...) @vararg(%struct.libcS %.fca.5.insert, i32 noundef %y) + ret void +} + +define hidden void @i32_v4f32(i32 noundef %x, <4 x float> noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@i32_v4f32(i32 noundef %x, <4 x float> noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %i32_v4f32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 20, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %i32_v4f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr addrspace(5) %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %i32_v4f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store <4 x float> %y, ptr addrspace(5) %1, align 16 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 20, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(i32 noundef %x, <4 x float> noundef %y) + ret void +} + +define hidden void @v4f32_i32(<4 x float> noundef %x, i32 noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@v4f32_i32(<4 x float> noundef %x, i32 noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %v4f32_i32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 20, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %v4f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <4 x float> %x, ptr addrspace(5) %0, align 16 +; CHECK-NEXT: %1 = getelementptr inbounds %v4f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store i32 %y, ptr addrspace(5) %1, align 4 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 20, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<4 x float> noundef %x, i32 noundef %y) + ret void +} + +define hidden void @i32_v8f32(i32 noundef %x, <8 x float> noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@i32_v8f32(i32 noundef %x, <8 x float> noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %i32_v8f32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 36, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %i32_v8f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr addrspace(5) %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %i32_v8f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store <8 x float> %y, ptr addrspace(5) %1, align 32 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 36, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(i32 noundef %x, <8 x float> noundef %y) + ret void +} + +define hidden void @v8f32_i32(<8 x float> noundef %x, i32 noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@v8f32_i32(<8 x float> noundef %x, i32 noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %v8f32_i32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 36, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %v8f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <8 x float> %x, ptr addrspace(5) %0, align 32 +; CHECK-NEXT: %1 = getelementptr inbounds %v8f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store i32 %y, ptr addrspace(5) %1, align 4 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 36, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<8 x float> noundef %x, i32 noundef %y) + ret void +} + +define hidden void @i32_v16f32(i32 noundef %x, <16 x float> noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@i32_v16f32(i32 noundef %x, <16 x float> noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %i32_v16f32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 68, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %i32_v16f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr addrspace(5) %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %i32_v16f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store <16 x float> %y, ptr addrspace(5) %1, align 64 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 68, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(i32 noundef %x, <16 x float> noundef %y) + ret void +} + +define hidden void @v16f32_i32(<16 x float> noundef %x, i32 noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@v16f32_i32(<16 x float> noundef %x, i32 noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %v16f32_i32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 68, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %v16f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <16 x float> %x, ptr addrspace(5) %0, align 64 +; CHECK-NEXT: %1 = getelementptr inbounds %v16f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store i32 %y, ptr addrspace(5) %1, align 4 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 68, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<16 x float> noundef %x, i32 noundef %y) + ret void +} + +define hidden void @i32_v32f32(i32 noundef %x, <32 x float> noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@i32_v32f32(i32 noundef %x, <32 x float> noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %i32_v32f32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 132, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %i32_v32f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr addrspace(5) %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %i32_v32f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store <32 x float> %y, ptr addrspace(5) %1, align 128 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 132, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(i32 noundef %x, <32 x float> noundef %y) + ret void +} + +define hidden void @v32f32_i32(<32 x float> noundef %x, i32 noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@v32f32_i32(<32 x float> noundef %x, i32 noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %v32f32_i32.vararg, align 4, addrspace(5) +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 132, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %v32f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <32 x float> %x, ptr addrspace(5) %0, align 128 +; CHECK-NEXT: %1 = getelementptr inbounds %v32f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store i32 %y, ptr addrspace(5) %1, align 4 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void @vararg(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 132, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<32 x float> noundef %x, i32 noundef %y) + ret void +} + +define hidden void @fptr_single_i32(i32 noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@fptr_single_i32(i32 noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %fptr_single_i32.vararg, align 4, addrspace(5) +; CHECK-NEXT: %0 = load volatile ptr, ptr addrspacecast (ptr addrspace(1) @vararg_ptr to ptr), align 8 +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %1 = getelementptr inbounds %fptr_single_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr addrspace(5) %1, align 4 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void %0(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + %0 = load volatile ptr, ptr addrspacecast (ptr addrspace(1) @vararg_ptr to ptr), align 8 + tail call void (...) %0(i32 noundef %x) + ret void +} + +define hidden void @fptr_libcS(i8 %x.coerce0, i16 %x.coerce1, i32 %x.coerce2, i64 %x.coerce3, float %x.coerce4, double %x.coerce5) { +; CHECK-LABEL: define {{[^@]+}}@fptr_libcS(i8 %x.coerce0, i16 %x.coerce1, i32 %x.coerce2, i64 %x.coerce3, float %x.coerce4, double %x.coerce5) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %fptr_libcS.vararg, align 4, addrspace(5) +; CHECK-NEXT: %0 = load volatile ptr, ptr addrspacecast (ptr addrspace(1) @vararg_ptr to ptr), align 8 +; CHECK-NEXT: %.fca.0.insert = insertvalue %struct.libcS poison, i8 %x.coerce0, 0 +; CHECK-NEXT: %.fca.1.insert = insertvalue %struct.libcS %.fca.0.insert, i16 %x.coerce1, 1 +; CHECK-NEXT: %.fca.2.insert = insertvalue %struct.libcS %.fca.1.insert, i32 %x.coerce2, 2 +; CHECK-NEXT: %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %x.coerce3, 3 +; CHECK-NEXT: %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %x.coerce4, 4 +; CHECK-NEXT: %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %x.coerce5, 5 +; CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 32, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: %1 = getelementptr inbounds %fptr_libcS.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store %struct.libcS %.fca.5.insert, ptr addrspace(5) %1, align 8 +; CHECK-NEXT: %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr +; CHECK-NEXT: call void %0(ptr %2) +; CHECK-NEXT: call void @llvm.lifetime.end.p5(i64 32, ptr addrspace(5) %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + %0 = load volatile ptr, ptr addrspacecast (ptr addrspace(1) @vararg_ptr to ptr), align 8 + %.fca.0.insert = insertvalue %struct.libcS poison, i8 %x.coerce0, 0 + %.fca.1.insert = insertvalue %struct.libcS %.fca.0.insert, i16 %x.coerce1, 1 + %.fca.2.insert = insertvalue %struct.libcS %.fca.1.insert, i32 %x.coerce2, 2 + %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %x.coerce3, 3 + %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %x.coerce4, 4 + %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %x.coerce5, 5 + tail call void (...) %0(%struct.libcS %.fca.5.insert) + ret void +} + + diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll index 0db88d1c095d..08cf83fd2bd0 100644 --- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll +++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll @@ -31,6 +31,7 @@ ; GCN-O0-NEXT: AMDGPU Remove Incompatible Functions ; GCN-O0-NEXT: AMDGPU Printf lowering ; GCN-O0-NEXT: Lower ctors and dtors for AMDGPU +; GCN-O0-NEXT: Expand variadic functions ; GCN-O0-NEXT: AMDGPU Inline All Functions ; GCN-O0-NEXT: Inliner for always_inline functions ; GCN-O0-NEXT: FunctionPass Manager @@ -178,6 +179,7 @@ ; GCN-O1-NEXT: AMDGPU Remove Incompatible Functions ; GCN-O1-NEXT: AMDGPU Printf lowering ; GCN-O1-NEXT: Lower ctors and dtors for AMDGPU +; GCN-O1-NEXT: Expand variadic functions ; GCN-O1-NEXT: AMDGPU Inline All Functions ; GCN-O1-NEXT: Inliner for always_inline functions ; GCN-O1-NEXT: FunctionPass Manager @@ -454,6 +456,7 @@ ; GCN-O1-OPTS-NEXT: AMDGPU Remove Incompatible Functions ; GCN-O1-OPTS-NEXT: AMDGPU Printf lowering ; GCN-O1-OPTS-NEXT: Lower ctors and dtors for AMDGPU +; GCN-O1-OPTS-NEXT: Expand variadic functions ; GCN-O1-OPTS-NEXT: AMDGPU Inline All Functions ; GCN-O1-OPTS-NEXT: Inliner for always_inline functions ; GCN-O1-OPTS-NEXT: FunctionPass Manager @@ -760,6 +763,7 @@ ; GCN-O2-NEXT: Lower ctors and dtors for AMDGPU ; GCN-O2-NEXT: FunctionPass Manager ; GCN-O2-NEXT: AMDGPU Image Intrinsic Optimizer +; GCN-O2-NEXT: Expand variadic functions ; GCN-O2-NEXT: AMDGPU Inline All Functions ; GCN-O2-NEXT: Inliner for always_inline functions ; GCN-O2-NEXT: FunctionPass Manager @@ -1070,6 +1074,7 @@ ; GCN-O3-NEXT: Lower ctors and dtors for AMDGPU ; GCN-O3-NEXT: FunctionPass Manager ; GCN-O3-NEXT: AMDGPU Image Intrinsic Optimizer +; GCN-O3-NEXT: Expand variadic functions ; GCN-O3-NEXT: AMDGPU Inline All Functions ; GCN-O3-NEXT: Inliner for always_inline functions ; GCN-O3-NEXT: FunctionPass Manager diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll b/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll index fc00937e6c8a..721114ece56d 100644 --- a/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll +++ b/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll @@ -43,25 +43,6 @@ define i32 @test_tail_call(ptr addrspace(1) %out, ptr addrspace(1) %in) { ret i32 %c } -declare void @external.varargs(i32, double, i64, ...) - -; GCN: error: :0:0: in function test_call_varargs void (): unsupported call to variadic function external.varargs -; R600: in function test_call_varargs{{.*}}: unsupported call to function external.varargs -define void @test_call_varargs() { - call void (i32, double, i64, ...) @external.varargs(i32 42, double 1.0, i64 12, i8 3, i16 1, i32 4, float 1.0, double 2.0) - ret void -} - -declare i32 @extern_variadic(...) - -; GCN: in function test_tail_call_bitcast_extern_variadic{{.*}}: unsupported required tail call to function extern_variadic -; R600: in function test_tail_call_bitcast_extern_variadic{{.*}}: unsupported call to function extern_variadic -define i32 @test_tail_call_bitcast_extern_variadic(<4 x float> %arg0, <4 x float> %arg1, i32 %arg2) { - %add = fadd <4 x float> %arg0, %arg1 - %call = tail call i32 @extern_variadic(<4 x float> %add) - ret i32 %call -} - ; R600: in function test_c_call{{.*}}: unsupported call to function defined_function define amdgpu_ps i32 @test_c_call_from_shader() { %call = call i32 @defined_function(i32 0) diff --git a/llvm/test/CodeGen/WebAssembly/expand-variadic-call.ll b/llvm/test/CodeGen/WebAssembly/expand-variadic-call.ll new file mode 100644 index 000000000000..80f3db0e52e0 --- /dev/null +++ b/llvm/test/CodeGen/WebAssembly/expand-variadic-call.ll @@ -0,0 +1,484 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --function-signature +; RUN: opt -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s +; REQUIRES: webassembly-registered-target +target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20" +target triple = "wasm32-unknown-unknown" + +; Check the variables are lowered to the locations this target expects + +; The types show the call frames +; CHECK: %single_i32.vararg = type <{ i32 }> +; CHECK: %single_double.vararg = type <{ double }> +; CHECK: %single_v4f32.vararg = type <{ <4 x float> }> +; CHECK: %single_v8f32.vararg = type <{ <8 x float> }> +; CHECK: %single_v16f32.vararg = type <{ <16 x float> }> +; CHECK: %single_v32f32.vararg = type <{ <32 x float> }> +; CHECK: %i32_double.vararg = type <{ i32, [4 x i8], double }> +; CHECK: %double_i32.vararg = type <{ double, i32 }> +; CHECK: %i32_libcS.vararg = type <{ i32, ptr }> +; CHECK: %libcS_i32.vararg = type <{ ptr, i32 }> +; CHECK: %i32_v4f32.vararg = type <{ i32, [12 x i8], <4 x float> }> +; CHECK: %v4f32_i32.vararg = type <{ <4 x float>, i32 }> +; CHECK: %i32_v8f32.vararg = type <{ i32, [28 x i8], <8 x float> }> +; CHECK: %v8f32_i32.vararg = type <{ <8 x float>, i32 }> +; CHECK: %i32_v16f32.vararg = type <{ i32, [60 x i8], <16 x float> }> +; CHECK: %v16f32_i32.vararg = type <{ <16 x float>, i32 }> +; CHECK: %i32_v32f32.vararg = type <{ i32, [124 x i8], <32 x float> }> +; CHECK: %v32f32_i32.vararg = type <{ <32 x float>, i32 }> +; CHECK: %fptr_single_i32.vararg = type <{ i32 }> +; CHECK: %fptr_libcS.vararg = type <{ ptr }> + +%struct.libcS = type { i8, i16, i32, i32, float, double } + +@vararg_ptr = hidden global ptr @vararg, align 4 + +define hidden void @copy(ptr noundef %va) { +; CHECK-LABEL: define {{[^@]+}}@copy(ptr noundef %va) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %va.addr = alloca ptr, align 4 +; CHECK-NEXT: %cp = alloca ptr, align 4 +; CHECK-NEXT: store ptr %va, ptr %va.addr, align 4 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %cp) +; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr %cp, ptr %va.addr, i32 4, i1 false) +; CHECK-NEXT: %0 = load ptr, ptr %cp, align 4 +; CHECK-NEXT: call void @valist(ptr noundef %0) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %cp) +; CHECK-NEXT: ret void +; +entry: + %va.addr = alloca ptr, align 4 + %cp = alloca ptr, align 4 + store ptr %va, ptr %va.addr, align 4 + call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %cp) + call void @llvm.va_copy.p0(ptr nonnull %cp, ptr nonnull %va.addr) + %0 = load ptr, ptr %cp, align 4 + call void @valist(ptr noundef %0) + call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %cp) + ret void +} + +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) + +declare void @llvm.va_copy.p0(ptr, ptr) + +declare void @valist(ptr noundef) + +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) + +define hidden void @start_once(...) { +; CHECK-LABEL: define {{[^@]+}}@start_once(ptr %varargs) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %s = alloca ptr, align 4 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s) +; CHECK-NEXT: store ptr %varargs, ptr %s, align 4 +; CHECK-NEXT: %0 = load ptr, ptr %s, align 4 +; CHECK-NEXT: call void @valist(ptr noundef %0) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s) +; CHECK-NEXT: ret void +; +entry: + %s = alloca ptr, align 4 + call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s) + call void @llvm.va_start.p0(ptr nonnull %s) + %0 = load ptr, ptr %s, align 4 + call void @valist(ptr noundef %0) + call void @llvm.va_end.p0(ptr %s) + call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s) + ret void +} + +declare void @llvm.va_start.p0(ptr) + +declare void @llvm.va_end.p0(ptr) + +define hidden void @start_twice(...) { +; CHECK-LABEL: define {{[^@]+}}@start_twice(ptr %varargs) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %s0 = alloca ptr, align 4 +; CHECK-NEXT: %s1 = alloca ptr, align 4 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s0) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s1) +; CHECK-NEXT: store ptr %varargs, ptr %s0, align 4 +; CHECK-NEXT: %0 = load ptr, ptr %s0, align 4 +; CHECK-NEXT: call void @valist(ptr noundef %0) +; CHECK-NEXT: store ptr %varargs, ptr %s1, align 4 +; CHECK-NEXT: %1 = load ptr, ptr %s1, align 4 +; CHECK-NEXT: call void @valist(ptr noundef %1) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s1) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s0) +; CHECK-NEXT: ret void +; +entry: + %s0 = alloca ptr, align 4 + %s1 = alloca ptr, align 4 + call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s0) + call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s1) + call void @llvm.va_start.p0(ptr nonnull %s0) + %0 = load ptr, ptr %s0, align 4 + call void @valist(ptr noundef %0) + call void @llvm.va_end.p0(ptr %s0) + call void @llvm.va_start.p0(ptr nonnull %s1) + %1 = load ptr, ptr %s1, align 4 + call void @valist(ptr noundef %1) + call void @llvm.va_end.p0(ptr %s1) + call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s1) + call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s0) + ret void +} + +define hidden void @single_i32(i32 noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@single_i32(i32 noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %single_i32.vararg, align 16 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %single_i32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr %0, align 4 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(i32 noundef %x) + ret void +} + +declare void @vararg(...) + +define hidden void @single_double(double noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@single_double(double noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %single_double.vararg, align 16 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %single_double.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store double %x, ptr %0, align 8 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(double noundef %x) + ret void +} + +define hidden void @single_v4f32(<4 x float> noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@single_v4f32(<4 x float> noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %single_v4f32.vararg, align 16 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %single_v4f32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <4 x float> %x, ptr %0, align 16 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<4 x float> noundef %x) + ret void +} + +define hidden void @single_v8f32(<8 x float> noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@single_v8f32(<8 x float> noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %single_v8f32.vararg, align 32 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 32, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %single_v8f32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <8 x float> %x, ptr %0, align 32 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 32, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<8 x float> noundef %x) + ret void +} + +define hidden void @single_v16f32(<16 x float> noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@single_v16f32(<16 x float> noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %single_v16f32.vararg, align 64 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 64, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %single_v16f32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <16 x float> %x, ptr %0, align 64 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<16 x float> noundef %x) + ret void +} + +define hidden void @single_v32f32(<32 x float> noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@single_v32f32(<32 x float> noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %single_v32f32.vararg, align 128 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 128, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %single_v32f32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <32 x float> %x, ptr %0, align 128 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 128, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<32 x float> noundef %x) + ret void +} + +define hidden void @i32_double(i32 noundef %x, double noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@i32_double(i32 noundef %x, double noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %i32_double.vararg, align 16 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %i32_double.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %i32_double.vararg, ptr %vararg_buffer, i32 0, i32 2 +; CHECK-NEXT: store double %y, ptr %1, align 8 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(i32 noundef %x, double noundef %y) + ret void +} + +define hidden void @double_i32(double noundef %x, i32 noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@double_i32(double noundef %x, i32 noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %double_i32.vararg, align 16 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 12, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %double_i32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store double %x, ptr %0, align 8 +; CHECK-NEXT: %1 = getelementptr inbounds %double_i32.vararg, ptr %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store i32 %y, ptr %1, align 4 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 12, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(double noundef %x, i32 noundef %y) + ret void +} + +define hidden void @i32_libcS(i32 noundef %x, ptr noundef byval(%struct.libcS) align 8 %y) { +; CHECK-LABEL: define {{[^@]+}}@i32_libcS(i32 noundef %x, ptr noundef byval(%struct.libcS) align 8 %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %IndirectAlloca = alloca %struct.libcS, align 8 +; CHECK-NEXT: %vararg_buffer = alloca %i32_libcS.vararg, align 16 +; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr %IndirectAlloca, ptr %y, i64 24, i1 false) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %i32_libcS.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %i32_libcS.vararg, ptr %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store ptr %IndirectAlloca, ptr %1, align 4 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(i32 noundef %x, ptr noundef nonnull byval(%struct.libcS) align 8 %y) + ret void +} + +define hidden void @libcS_i32(ptr noundef byval(%struct.libcS) align 8 %x, i32 noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@libcS_i32(ptr noundef byval(%struct.libcS) align 8 %x, i32 noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %IndirectAlloca = alloca %struct.libcS, align 8 +; CHECK-NEXT: %vararg_buffer = alloca %libcS_i32.vararg, align 16 +; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr %IndirectAlloca, ptr %x, i64 24, i1 false) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %libcS_i32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store ptr %IndirectAlloca, ptr %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %libcS_i32.vararg, ptr %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store i32 %y, ptr %1, align 4 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(ptr noundef nonnull byval(%struct.libcS) align 8 %x, i32 noundef %y) + ret void +} + +define hidden void @i32_v4f32(i32 noundef %x, <4 x float> noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@i32_v4f32(i32 noundef %x, <4 x float> noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %i32_v4f32.vararg, align 16 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 32, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %i32_v4f32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %i32_v4f32.vararg, ptr %vararg_buffer, i32 0, i32 2 +; CHECK-NEXT: store <4 x float> %y, ptr %1, align 16 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 32, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(i32 noundef %x, <4 x float> noundef %y) + ret void +} + +define hidden void @v4f32_i32(<4 x float> noundef %x, i32 noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@v4f32_i32(<4 x float> noundef %x, i32 noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %v4f32_i32.vararg, align 16 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 20, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %v4f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <4 x float> %x, ptr %0, align 16 +; CHECK-NEXT: %1 = getelementptr inbounds %v4f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store i32 %y, ptr %1, align 4 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 20, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<4 x float> noundef %x, i32 noundef %y) + ret void +} + +define hidden void @i32_v8f32(i32 noundef %x, <8 x float> noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@i32_v8f32(i32 noundef %x, <8 x float> noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %i32_v8f32.vararg, align 32 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 64, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %i32_v8f32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %i32_v8f32.vararg, ptr %vararg_buffer, i32 0, i32 2 +; CHECK-NEXT: store <8 x float> %y, ptr %1, align 32 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(i32 noundef %x, <8 x float> noundef %y) + ret void +} + +define hidden void @v8f32_i32(<8 x float> noundef %x, i32 noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@v8f32_i32(<8 x float> noundef %x, i32 noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %v8f32_i32.vararg, align 32 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 36, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %v8f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <8 x float> %x, ptr %0, align 32 +; CHECK-NEXT: %1 = getelementptr inbounds %v8f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store i32 %y, ptr %1, align 4 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 36, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<8 x float> noundef %x, i32 noundef %y) + ret void +} + +define hidden void @i32_v16f32(i32 noundef %x, <16 x float> noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@i32_v16f32(i32 noundef %x, <16 x float> noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %i32_v16f32.vararg, align 64 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 128, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %i32_v16f32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %i32_v16f32.vararg, ptr %vararg_buffer, i32 0, i32 2 +; CHECK-NEXT: store <16 x float> %y, ptr %1, align 64 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 128, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(i32 noundef %x, <16 x float> noundef %y) + ret void +} + +define hidden void @v16f32_i32(<16 x float> noundef %x, i32 noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@v16f32_i32(<16 x float> noundef %x, i32 noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %v16f32_i32.vararg, align 64 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 68, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %v16f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <16 x float> %x, ptr %0, align 64 +; CHECK-NEXT: %1 = getelementptr inbounds %v16f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store i32 %y, ptr %1, align 4 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 68, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<16 x float> noundef %x, i32 noundef %y) + ret void +} + +define hidden void @i32_v32f32(i32 noundef %x, <32 x float> noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@i32_v32f32(i32 noundef %x, <32 x float> noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %i32_v32f32.vararg, align 128 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 256, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %i32_v32f32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %i32_v32f32.vararg, ptr %vararg_buffer, i32 0, i32 2 +; CHECK-NEXT: store <32 x float> %y, ptr %1, align 128 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 256, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(i32 noundef %x, <32 x float> noundef %y) + ret void +} + +define hidden void @v32f32_i32(<32 x float> noundef %x, i32 noundef %y) { +; CHECK-LABEL: define {{[^@]+}}@v32f32_i32(<32 x float> noundef %x, i32 noundef %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %v32f32_i32.vararg, align 128 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 132, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %v32f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store <32 x float> %x, ptr %0, align 128 +; CHECK-NEXT: %1 = getelementptr inbounds %v32f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store i32 %y, ptr %1, align 4 +; CHECK-NEXT: call void @vararg(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 132, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + tail call void (...) @vararg(<32 x float> noundef %x, i32 noundef %y) + ret void +} + +define hidden void @fptr_single_i32(i32 noundef %x) { +; CHECK-LABEL: define {{[^@]+}}@fptr_single_i32(i32 noundef %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %fptr_single_i32.vararg, align 16 +; CHECK-NEXT: %0 = load volatile ptr, ptr @vararg_ptr, align 4 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr %vararg_buffer) +; CHECK-NEXT: %1 = getelementptr inbounds %fptr_single_i32.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr %1, align 4 +; CHECK-NEXT: call void %0(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + %0 = load volatile ptr, ptr @vararg_ptr, align 4 + tail call void (...) %0(i32 noundef %x) + ret void +} + +define hidden void @fptr_libcS(ptr noundef byval(%struct.libcS) align 8 %x) { +; CHECK-LABEL: define {{[^@]+}}@fptr_libcS(ptr noundef byval(%struct.libcS) align 8 %x) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %IndirectAlloca = alloca %struct.libcS, align 8 +; CHECK-NEXT: %vararg_buffer = alloca %fptr_libcS.vararg, align 16 +; CHECK-NEXT: %0 = load volatile ptr, ptr @vararg_ptr, align 4 +; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr %IndirectAlloca, ptr %x, i64 24, i1 false) +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr %vararg_buffer) +; CHECK-NEXT: %1 = getelementptr inbounds %fptr_libcS.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store ptr %IndirectAlloca, ptr %1, align 4 +; CHECK-NEXT: call void %0(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr %vararg_buffer) +; CHECK-NEXT: ret void +; +entry: + %0 = load volatile ptr, ptr @vararg_ptr, align 4 + tail call void (...) %0(ptr noundef nonnull byval(%struct.libcS) align 8 %x) + ret void +} + diff --git a/llvm/test/CodeGen/WebAssembly/vararg-frame.ll b/llvm/test/CodeGen/WebAssembly/vararg-frame.ll new file mode 100644 index 000000000000..5c76040325cc --- /dev/null +++ b/llvm/test/CodeGen/WebAssembly/vararg-frame.ll @@ -0,0 +1,526 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -verify-machineinstrs | FileCheck %s +; REQUIRES: webassembly-registered-target +target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20" +target triple = "wasm32-unknown-unknown" + +; Function Attrs: nounwind +define void @pass_s0() { +; CHECK-LABEL: pass_s0: +; CHECK: .functype pass_s0 () -> () +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: i32.const 0 +; CHECK-NEXT: call sink +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink() + ret void +} + +declare void @sink(...) + +; Function Attrs: nounwind +define void @pass_s1(i8 %x.coerce) { +; CHECK-LABEL: pass_s1: +; CHECK: .functype pass_s1 (i32) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 1 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(i8 %x.coerce) + ret void +} + +; Function Attrs: nounwind +define void @pass_s2(i16 %x.coerce) { +; CHECK-LABEL: pass_s2: +; CHECK: .functype pass_s2 (i32) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 1 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(i16 %x.coerce) + ret void +} + +; Function Attrs: nounwind +define void @pass_s3(i32 %x.coerce) { +; CHECK-LABEL: pass_s3: +; CHECK: .functype pass_s3 (i32) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 1 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(i32 %x.coerce) + ret void +} + +; Function Attrs: nounwind +define void @pass_s4(i64 %x.coerce) { +; CHECK-LABEL: pass_s4: +; CHECK: .functype pass_s4 (i64) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 1 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i64.store 0 +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(i64 %x.coerce) + ret void +} + +; Function Attrs: nounwind +define void @pass_s5(<4 x i32> noundef %x) { +; CHECK-LABEL: pass_s5: +; CHECK: .functype pass_s5 (i32, i32, i32, i32) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 4 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 4 +; CHECK-NEXT: local.get 3 +; CHECK-NEXT: i32.store 12 +; CHECK-NEXT: local.get 4 +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: i32.store 8 +; CHECK-NEXT: local.get 4 +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i32.store 4 +; CHECK-NEXT: local.get 4 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 4 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 4 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(<4 x i32> noundef %x) + ret void +} + +; Function Attrs: nounwind +define void @pass_int_s0(i32 noundef %i) { +; CHECK-LABEL: pass_int_s0: +; CHECK: .functype pass_int_s0 (i32) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 1 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(i32 noundef %i) + ret void +} + +; Function Attrs: nounwind +define void @pass_int_s1(i32 noundef %i, i8 %x.coerce) { +; CHECK-LABEL: pass_int_s1: +; CHECK: .functype pass_int_s1 (i32, i32) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 2 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i32.store 4 +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(i32 noundef %i, i8 %x.coerce) + ret void +} + +; Function Attrs: nounwind +define void @pass_int_s2(i32 noundef %i, i16 %x.coerce) { +; CHECK-LABEL: pass_int_s2: +; CHECK: .functype pass_int_s2 (i32, i32) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 2 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i32.store 4 +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(i32 noundef %i, i16 %x.coerce) + ret void +} + +; Function Attrs: nounwind +define void @pass_int_s3(i32 noundef %i, i32 %x.coerce) { +; CHECK-LABEL: pass_int_s3: +; CHECK: .functype pass_int_s3 (i32, i32) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 2 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i32.store 4 +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(i32 noundef %i, i32 %x.coerce) + ret void +} + +; Function Attrs: nounwind +define void @pass_int_s4(i32 noundef %i, i64 %x.coerce) { +; CHECK-LABEL: pass_int_s4: +; CHECK: .functype pass_int_s4 (i32, i64) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 2 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i64.store 8 +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(i32 noundef %i, i64 %x.coerce) + ret void +} + +; Function Attrs: nounwind +define void @pass_int_s5(i32 noundef %i, <4 x i32> noundef %x) { +; CHECK-LABEL: pass_int_s5: +; CHECK: .functype pass_int_s5 (i32, i32, i32, i32, i32) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 32 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 5 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 5 +; CHECK-NEXT: i32.const 28 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 4 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 5 +; CHECK-NEXT: i32.const 24 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 3 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 5 +; CHECK-NEXT: i32.const 20 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 5 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 5 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 5 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 5 +; CHECK-NEXT: i32.const 32 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(i32 noundef %i, <4 x i32> noundef %x) + ret void +} + +; Function Attrs: nounwind +define void @pass_asc(i8 %x1.coerce, i16 %x2.coerce, i32 %x3.coerce, i64 %x4.coerce, <4 x i32> noundef %x5) { +; CHECK-LABEL: pass_asc: +; CHECK: .functype pass_asc (i32, i32, i32, i64, i32, i32, i32, i32) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 48 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 8 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: i32.const 44 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 7 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: i32.const 40 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 6 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: i32.const 36 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 5 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: i32.const 32 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 4 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 3 +; CHECK-NEXT: i64.store 0 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: i32.store 8 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i32.store 4 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: i32.const 48 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(i8 %x1.coerce, i16 %x2.coerce, i32 %x3.coerce, i64 %x4.coerce, <4 x i32> noundef %x5) + ret void +} + +; Function Attrs: nounwind +define void @pass_dsc(<4 x i32> noundef %x0, i64 %x1.coerce, i32 %x2.coerce, i16 %x3.coerce, i8 %x4.coerce) { +; CHECK-LABEL: pass_dsc: +; CHECK: .functype pass_dsc (i32, i32, i32, i32, i64, i32, i32, i32) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 48 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 8 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: i32.const 32 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 7 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: i32.const 28 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 6 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: i32.const 24 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 5 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 4 +; CHECK-NEXT: i64.store 0 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: local.get 3 +; CHECK-NEXT: i32.store 12 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: i32.store 8 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i32.store 4 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: i32.const 48 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(<4 x i32> noundef %x0, i64 %x1.coerce, i32 %x2.coerce, i16 %x3.coerce, i8 %x4.coerce) + ret void +} + +; Function Attrs: nounwind +define void @pass_multiple(i32 noundef %i, i8 %x1.coerce, i16 %x2.coerce, i32 %x3.coerce, i64 %x4.coerce, <4 x i32> noundef %x5) { +; CHECK-LABEL: pass_multiple: +; CHECK: .functype pass_multiple (i32, i32, i32, i32, i64, i32, i32, i32, i32) -> () +; CHECK-NEXT: .local i32 +; CHECK-NEXT: # %bb.0: # %entry +; CHECK-NEXT: global.get __stack_pointer +; CHECK-NEXT: i32.const 48 +; CHECK-NEXT: i32.sub +; CHECK-NEXT: local.tee 9 +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: local.get 9 +; CHECK-NEXT: local.get 4 +; CHECK-NEXT: i64.store 40 +; CHECK-NEXT: local.get 9 +; CHECK-NEXT: local.get 2 +; CHECK-NEXT: i32.store 36 +; CHECK-NEXT: local.get 9 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 32 +; CHECK-NEXT: local.get 9 +; CHECK-NEXT: i32.const 32 +; CHECK-NEXT: i32.add +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 9 +; CHECK-NEXT: i32.const 28 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 8 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 9 +; CHECK-NEXT: i32.const 24 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 7 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 9 +; CHECK-NEXT: i32.const 20 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 6 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 9 +; CHECK-NEXT: i32.const 16 +; CHECK-NEXT: i32.add +; CHECK-NEXT: local.get 5 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 9 +; CHECK-NEXT: local.get 3 +; CHECK-NEXT: i32.store 8 +; CHECK-NEXT: local.get 9 +; CHECK-NEXT: local.get 1 +; CHECK-NEXT: i32.store 4 +; CHECK-NEXT: local.get 9 +; CHECK-NEXT: local.get 0 +; CHECK-NEXT: i32.store 0 +; CHECK-NEXT: local.get 9 +; CHECK-NEXT: call sink +; CHECK-NEXT: local.get 9 +; CHECK-NEXT: i32.const 48 +; CHECK-NEXT: i32.add +; CHECK-NEXT: global.set __stack_pointer +; CHECK-NEXT: # fallthrough-return +entry: + tail call void (...) @sink(i32 noundef %i, i16 %x2.coerce, i64 %x4.coerce) + tail call void (...) @sink(i32 noundef %i, i8 %x1.coerce, i32 %x3.coerce, <4 x i32> noundef %x5) + ret void +} + diff --git a/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-linkage.ll b/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-linkage.ll new file mode 100644 index 000000000000..f7e21cd586e6 --- /dev/null +++ b/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-linkage.ll @@ -0,0 +1,232 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --function-signature +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s --check-prefixes=OPT +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s --check-prefixes=ABI +; REQUIRES: webassembly-registered-target + +; Split variadic functions into two functions: +; - one equivalent to the original, same symbol etc +; - one implementing the contents of the original but taking a valist +; IR here is applicable to any target that uses a ptr for valist +; +; Defines a function with each linkage (in the order of the llvm documentation). +; If split applies it does the same transform to each. +; Whether split applies depends on whether the ABI is being changed or not - e.g. a weak +; function is not normally useful to split as the contents cannot be called from elsewhere. +; If the ABI is being rewritten then the function is still converted. Call sites tested elsewhere. + +; Update test checks doesn't emit checks for declares + +declare void @sink_valist(ptr) +declare void @llvm.va_start(ptr) +declare void @llvm.va_end(ptr) + +declare void @decl_simple(...) +define void @defn_simple(...) { +; OPT-LABEL: define {{[^@]+}}@defn_simple(...) { +; OPT-NEXT: entry: +; OPT-NEXT: %va_start = alloca ptr, align 4 +; OPT-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr %va_start) +; OPT-NEXT: call void @llvm.va_start.p0(ptr %va_start) +; OPT-NEXT: %0 = load ptr, ptr %va_start, align 4 +; OPT-NEXT: call void @defn_simple.valist(ptr %0) +; OPT-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr %va_start) +; OPT-NEXT: ret void +; +; ABI-LABEL: define {{[^@]+}}@defn_simple(ptr %varargs) { +; ABI-NEXT: %va = alloca ptr, align 4 +; ABI-NEXT: store ptr %varargs, ptr %va, align 4 +; ABI-NEXT: call void @sink_valist(ptr %va) +; ABI-NEXT: ret void +; + %va = alloca ptr, align 4 + call void @llvm.va_start(ptr %va) + call void @sink_valist(ptr %va) + call void @llvm.va_end(ptr %va) + ret void +} + +; no declare for private +define private void @defn_private_simple(...) { +; OPT-LABEL: define {{[^@]+}}@defn_private_simple(...) { +; OPT-NEXT: entry: +; OPT-NEXT: %va_start = alloca ptr, align 4 +; OPT-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr %va_start) +; OPT-NEXT: call void @llvm.va_start.p0(ptr %va_start) +; OPT-NEXT: %0 = load ptr, ptr %va_start, align 4 +; OPT-NEXT: call void @defn_private_simple.valist(ptr %0) +; OPT-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr %va_start) +; OPT-NEXT: ret void +; +; ABI-LABEL: define {{[^@]+}}@defn_private_simple(ptr %varargs) { +; ABI-NEXT: %va = alloca ptr, align 4 +; ABI-NEXT: store ptr %varargs, ptr %va, align 4 +; ABI-NEXT: call void @sink_valist(ptr %va) +; ABI-NEXT: ret void +; + %va = alloca ptr, align 4 + call void @llvm.va_start(ptr %va) + call void @sink_valist(ptr %va) + call void @llvm.va_end(ptr %va) + ret void +} + +; no declare for internal +define internal void @defn_internal_simple(...) { +; OPT-LABEL: define {{[^@]+}}@defn_internal_simple(...) { +; OPT-NEXT: entry: +; OPT-NEXT: %va_start = alloca ptr, align 4 +; OPT-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr %va_start) +; OPT-NEXT: call void @llvm.va_start.p0(ptr %va_start) +; OPT-NEXT: %0 = load ptr, ptr %va_start, align 4 +; OPT-NEXT: call void @defn_internal_simple.valist(ptr %0) +; OPT-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr %va_start) +; OPT-NEXT: ret void +; +; ABI-LABEL: define {{[^@]+}}@defn_internal_simple(ptr %varargs) { +; ABI-NEXT: %va = alloca ptr, align 4 +; ABI-NEXT: store ptr %varargs, ptr %va, align 4 +; ABI-NEXT: call void @sink_valist(ptr %va) +; ABI-NEXT: ret void +; + %va = alloca ptr, align 4 + call void @llvm.va_start(ptr %va) + call void @sink_valist(ptr %va) + call void @llvm.va_end(ptr %va) + ret void +} + +; no declare for available_externally +define available_externally void @available_externally_simple(...) { +; OPT-LABEL: define {{[^@]+}}@available_externally_simple(...) { +; OPT-NEXT: %va = alloca ptr, align 4 +; OPT-NEXT: call void @llvm.va_start.p0(ptr %va) +; OPT-NEXT: call void @sink_valist(ptr %va) +; OPT-NEXT: ret void +; +; ABI-LABEL: define {{[^@]+}}@available_externally_simple(ptr %varargs) { +; ABI-NEXT: %va = alloca ptr, align 4 +; ABI-NEXT: store ptr %varargs, ptr %va, align 4 +; ABI-NEXT: call void @sink_valist(ptr %va) +; ABI-NEXT: ret void +; + %va = alloca ptr, align 4 + call void @llvm.va_start(ptr %va) + call void @sink_valist(ptr %va) + call void @llvm.va_end(ptr %va) + ret void +} + +; no declare for linkonce +define linkonce void @defn_linkonce_simple(...) { +; OPT-LABEL: define {{[^@]+}}@defn_linkonce_simple(...) { +; OPT-NEXT: %va = alloca ptr, align 4 +; OPT-NEXT: call void @llvm.va_start.p0(ptr %va) +; OPT-NEXT: call void @sink_valist(ptr %va) +; OPT-NEXT: ret void +; +; ABI-LABEL: define {{[^@]+}}@defn_linkonce_simple(ptr %varargs) { +; ABI-NEXT: %va = alloca ptr, align 4 +; ABI-NEXT: store ptr %varargs, ptr %va, align 4 +; ABI-NEXT: call void @sink_valist(ptr %va) +; ABI-NEXT: ret void +; + %va = alloca ptr, align 4 + call void @llvm.va_start(ptr %va) + call void @sink_valist(ptr %va) + call void @llvm.va_end(ptr %va) + ret void +} + +; no declare for weak +define weak void @defn_weak_simple(...) { +; OPT-LABEL: define {{[^@]+}}@defn_weak_simple(...) { +; OPT-NEXT: %va = alloca ptr, align 4 +; OPT-NEXT: call void @llvm.va_start.p0(ptr %va) +; OPT-NEXT: call void @sink_valist(ptr %va) +; OPT-NEXT: ret void +; +; ABI-LABEL: define {{[^@]+}}@defn_weak_simple(ptr %varargs) { +; ABI-NEXT: %va = alloca ptr, align 4 +; ABI-NEXT: store ptr %varargs, ptr %va, align 4 +; ABI-NEXT: call void @sink_valist(ptr %va) +; ABI-NEXT: ret void +; + %va = alloca ptr, align 4 + call void @llvm.va_start(ptr %va) + call void @sink_valist(ptr %va) + call void @llvm.va_end(ptr %va) + ret void +} + +; common is not applicable to functions +; appending is not applicable to functions + +declare extern_weak void @decl_extern_weak_simple(...) +; no define for extern_weak + +; no declare for linkonce_odr +define linkonce_odr void @defn_linkonce_odr_simple(...) { +; OPT-LABEL: define {{[^@]+}}@defn_linkonce_odr_simple(...) { +; OPT-NEXT: %va = alloca ptr, align 4 +; OPT-NEXT: call void @llvm.va_start.p0(ptr %va) +; OPT-NEXT: call void @sink_valist(ptr %va) +; OPT-NEXT: ret void +; +; ABI-LABEL: define {{[^@]+}}@defn_linkonce_odr_simple(ptr %varargs) { +; ABI-NEXT: %va = alloca ptr, align 4 +; ABI-NEXT: store ptr %varargs, ptr %va, align 4 +; ABI-NEXT: call void @sink_valist(ptr %va) +; ABI-NEXT: ret void +; + %va = alloca ptr, align 4 + call void @llvm.va_start(ptr %va) + call void @sink_valist(ptr %va) + call void @llvm.va_end(ptr %va) + ret void +} + +; no declare for weak_odr +define weak_odr void @defn_weak_odr_simple(...) { +; OPT-LABEL: define {{[^@]+}}@defn_weak_odr_simple(...) { +; OPT-NEXT: %va = alloca ptr, align 4 +; OPT-NEXT: call void @llvm.va_start.p0(ptr %va) +; OPT-NEXT: call void @sink_valist(ptr %va) +; OPT-NEXT: ret void +; +; ABI-LABEL: define {{[^@]+}}@defn_weak_odr_simple(ptr %varargs) { +; ABI-NEXT: %va = alloca ptr, align 4 +; ABI-NEXT: store ptr %varargs, ptr %va, align 4 +; ABI-NEXT: call void @sink_valist(ptr %va) +; ABI-NEXT: ret void +; + %va = alloca ptr, align 4 + call void @llvm.va_start(ptr %va) + call void @sink_valist(ptr %va) + call void @llvm.va_end(ptr %va) + ret void +} + +declare external void @decl_external_simple(...) +define external void @defn_external_simple(...) { +; OPT-LABEL: define {{[^@]+}}@defn_external_simple(...) { +; OPT-NEXT: entry: +; OPT-NEXT: %va_start = alloca ptr, align 4 +; OPT-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr %va_start) +; OPT-NEXT: call void @llvm.va_start.p0(ptr %va_start) +; OPT-NEXT: %0 = load ptr, ptr %va_start, align 4 +; OPT-NEXT: call void @defn_external_simple.valist(ptr %0) +; OPT-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr %va_start) +; OPT-NEXT: ret void +; +; ABI-LABEL: define {{[^@]+}}@defn_external_simple(ptr %varargs) { +; ABI-NEXT: %va = alloca ptr, align 4 +; ABI-NEXT: store ptr %varargs, ptr %va, align 4 +; ABI-NEXT: call void @sink_valist(ptr %va) +; ABI-NEXT: ret void +; + %va = alloca ptr, align 4 + call void @llvm.va_start(ptr %va) + call void @sink_valist(ptr %va) + call void @llvm.va_end(ptr %va) + ret void +} diff --git a/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-simple.ll b/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-simple.ll new file mode 100644 index 000000000000..9a86540ba2d5 --- /dev/null +++ b/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-simple.ll @@ -0,0 +1,214 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --function-signature +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s --check-prefixes=OPT +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s --check-prefixes=ABI +; REQUIRES: webassembly-registered-target + +; Examples are variadic functions that return the first or the second of an int and a double +; Split the functions into an internal equivalent that takes a va_list and a ABI preserving wrapper + +define i32 @variadic_int_double_get_firstz(...) { +; OPT-LABEL: define {{[^@]+}}@variadic_int_double_get_firstz(...) { +; OPT-NEXT: entry: +; OPT-NEXT: %va_start = alloca ptr, align 4 +; OPT-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr %va_start) +; OPT-NEXT: call void @llvm.va_start.p0(ptr %va_start) +; OPT-NEXT: %0 = load ptr, ptr %va_start, align 4 +; OPT-NEXT: %1 = call i32 @variadic_int_double_get_firstz.valist(ptr %0) +; OPT-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr %va_start) +; OPT-NEXT: ret i32 %1 +; +; ABI-LABEL: define {{[^@]+}}@variadic_int_double_get_firstz(ptr %varargs) { +; ABI-NEXT: entry: +; ABI-NEXT: %va = alloca ptr, align 4 +; ABI-NEXT: store ptr %varargs, ptr %va, align 4 +; ABI-NEXT: %argp.cur = load ptr, ptr %va, align 4 +; ABI-NEXT: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 +; ABI-NEXT: store ptr %argp.next, ptr %va, align 4 +; ABI-NEXT: %0 = load i32, ptr %argp.cur, align 4 +; ABI-NEXT: ret i32 %0 +; +entry: + %va = alloca ptr, align 4 + call void @llvm.va_start.p0(ptr nonnull %va) + %argp.cur = load ptr, ptr %va, align 4 + %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 + store ptr %argp.next, ptr %va, align 4 + %0 = load i32, ptr %argp.cur, align 4 + call void @llvm.va_end.p0(ptr %va) + ret i32 %0 +} + +; CHECK-LABEL: define i32 @variadic_int_double_get_firstz(...) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %va_list = alloca ptr, align 4 +; CHECK-NEXT: call void @llvm.va_start.p0(ptr %va_list) +; CHECK-NEXT: %0 = tail call i32 @variadic_int_double_get_firstz.valist(ptr %va_list) +; CHECK-NEXT: ret i32 %0 +; CHECK-NEXT: } + +; CHECK-LABEL: define internal i32 @variadic_int_double_get_firstz.valist(ptr noalias %varargs) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %va = alloca ptr, align 4 +; CHECK-NEXT: store ptr %varargs, ptr %va, align 4 +; CHECK-NEXT: %argp.cur = load ptr, ptr %va, align 4 +; CHECK-NEXT: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 +; CHECK-NEXT: store ptr %argp.next, ptr %va, align 4 +; CHECK-NEXT: %0 = load i32, ptr %argp.cur, align 4 +; CHECK-NEXT: ret i32 %0 +; CHECK-NEXT: } + +define double @variadic_int_double_get_secondz(...) { +; OPT-LABEL: define {{[^@]+}}@variadic_int_double_get_secondz(...) { +; OPT-NEXT: entry: +; OPT-NEXT: %va_start = alloca ptr, align 4 +; OPT-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr %va_start) +; OPT-NEXT: call void @llvm.va_start.p0(ptr %va_start) +; OPT-NEXT: %0 = load ptr, ptr %va_start, align 4 +; OPT-NEXT: %1 = call double @variadic_int_double_get_secondz.valist(ptr %0) +; OPT-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr %va_start) +; OPT-NEXT: ret double %1 +; +; ABI-LABEL: define {{[^@]+}}@variadic_int_double_get_secondz(ptr %varargs) { +; ABI-NEXT: entry: +; ABI-NEXT: %va = alloca ptr, align 4 +; ABI-NEXT: store ptr %varargs, ptr %va, align 4 +; ABI-NEXT: %argp.cur = load ptr, ptr %va, align 4 +; ABI-NEXT: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 +; ABI-NEXT: %argp.next2 = getelementptr inbounds i8, ptr %argp.cur, i32 12 +; ABI-NEXT: store ptr %argp.next2, ptr %va, align 4 +; ABI-NEXT: %0 = load double, ptr %argp.next, align 4 +; ABI-NEXT: ret double %0 +; +entry: + %va = alloca ptr, align 4 + call void @llvm.va_start.p0(ptr nonnull %va) + %argp.cur = load ptr, ptr %va, align 4 + %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 + %argp.next2 = getelementptr inbounds i8, ptr %argp.cur, i32 12 + store ptr %argp.next2, ptr %va, align 4 + %0 = load double, ptr %argp.next, align 4 + call void @llvm.va_end.p0(ptr %va) + ret double %0 +} + +; CHECK-LABEL: define double @variadic_int_double_get_secondz(...) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %va_list = alloca ptr, align 4 +; CHECK-NEXT: call void @llvm.va_start.p0(ptr %va_list) +; CHECK-NEXT: %0 = tail call double @variadic_int_double_get_secondz.valist(ptr %va_list) +; CHECK-NEXT: ret double %0 +; CHECK-NEXT: } + +; CHECK-LABEL: define internal double @variadic_int_double_get_secondz.valist(ptr noalias %varargs) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %va = alloca ptr, align 4 +; CHECK-NEXT: store ptr %varargs, ptr %va, align 4 +; CHECK-NEXT: %argp.cur = load ptr, ptr %va, align 4 +; CHECK-NEXT: %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 +; CHECK-NEXT: %argp.next2 = getelementptr inbounds i8, ptr %argp.cur, i32 12 +; CHECK-NEXT: store ptr %argp.next2, ptr %va, align 4 +; CHECK-NEXT: %0 = load double, ptr %argp.next, align 4 +; CHECK-NEXT: ret double %0 +; CHECK-NEXT: } + + +; CHECK-LABEL: @variadic_can_get_firstIidEEbT_T0_(i32 %x, double %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %variadic_can_get_firstIidEEbT_T0_.vararg, align 16 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 12, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store double %y, ptr %1, align 4 +; CHECK-NEXT: %call = call i32 @variadic_int_double_get_firstz.valist(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 12, ptr %vararg_buffer) +; CHECK-NEXT: %cmp.i = icmp eq i32 %call, %x +; CHECK-NEXT: ret i1 %cmp.i +; CHECK-NEXT: } + +define zeroext i1 @variadic_can_get_firstIidEEbT_T0_(i32 %x, double %y) { +; OPT-LABEL: define {{[^@]+}}@variadic_can_get_firstIidEEbT_T0_(i32 %x, double %y) { +; OPT-NEXT: entry: +; OPT-NEXT: %vararg_buffer = alloca %variadic_can_get_firstIidEEbT_T0_.vararg, align 16 +; OPT-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer) +; OPT-NEXT: %0 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0 +; OPT-NEXT: store i32 %x, ptr %0, align 4 +; OPT-NEXT: %1 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 2 +; OPT-NEXT: store double %y, ptr %1, align 8 +; OPT-NEXT: %call = call i32 @variadic_int_double_get_firstz.valist(ptr %vararg_buffer) +; OPT-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer) +; OPT-NEXT: %cmp.i = icmp eq i32 %call, %x +; OPT-NEXT: ret i1 %cmp.i +; +; ABI-LABEL: define {{[^@]+}}@variadic_can_get_firstIidEEbT_T0_(i32 %x, double %y) { +; ABI-NEXT: entry: +; ABI-NEXT: %vararg_buffer = alloca %variadic_can_get_firstIidEEbT_T0_.vararg, align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer) +; ABI-NEXT: %0 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0 +; ABI-NEXT: store i32 %x, ptr %0, align 4 +; ABI-NEXT: %1 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 2 +; ABI-NEXT: store double %y, ptr %1, align 8 +; ABI-NEXT: %call = call i32 @variadic_int_double_get_firstz(ptr %vararg_buffer) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer) +; ABI-NEXT: %cmp.i = icmp eq i32 %call, %x +; ABI-NEXT: ret i1 %cmp.i +; +entry: + %call = call i32 (...) @variadic_int_double_get_firstz(i32 %x, double %y) + %cmp.i = icmp eq i32 %call, %x + ret i1 %cmp.i +} + +; CHECK-LABEL: @variadic_can_get_secondIidEEbT_T0_(i32 %x, double %y) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %vararg_buffer = alloca %variadic_can_get_secondIidEEbT_T0_.vararg, align 16 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 12, ptr %vararg_buffer) +; CHECK-NEXT: %0 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0 +; CHECK-NEXT: store i32 %x, ptr %0, align 4 +; CHECK-NEXT: %1 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 1 +; CHECK-NEXT: store double %y, ptr %1, align 4 +; CHECK-NEXT: %call = call double @variadic_int_double_get_secondz.valist(ptr %vararg_buffer) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 12, ptr %vararg_buffer) +; CHECK-NEXT: %cmp.i = fcmp oeq double %call, %y +; CHECK-NEXT: ret i1 %cmp.i +; CHECK-NEXT: } + +define zeroext i1 @variadic_can_get_secondIidEEbT_T0_(i32 %x, double %y) { +; OPT-LABEL: define {{[^@]+}}@variadic_can_get_secondIidEEbT_T0_(i32 %x, double %y) { +; OPT-NEXT: entry: +; OPT-NEXT: %vararg_buffer = alloca %variadic_can_get_secondIidEEbT_T0_.vararg, align 16 +; OPT-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer) +; OPT-NEXT: %0 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0 +; OPT-NEXT: store i32 %x, ptr %0, align 4 +; OPT-NEXT: %1 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 2 +; OPT-NEXT: store double %y, ptr %1, align 8 +; OPT-NEXT: %call = call double @variadic_int_double_get_secondz.valist(ptr %vararg_buffer) +; OPT-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer) +; OPT-NEXT: %cmp.i = fcmp oeq double %call, %y +; OPT-NEXT: ret i1 %cmp.i +; +; ABI-LABEL: define {{[^@]+}}@variadic_can_get_secondIidEEbT_T0_(i32 %x, double %y) { +; ABI-NEXT: entry: +; ABI-NEXT: %vararg_buffer = alloca %variadic_can_get_secondIidEEbT_T0_.vararg, align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer) +; ABI-NEXT: %0 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0 +; ABI-NEXT: store i32 %x, ptr %0, align 4 +; ABI-NEXT: %1 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 2 +; ABI-NEXT: store double %y, ptr %1, align 8 +; ABI-NEXT: %call = call double @variadic_int_double_get_secondz(ptr %vararg_buffer) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer) +; ABI-NEXT: %cmp.i = fcmp oeq double %call, %y +; ABI-NEXT: ret i1 %cmp.i +; +entry: + %call = call double (...) @variadic_int_double_get_secondz(i32 %x, double %y) + %cmp.i = fcmp oeq double %call, %y + ret i1 %cmp.i +} + +; Declaration unchanged +; CHECK: declare void @variadic_without_callers(...) +declare void @variadic_without_callers(...) + +declare void @llvm.va_start.p0(ptr) +declare void @llvm.va_end.p0(ptr) diff --git a/llvm/test/Transforms/ExpandVariadics/indirect-calls.ll b/llvm/test/Transforms/ExpandVariadics/indirect-calls.ll new file mode 100644 index 000000000000..de04c7235ad1 --- /dev/null +++ b/llvm/test/Transforms/ExpandVariadics/indirect-calls.ll @@ -0,0 +1,59 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s -check-prefixes=OPT +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s -check-prefixes=ABI +; REQUIRES: webassembly-registered-target + +declare void @vararg(...) +@vararg_ptr = hidden global ptr @vararg, align 4 + +%struct.libcS = type { i8, i16, i32, i32, float, double } + +define hidden void @fptr_single_i32(i32 noundef %x) { +; OPT-LABEL: @fptr_single_i32( +; OPT-NEXT: entry: +; OPT-NEXT: [[TMP0:%.*]] = load volatile ptr, ptr @vararg_ptr, align 4 +; OPT-NEXT: tail call void (...) [[TMP0]](i32 noundef [[X:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @fptr_single_i32( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[FPTR_SINGLE_I32_VARARG:%.*]], align 16 +; ABI-NEXT: [[TMP0:%.*]] = load volatile ptr, ptr @vararg_ptr, align 4 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[FPTR_SINGLE_I32_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i32 [[X:%.*]], ptr [[TMP1]], align 4 +; ABI-NEXT: call void [[TMP0]](ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + %0 = load volatile ptr, ptr @vararg_ptr, align 4 + tail call void (...) %0(i32 noundef %x) + ret void +} + +define hidden void @fptr_libcS(ptr noundef byval(%struct.libcS) align 8 %x) { +; OPT-LABEL: @fptr_libcS( +; OPT-NEXT: entry: +; OPT-NEXT: [[TMP0:%.*]] = load volatile ptr, ptr @vararg_ptr, align 4 +; OPT-NEXT: tail call void (...) [[TMP0]](ptr noundef nonnull byval([[STRUCT_LIBCS:%.*]]) align 8 [[X:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @fptr_libcS( +; ABI-NEXT: entry: +; ABI-NEXT: [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8 +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[FPTR_LIBCS_VARARG:%.*]], align 16 +; ABI-NEXT: [[TMP0:%.*]] = load volatile ptr, ptr @vararg_ptr, align 4 +; ABI-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[INDIRECTALLOCA]], ptr [[X:%.*]], i64 24, i1 false) +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[FPTR_LIBCS_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store ptr [[INDIRECTALLOCA]], ptr [[TMP1]], align 4 +; ABI-NEXT: call void [[TMP0]](ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + %0 = load volatile ptr, ptr @vararg_ptr, align 4 + tail call void (...) %0(ptr noundef nonnull byval(%struct.libcS) align 8 %x) + ret void +} diff --git a/llvm/test/Transforms/ExpandVariadics/intrinsics.ll b/llvm/test/Transforms/ExpandVariadics/intrinsics.ll new file mode 100644 index 000000000000..1782c9229574 --- /dev/null +++ b/llvm/test/Transforms/ExpandVariadics/intrinsics.ll @@ -0,0 +1,120 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s -check-prefixes=CHECK,OPT +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s -check-prefixes=CHECK,ABI +; REQUIRES: webassembly-registered-target + +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) + +declare void @llvm.va_copy.p0(ptr, ptr) + +declare void @valist(ptr noundef) + +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) + +declare void @llvm.va_start.p0(ptr) + +declare void @llvm.va_end.p0(ptr) + + +define void @start_once(...) { +; OPT-LABEL: @start_once( +; OPT-NEXT: entry: +; OPT-NEXT: [[VA_START:%.*]] = alloca ptr, align 4 +; OPT-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[VA_START]]) +; OPT-NEXT: call void @llvm.va_start.p0(ptr [[VA_START]]) +; OPT-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VA_START]], align 4 +; OPT-NEXT: call void @start_once.valist(ptr [[TMP0]]) +; OPT-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[VA_START]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @start_once( +; ABI-NEXT: entry: +; ABI-NEXT: [[S:%.*]] = alloca ptr, align 4 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[S]]) +; ABI-NEXT: store ptr [[VARARGS:%.*]], ptr [[S]], align 4 +; ABI-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S]], align 4 +; ABI-NEXT: call void @valist(ptr noundef [[TMP0]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[S]]) +; ABI-NEXT: ret void +; +entry: + %s = alloca ptr, align 4 + call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s) + call void @llvm.va_start.p0(ptr nonnull %s) + %0 = load ptr, ptr %s, align 4 + call void @valist(ptr noundef %0) + call void @llvm.va_end.p0(ptr %s) + call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s) + ret void +} + + +define void @start_twice(...) { +; OPT-LABEL: @start_twice( +; OPT-NEXT: entry: +; OPT-NEXT: [[VA_START:%.*]] = alloca ptr, align 4 +; OPT-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[VA_START]]) +; OPT-NEXT: call void @llvm.va_start.p0(ptr [[VA_START]]) +; OPT-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VA_START]], align 4 +; OPT-NEXT: call void @start_twice.valist(ptr [[TMP0]]) +; OPT-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[VA_START]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @start_twice( +; ABI-NEXT: entry: +; ABI-NEXT: [[S0:%.*]] = alloca ptr, align 4 +; ABI-NEXT: [[S1:%.*]] = alloca ptr, align 4 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[S0]]) +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[S1]]) +; ABI-NEXT: store ptr [[VARARGS:%.*]], ptr [[S0]], align 4 +; ABI-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S0]], align 4 +; ABI-NEXT: call void @valist(ptr noundef [[TMP0]]) +; ABI-NEXT: store ptr [[VARARGS]], ptr [[S1]], align 4 +; ABI-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S1]], align 4 +; ABI-NEXT: call void @valist(ptr noundef [[TMP1]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[S1]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[S0]]) +; ABI-NEXT: ret void +; +entry: + %s0 = alloca ptr, align 4 + %s1 = alloca ptr, align 4 + call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s0) + call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s1) + call void @llvm.va_start.p0(ptr nonnull %s0) + %0 = load ptr, ptr %s0, align 4 + call void @valist(ptr noundef %0) + call void @llvm.va_end.p0(ptr %s0) + call void @llvm.va_start.p0(ptr nonnull %s1) + %1 = load ptr, ptr %s1, align 4 + call void @valist(ptr noundef %1) + call void @llvm.va_end.p0(ptr %s1) + call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s1) + call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s0) + ret void +} + +define void @copy(ptr noundef %va) { +; CHECK-LABEL: @copy( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[VA_ADDR:%.*]] = alloca ptr, align 4 +; CHECK-NEXT: [[CP:%.*]] = alloca ptr, align 4 +; CHECK-NEXT: store ptr [[VA:%.*]], ptr [[VA_ADDR]], align 4 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[CP]]) +; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr [[CP]], ptr [[VA_ADDR]], i32 4, i1 false) +; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[CP]], align 4 +; CHECK-NEXT: call void @valist(ptr noundef [[TMP0]]) +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[CP]]) +; CHECK-NEXT: ret void +; +entry: + %va.addr = alloca ptr, align 4 + %cp = alloca ptr, align 4 + store ptr %va, ptr %va.addr, align 4 + call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %cp) + call void @llvm.va_copy.p0(ptr nonnull %cp, ptr nonnull %va.addr) + %0 = load ptr, ptr %cp, align 4 + call void @valist(ptr noundef %0) + call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %cp) + ret void +} diff --git a/llvm/test/Transforms/ExpandVariadics/invoke.ll b/llvm/test/Transforms/ExpandVariadics/invoke.ll new file mode 100644 index 000000000000..ced2edf9274f --- /dev/null +++ b/llvm/test/Transforms/ExpandVariadics/invoke.ll @@ -0,0 +1,89 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s -check-prefixes=CHECK +; RUN: not --crash opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s 2>&1 | FileCheck %s -check-prefixes=ERROR +; REQUIRES: webassembly-registered-target +target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20" + +; ERROR: LLVM ERROR: Cannot lower callbase instruction + +@_ZTIi = external constant ptr + +; Function Attrs: mustprogress +define hidden void @test0(i32 noundef %x) #0 personality ptr @__gxx_wasm_personality_v0 { +; CHECK-LABEL: @test0( +; CHECK-NEXT: entry: +; CHECK-NEXT: invoke void (...) @may_throw(i32 noundef [[X:%.*]]) +; CHECK-NEXT: to label [[TRY_CONT:%.*]] unwind label [[CATCH_DISPATCH:%.*]] +; CHECK: catch.dispatch: +; CHECK-NEXT: [[TMP0:%.*]] = catchswitch within none [label %catch.start] unwind to caller +; CHECK: catch.start: +; CHECK-NEXT: [[TMP1:%.*]] = catchpad within [[TMP0]] [ptr @_ZTIi] +; CHECK-NEXT: [[TMP2:%.*]] = tail call ptr @llvm.wasm.get.exception(token [[TMP1]]) +; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.wasm.get.ehselector(token [[TMP1]]) +; CHECK-NEXT: [[TMP4:%.*]] = tail call i32 @llvm.eh.typeid.for.p0(ptr nonnull @_ZTIi) +; CHECK-NEXT: [[MATCHES:%.*]] = icmp eq i32 [[TMP3]], [[TMP4]] +; CHECK-NEXT: br i1 [[MATCHES]], label [[CATCH:%.*]], label [[RETHROW:%.*]] +; CHECK: catch: +; CHECK-NEXT: [[TMP5:%.*]] = call ptr @__cxa_begin_catch(ptr [[TMP2]]) [ "funclet"(token [[TMP1]]) ] +; CHECK-NEXT: call void (...) @dont_throw(i32 noundef [[X]]) [ "funclet"(token [[TMP1]]) ] +; CHECK-NEXT: call void @__cxa_end_catch() [ "funclet"(token [[TMP1]]) ] +; CHECK-NEXT: catchret from [[TMP1]] to label [[TRY_CONT]] +; CHECK: rethrow: +; CHECK-NEXT: call void @llvm.wasm.rethrow() [ "funclet"(token [[TMP1]]) ] +; CHECK-NEXT: unreachable +; CHECK: try.cont: +; CHECK-NEXT: ret void +; +entry: + invoke void (...) @may_throw(i32 noundef %x) + to label %try.cont unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %0 = catchswitch within none [label %catch.start] unwind to caller + +catch.start: ; preds = %catch.dispatch + %1 = catchpad within %0 [ptr @_ZTIi] + %2 = tail call ptr @llvm.wasm.get.exception(token %1) + %3 = tail call i32 @llvm.wasm.get.ehselector(token %1) + %4 = tail call i32 @llvm.eh.typeid.for.p0(ptr nonnull @_ZTIi) + %matches = icmp eq i32 %3, %4 + br i1 %matches, label %catch, label %rethrow + +catch: ; preds = %catch.start + %5 = call ptr @__cxa_begin_catch(ptr %2) #6 [ "funclet"(token %1) ] + call void (...) @dont_throw(i32 noundef %x) #6 [ "funclet"(token %1) ] + call void @__cxa_end_catch() #6 [ "funclet"(token %1) ] + catchret from %1 to label %try.cont + +rethrow: ; preds = %catch.start + call void @llvm.wasm.rethrow() #5 [ "funclet"(token %1) ] + unreachable + +try.cont: ; preds = %entry, %catch + ret void +} + +declare void @may_throw(...) + +declare i32 @__gxx_wasm_personality_v0(...) + +; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn +declare ptr @llvm.wasm.get.exception(token) + +; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn +declare i32 @llvm.wasm.get.ehselector(token) + +; Function Attrs: nofree nosync nounwind memory(none) +declare i32 @llvm.eh.typeid.for.p0(ptr) + +declare ptr @__cxa_begin_catch(ptr) + +; Function Attrs: nounwind +declare void @dont_throw(...) + +declare void @__cxa_end_catch() + +; Function Attrs: noreturn +declare void @llvm.wasm.rethrow() + + diff --git a/llvm/test/Transforms/ExpandVariadics/pass-byval-byref.ll b/llvm/test/Transforms/ExpandVariadics/pass-byval-byref.ll new file mode 100644 index 000000000000..85fefda36a76 --- /dev/null +++ b/llvm/test/Transforms/ExpandVariadics/pass-byval-byref.ll @@ -0,0 +1,153 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s -check-prefixes=OPT +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s -check-prefixes=ABI +; REQUIRES: webassembly-registered-target + +; CHECK: @sink +declare void @sink(...) + + +define void @pass_byval(ptr byval(i32) %b) { +; OPT-LABEL: @pass_byval( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(ptr byval(i32) [[B:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_byval( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_BYVAL_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[TMP0]], ptr [[B:%.*]], i64 4, i1 false) +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(ptr byval(i32) %b) + ret void +} + +%struct.libcS = type { i8, i16, i32, i32, float, double } + +define void @i32_libcS_byval(i32 %x, ptr noundef byval(%struct.libcS) align 8 %y) { +; OPT-LABEL: @i32_libcS_byval( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i32 [[X:%.*]], ptr byval([[STRUCT_LIBCS:%.*]]) align 8 [[Y:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @i32_libcS_byval( +; ABI-NEXT: entry: +; ABI-NEXT: [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8 +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[I32_LIBCS_BYVAL_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[INDIRECTALLOCA]], ptr [[Y:%.*]], i64 24, i1 false) +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[I32_LIBCS_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i32 [[X:%.*]], ptr [[TMP0]], align 4 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[I32_LIBCS_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1 +; ABI-NEXT: store ptr [[INDIRECTALLOCA]], ptr [[TMP1]], align 4 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i32 %x, ptr byval(%struct.libcS) align 8 %y) + ret void +} + +define void @libcS_i32_byval(ptr byval(%struct.libcS) align 8 %x, i32 %y) { +; OPT-LABEL: @libcS_i32_byval( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(ptr byval([[STRUCT_LIBCS:%.*]]) align 8 [[X:%.*]], i32 [[Y:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @libcS_i32_byval( +; ABI-NEXT: entry: +; ABI-NEXT: [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8 +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[LIBCS_I32_BYVAL_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[INDIRECTALLOCA]], ptr [[X:%.*]], i64 24, i1 false) +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[LIBCS_I32_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store ptr [[INDIRECTALLOCA]], ptr [[TMP0]], align 4 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[LIBCS_I32_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1 +; ABI-NEXT: store i32 [[Y:%.*]], ptr [[TMP1]], align 4 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(ptr byval(%struct.libcS) align 8 %x, i32 %y) + ret void +} + + +define void @pass_byref(ptr byref(i32) %b) { +; OPT-LABEL: @pass_byref( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(ptr byref(i32) [[B:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_byref( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_BYREF_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store ptr [[B:%.*]], ptr [[TMP0]], align 4 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(ptr byref(i32) %b) + ret void +} + +define void @i32_libcS_byref(i32 %x, ptr noundef byref(%struct.libcS) align 8 %y) { +; OPT-LABEL: @i32_libcS_byref( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i32 [[X:%.*]], ptr byref([[STRUCT_LIBCS:%.*]]) align 8 [[Y:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @i32_libcS_byref( +; ABI-NEXT: entry: +; ABI-NEXT: [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8 +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[I32_LIBCS_BYREF_VARARG:%.*]], align 16 +; ABI-NEXT: store ptr [[Y:%.*]], ptr [[INDIRECTALLOCA]], align 4 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[I32_LIBCS_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i32 [[X:%.*]], ptr [[TMP0]], align 4 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[I32_LIBCS_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1 +; ABI-NEXT: store ptr [[INDIRECTALLOCA]], ptr [[TMP1]], align 4 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i32 %x, ptr byref(%struct.libcS) align 8 %y) + ret void +} + +define void @libcS_i32_byref(ptr byref(%struct.libcS) align 8 %x, i32 %y) { +; OPT-LABEL: @libcS_i32_byref( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(ptr byref([[STRUCT_LIBCS:%.*]]) align 8 [[X:%.*]], i32 [[Y:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @libcS_i32_byref( +; ABI-NEXT: entry: +; ABI-NEXT: [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8 +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[LIBCS_I32_BYREF_VARARG:%.*]], align 16 +; ABI-NEXT: store ptr [[X:%.*]], ptr [[INDIRECTALLOCA]], align 4 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[LIBCS_I32_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store ptr [[INDIRECTALLOCA]], ptr [[TMP0]], align 4 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[LIBCS_I32_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1 +; ABI-NEXT: store i32 [[Y:%.*]], ptr [[TMP1]], align 4 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(ptr byref(%struct.libcS) align 8 %x, i32 %y) + ret void +} diff --git a/llvm/test/Transforms/ExpandVariadics/pass-indirect.ll b/llvm/test/Transforms/ExpandVariadics/pass-indirect.ll new file mode 100644 index 000000000000..8dcbb86d02d6 --- /dev/null +++ b/llvm/test/Transforms/ExpandVariadics/pass-indirect.ll @@ -0,0 +1,59 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s -check-prefixes=OPT +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s -check-prefixes=ABI +; REQUIRES: webassembly-registered-target + +; CHECK: @sink +declare void @sink(...) + +%struct.libcS = type { i8, i16, i32, i32, float, double } + +define void @i32_libcS(i32 %x, %struct.libcS %y) { +; OPT-LABEL: @i32_libcS( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i32 [[X:%.*]], [[STRUCT_LIBCS:%.*]] [[Y:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @i32_libcS( +; ABI-NEXT: entry: +; ABI-NEXT: [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8 +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[I32_LIBCS_VARARG:%.*]], align 16 +; ABI-NEXT: store [[STRUCT_LIBCS]] [[Y:%.*]], ptr [[INDIRECTALLOCA]], align 8 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[I32_LIBCS_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i32 [[X:%.*]], ptr [[TMP0]], align 4 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[I32_LIBCS_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1 +; ABI-NEXT: store ptr [[INDIRECTALLOCA]], ptr [[TMP1]], align 4 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i32 %x, %struct.libcS %y) + ret void +} + +define void @libcS_i32(%struct.libcS %x, i32 %y) { +; OPT-LABEL: @libcS_i32( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink([[STRUCT_LIBCS:%.*]] [[X:%.*]], i32 [[Y:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @libcS_i32( +; ABI-NEXT: entry: +; ABI-NEXT: [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8 +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[LIBCS_I32_VARARG:%.*]], align 16 +; ABI-NEXT: store [[STRUCT_LIBCS]] [[X:%.*]], ptr [[INDIRECTALLOCA]], align 8 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[LIBCS_I32_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store ptr [[INDIRECTALLOCA]], ptr [[TMP0]], align 4 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[LIBCS_I32_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1 +; ABI-NEXT: store i32 [[Y:%.*]], ptr [[TMP1]], align 4 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(%struct.libcS %x, i32 %y) + ret void +} diff --git a/llvm/test/Transforms/ExpandVariadics/pass-integers.ll b/llvm/test/Transforms/ExpandVariadics/pass-integers.ll new file mode 100644 index 000000000000..a1cb6811800c --- /dev/null +++ b/llvm/test/Transforms/ExpandVariadics/pass-integers.ll @@ -0,0 +1,345 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=optimize < %s | FileCheck %s -check-prefixes=OPT +; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s -check-prefixes=ABI +; REQUIRES: webassembly-registered-target + +; Wasm passes struct {char} as an i8 so can check the varargs passing works on integers smaller than the slot size + +declare void @sink(...) + + +define void @pass_nothing() { +; OPT-LABEL: @pass_nothing( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink() +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_nothing( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_NOTHING_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 1, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 1, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink() + ret void +} + +define void @pass_s1(i8 %x) { +; OPT-LABEL: @pass_s1( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i8 [[X:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_s1( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_S1_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 1, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_S1_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i8 [[X:%.*]], ptr [[TMP0]], align 1 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 1, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i8 %x) + ret void +} + +define void @pass_s2(i16 %x) { +; OPT-LABEL: @pass_s2( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i16 [[X:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_s2( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_S2_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 2, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_S2_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i16 [[X:%.*]], ptr [[TMP0]], align 2 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 2, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i16 %x) + ret void +} + +define void @pass_s3(i32 %x) { +; OPT-LABEL: @pass_s3( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i32 [[X:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_s3( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_S3_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_S3_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i32 [[X:%.*]], ptr [[TMP0]], align 4 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i32 %x) + ret void +} + +define void @pass_s4(i64 %x) { +; OPT-LABEL: @pass_s4( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i64 [[X:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_s4( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_S4_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_S4_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i64 [[X:%.*]], ptr [[TMP0]], align 8 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i64 %x) + ret void +} + +define void @pass_s5(<4 x i32> %x) { +; OPT-LABEL: @pass_s5( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(<4 x i32> [[X:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_s5( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_S5_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_S5_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store <4 x i32> [[X:%.*]], ptr [[TMP0]], align 16 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(<4 x i32> %x) + ret void +} + +define void @pass_int_s1(i32 %i, i8 %x) { +; OPT-LABEL: @pass_int_s1( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i32 [[I:%.*]], i8 [[X:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_int_s1( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S1_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 5, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_INT_S1_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i32 [[I:%.*]], ptr [[TMP0]], align 4 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[PASS_INT_S1_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1 +; ABI-NEXT: store i8 [[X:%.*]], ptr [[TMP1]], align 1 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 5, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i32 %i, i8 %x) + ret void +} + +define void @pass_int_s2(i32 %i, i16 %x) { +; OPT-LABEL: @pass_int_s2( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i32 [[I:%.*]], i16 [[X:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_int_s2( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S2_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 6, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_INT_S2_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i32 [[I:%.*]], ptr [[TMP0]], align 4 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[PASS_INT_S2_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1 +; ABI-NEXT: store i16 [[X:%.*]], ptr [[TMP1]], align 2 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 6, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i32 %i, i16 %x) + ret void +} + +define void @pass_int_s3(i32 %i, i32 %x) { +; OPT-LABEL: @pass_int_s3( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i32 [[I:%.*]], i32 [[X:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_int_s3( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S3_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_INT_S3_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i32 [[I:%.*]], ptr [[TMP0]], align 4 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[PASS_INT_S3_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1 +; ABI-NEXT: store i32 [[X:%.*]], ptr [[TMP1]], align 4 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i32 %i, i32 %x) + ret void +} + +define void @pass_int_s4(i32 %i, i64 %x) { +; OPT-LABEL: @pass_int_s4( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i32 [[I:%.*]], i64 [[X:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_int_s4( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S4_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_INT_S4_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i32 [[I:%.*]], ptr [[TMP0]], align 4 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[PASS_INT_S4_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 2 +; ABI-NEXT: store i64 [[X:%.*]], ptr [[TMP1]], align 8 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i32 %i, i64 %x) + ret void +} + +define void @pass_int_s5(i32 %i, <4 x i32> %x) { +; OPT-LABEL: @pass_int_s5( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i32 [[I:%.*]], <4 x i32> [[X:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_int_s5( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S5_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 32, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_INT_S5_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i32 [[I:%.*]], ptr [[TMP0]], align 4 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[PASS_INT_S5_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 2 +; ABI-NEXT: store <4 x i32> [[X:%.*]], ptr [[TMP1]], align 16 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 32, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i32 %i, <4 x i32> %x) + ret void +} + +define void @pass_asc(i8 %x1, i16 %x2, i32 %x3, i64 %x4, <4 x i32> %x5) { +; OPT-LABEL: @pass_asc( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i8 [[X1:%.*]], i16 [[X2:%.*]], i32 [[X3:%.*]], i64 [[X4:%.*]], <4 x i32> [[X5:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_asc( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_ASC_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 48, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_ASC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i8 [[X1:%.*]], ptr [[TMP0]], align 1 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[PASS_ASC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 2 +; ABI-NEXT: store i16 [[X2:%.*]], ptr [[TMP1]], align 2 +; ABI-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[PASS_ASC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 4 +; ABI-NEXT: store i32 [[X3:%.*]], ptr [[TMP2]], align 4 +; ABI-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PASS_ASC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 6 +; ABI-NEXT: store i64 [[X4:%.*]], ptr [[TMP3]], align 8 +; ABI-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[PASS_ASC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 8 +; ABI-NEXT: store <4 x i32> [[X5:%.*]], ptr [[TMP4]], align 16 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 48, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i8 %x1, i16 %x2, i32 %x3, i64 %x4, <4 x i32> %x5) + ret void +} + +define void @pass_dsc(<4 x i32> %x0, i64 %x1, i32 %x2, i16 %x3, i8 %x4) { +; OPT-LABEL: @pass_dsc( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(<4 x i32> [[X0:%.*]], i64 [[X1:%.*]], i32 [[X2:%.*]], i16 [[X3:%.*]], i8 [[X4:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_dsc( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_DSC_VARARG:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 33, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_DSC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store <4 x i32> [[X0:%.*]], ptr [[TMP0]], align 16 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[PASS_DSC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1 +; ABI-NEXT: store i64 [[X1:%.*]], ptr [[TMP1]], align 8 +; ABI-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[PASS_DSC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 2 +; ABI-NEXT: store i32 [[X2:%.*]], ptr [[TMP2]], align 4 +; ABI-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PASS_DSC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 3 +; ABI-NEXT: store i16 [[X3:%.*]], ptr [[TMP3]], align 2 +; ABI-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[PASS_DSC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 5 +; ABI-NEXT: store i8 [[X4:%.*]], ptr [[TMP4]], align 1 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 33, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(<4 x i32> %x0, i64 %x1, i32 %x2, i16 %x3, i8 %x4) + ret void +} + +define void @pass_multiple(i32 %i, i8 %x1, i16 %x2, i32 %x3, i64 %x4, <4 x i32> %x5) { +; OPT-LABEL: @pass_multiple( +; OPT-NEXT: entry: +; OPT-NEXT: tail call void (...) @sink(i32 [[I:%.*]], i16 [[X2:%.*]], i64 [[X4:%.*]]) +; OPT-NEXT: tail call void (...) @sink(i32 [[I]], i8 [[X1:%.*]], i32 [[X3:%.*]], <4 x i32> [[X5:%.*]]) +; OPT-NEXT: ret void +; +; ABI-LABEL: @pass_multiple( +; ABI-NEXT: entry: +; ABI-NEXT: [[VARARG_BUFFER:%.*]] = alloca [[PASS_MULTIPLE_VARARG:%.*]], align 16 +; ABI-NEXT: [[VARARG_BUFFER1:%.*]] = alloca [[PASS_MULTIPLE_VARARG_0:%.*]], align 16 +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0 +; ABI-NEXT: store i32 [[I:%.*]], ptr [[TMP0]], align 4 +; ABI-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1 +; ABI-NEXT: store i16 [[X2:%.*]], ptr [[TMP1]], align 2 +; ABI-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 3 +; ABI-NEXT: store i64 [[X4:%.*]], ptr [[TMP2]], align 8 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr [[VARARG_BUFFER]]) +; ABI-NEXT: call void @llvm.lifetime.start.p0(i64 32, ptr [[VARARG_BUFFER1]]) +; ABI-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG_0]], ptr [[VARARG_BUFFER1]], i32 0, i32 0 +; ABI-NEXT: store i32 [[I]], ptr [[TMP3]], align 4 +; ABI-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG_0]], ptr [[VARARG_BUFFER1]], i32 0, i32 1 +; ABI-NEXT: store i8 [[X1:%.*]], ptr [[TMP4]], align 1 +; ABI-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG_0]], ptr [[VARARG_BUFFER1]], i32 0, i32 3 +; ABI-NEXT: store i32 [[X3:%.*]], ptr [[TMP5]], align 4 +; ABI-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PASS_MULTIPLE_VARARG_0]], ptr [[VARARG_BUFFER1]], i32 0, i32 5 +; ABI-NEXT: store <4 x i32> [[X5:%.*]], ptr [[TMP6]], align 16 +; ABI-NEXT: call void @sink(ptr [[VARARG_BUFFER1]]) +; ABI-NEXT: call void @llvm.lifetime.end.p0(i64 32, ptr [[VARARG_BUFFER1]]) +; ABI-NEXT: ret void +; +entry: + tail call void (...) @sink(i32 %i, i16 %x2, i64 %x4) + tail call void (...) @sink(i32 %i, i8 %x1, i32 %x3, <4 x i32> %x5) + ret void +} -- GitLab From 86295dc197db2f08f4eb582ed1026a8f74ac3338 Mon Sep 17 00:00:00 2001 From: bgra8 <32298056+bgra8@users.noreply.github.com> Date: Thu, 6 Jun 2024 09:46:33 +0000 Subject: [PATCH 069/462] Revert "[Analyzer][CFG] Correctly handle rebuilt default arg and default init expression (#91879)" (#94597) This depends on https://github.com/llvm/llvm-project/pull/92527 which needs to be reverted due to https://github.com/llvm/llvm-project/pull/92527#issuecomment-2149120420. This reverts commit 905b402a5d8f1490d668f40942390ebd6e87aa8f. Co-authored-by: Bogdan Graur --- clang/lib/AST/ParentMap.cpp | 16 ------ clang/lib/Analysis/CFG.cpp | 50 +++-------------- clang/lib/Sema/SemaExpr.cpp | 4 +- clang/lib/StaticAnalyzer/Core/ExprEngine.cpp | 56 ++++++++----------- .../Analysis/cxx-uninitialized-object.cpp | 12 ++-- .../Analysis/lifetime-extended-regions.cpp | 7 ++- 6 files changed, 42 insertions(+), 103 deletions(-) diff --git a/clang/lib/AST/ParentMap.cpp b/clang/lib/AST/ParentMap.cpp index 534793b837bb..3d6a1cc84c7b 100644 --- a/clang/lib/AST/ParentMap.cpp +++ b/clang/lib/AST/ParentMap.cpp @@ -97,22 +97,6 @@ static void BuildParentMap(MapTy& M, Stmt* S, BuildParentMap(M, SubStmt, OVMode); } break; - case Stmt::CXXDefaultArgExprClass: - if (auto *Arg = dyn_cast(S)) { - if (Arg->hasRewrittenInit()) { - M[Arg->getExpr()] = S; - BuildParentMap(M, Arg->getExpr(), OVMode); - } - } - break; - case Stmt::CXXDefaultInitExprClass: - if (auto *Init = dyn_cast(S)) { - if (Init->hasRewrittenInit()) { - M[Init->getExpr()] = S; - BuildParentMap(M, Init->getExpr(), OVMode); - } - } - break; default: for (Stmt *SubStmt : S->children()) { if (SubStmt) { diff --git a/clang/lib/Analysis/CFG.cpp b/clang/lib/Analysis/CFG.cpp index 02317257c274..64e6155de090 100644 --- a/clang/lib/Analysis/CFG.cpp +++ b/clang/lib/Analysis/CFG.cpp @@ -556,10 +556,6 @@ public: private: // Visitors to walk an AST and construct the CFG. - CFGBlock *VisitCXXDefaultArgExpr(CXXDefaultArgExpr *Default, - AddStmtChoice asc); - CFGBlock *VisitCXXDefaultInitExpr(CXXDefaultInitExpr *Default, - AddStmtChoice asc); CFGBlock *VisitInitListExpr(InitListExpr *ILE, AddStmtChoice asc); CFGBlock *VisitAddrLabelExpr(AddrLabelExpr *A, AddStmtChoice asc); CFGBlock *VisitAttributedStmt(AttributedStmt *A, AddStmtChoice asc); @@ -2258,10 +2254,16 @@ CFGBlock *CFGBuilder::Visit(Stmt * S, AddStmtChoice asc, asc, ExternallyDestructed); case Stmt::CXXDefaultArgExprClass: - return VisitCXXDefaultArgExpr(cast(S), asc); - case Stmt::CXXDefaultInitExprClass: - return VisitCXXDefaultInitExpr(cast(S), asc); + // FIXME: The expression inside a CXXDefaultArgExpr is owned by the + // called function's declaration, not by the caller. If we simply add + // this expression to the CFG, we could end up with the same Expr + // appearing multiple times (PR13385). + // + // It's likewise possible for multiple CXXDefaultInitExprs for the same + // expression to be used in the same function (through aggregate + // initialization). + return VisitStmt(S, asc); case Stmt::CXXBindTemporaryExprClass: return VisitCXXBindTemporaryExpr(cast(S), asc); @@ -2431,40 +2433,6 @@ CFGBlock *CFGBuilder::VisitChildren(Stmt *S) { return B; } -CFGBlock *CFGBuilder::VisitCXXDefaultArgExpr(CXXDefaultArgExpr *Arg, - AddStmtChoice asc) { - if (Arg->hasRewrittenInit()) { - if (asc.alwaysAdd(*this, Arg)) { - autoCreateBlock(); - appendStmt(Block, Arg); - } - return VisitStmt(Arg->getExpr(), asc); - } - - // We can't add the default argument if it's not rewritten because the - // expression inside a CXXDefaultArgExpr is owned by the called function's - // declaration, not by the caller, we could end up with the same expression - // appearing multiple times. - return VisitStmt(Arg, asc); -} - -CFGBlock *CFGBuilder::VisitCXXDefaultInitExpr(CXXDefaultInitExpr *Init, - AddStmtChoice asc) { - if (Init->hasRewrittenInit()) { - if (asc.alwaysAdd(*this, Init)) { - autoCreateBlock(); - appendStmt(Block, Init); - } - return VisitStmt(Init->getExpr(), asc); - } - - // We can't add the default initializer if it's not rewritten because multiple - // CXXDefaultInitExprs for the same sub-expression to be used in the same - // function (through aggregate initialization). we could end up with the same - // expression appearing multiple times. - return VisitStmt(Init, asc); -} - CFGBlock *CFGBuilder::VisitInitListExpr(InitListExpr *ILE, AddStmtChoice asc) { if (asc.alwaysAdd(*this, ILE)) { autoCreateBlock(); diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp index fb5ca199b3fc..d8c77e3e0a5c 100644 --- a/clang/lib/Sema/SemaExpr.cpp +++ b/clang/lib/Sema/SemaExpr.cpp @@ -5608,7 +5608,6 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) { InitializationContext.emplace(Loc, Field, CurContext); Expr *Init = nullptr; - bool HasRewrittenInit = false; bool NestedDefaultChecking = isCheckingDefaultArgumentOrInitializer(); bool InLifetimeExtendingContext = isInLifetimeExtendingContext(); @@ -5658,7 +5657,6 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) { isa_and_present(Field->getInClassInitializer()); if (V.HasImmediateCalls || InLifetimeExtendingContext || ContainsAnyTemporaries) { - HasRewrittenInit = true; ExprEvalContexts.back().DelayedDefaultInitializationContext = {Loc, Field, CurContext}; ExprEvalContexts.back().IsCurrentlyCheckingDefaultArgumentOrInitializer = @@ -5702,7 +5700,7 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) { return CXXDefaultInitExpr::Create(Context, InitializationContext->Loc, Field, InitializationContext->Context, - HasRewrittenInit ? Init : nullptr); + Init); } // DR1351: diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp index 290d96611d46..197d67310728 100644 --- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp +++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp @@ -1971,45 +1971,33 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred, ExplodedNodeSet Tmp; StmtNodeBuilder Bldr2(PreVisit, Tmp, *currBldrCtx); - bool HasRewrittenInit = false; - const Expr *ArgE = nullptr; - if (const auto *DefE = dyn_cast(S)) { + const Expr *ArgE; + if (const auto *DefE = dyn_cast(S)) ArgE = DefE->getExpr(); - HasRewrittenInit = DefE->hasRewrittenInit(); - } else if (const auto *DefE = dyn_cast(S)) { + else if (const auto *DefE = dyn_cast(S)) ArgE = DefE->getExpr(); - HasRewrittenInit = DefE->hasRewrittenInit(); - } else + else llvm_unreachable("unknown constant wrapper kind"); - if (HasRewrittenInit) { - for (auto *N : PreVisit) { - ProgramStateRef state = N->getState(); - const LocationContext *LCtx = N->getLocationContext(); - state = state->BindExpr(S, LCtx, state->getSVal(ArgE, LCtx)); - Bldr2.generateNode(S, N, state); - } - } else { - // If it's not rewritten, the contents of these expressions are not - // actually part of the current function, so we fall back to constant - // evaluation. - bool IsTemporary = false; - if (const auto *MTE = dyn_cast(ArgE)) { - ArgE = MTE->getSubExpr(); - IsTemporary = true; - } - - std::optional ConstantVal = svalBuilder.getConstantVal(ArgE); - const LocationContext *LCtx = Pred->getLocationContext(); - for (auto *I : PreVisit) { - ProgramStateRef State = I->getState(); - State = State->BindExpr(S, LCtx, ConstantVal.value_or(UnknownVal())); - if (IsTemporary) - State = createTemporaryRegionIfNeeded(State, LCtx, cast(S), - cast(S)); + bool IsTemporary = false; + if (const auto *MTE = dyn_cast(ArgE)) { + ArgE = MTE->getSubExpr(); + IsTemporary = true; + } - Bldr2.generateNode(S, I, State); - } + std::optional ConstantVal = svalBuilder.getConstantVal(ArgE); + if (!ConstantVal) + ConstantVal = UnknownVal(); + + const LocationContext *LCtx = Pred->getLocationContext(); + for (const auto I : PreVisit) { + ProgramStateRef State = I->getState(); + State = State->BindExpr(S, LCtx, *ConstantVal); + if (IsTemporary) + State = createTemporaryRegionIfNeeded(State, LCtx, + cast(S), + cast(S)); + Bldr2.generateNode(S, I, State); } getCheckerManager().runCheckersForPostStmt(Dst, Tmp, S, *this); diff --git a/clang/test/Analysis/cxx-uninitialized-object.cpp b/clang/test/Analysis/cxx-uninitialized-object.cpp index aee0dae15fbf..e3fa8ae8d7f2 100644 --- a/clang/test/Analysis/cxx-uninitialized-object.cpp +++ b/clang/test/Analysis/cxx-uninitialized-object.cpp @@ -1114,27 +1114,27 @@ void fCXX11MemberInitTest1() { CXX11MemberInitTest1(); } -#ifdef PEDANTIC struct CXX11MemberInitTest2 { struct RecordType { - int a; // expected-note {{uninitialized field 'this->a'}} - int b; // expected-note {{uninitialized field 'this->b'}} + // TODO: we'd expect the note: {{uninitialized field 'this->rec.a'}} + int a; // no-note + // TODO: we'd expect the note: {{uninitialized field 'this->rec.b'}} + int b; // no-note RecordType(int) {} }; - RecordType rec = RecordType(int()); // expected-warning {{2 uninitialized fields}} + RecordType rec = RecordType(int()); int dontGetFilteredByNonPedanticMode = 0; CXX11MemberInitTest2() {} }; void fCXX11MemberInitTest2() { + // TODO: we'd expect the warning: {{2 uninitializeds field}} CXX11MemberInitTest2(); // no-warning } -#endif // PEDANTIC - //===----------------------------------------------------------------------===// // "Esoteric" primitive type tests. //===----------------------------------------------------------------------===// diff --git a/clang/test/Analysis/lifetime-extended-regions.cpp b/clang/test/Analysis/lifetime-extended-regions.cpp index 524f4e0c400d..4458ad294af7 100644 --- a/clang/test/Analysis/lifetime-extended-regions.cpp +++ b/clang/test/Analysis/lifetime-extended-regions.cpp @@ -121,10 +121,11 @@ void aggregateWithReferences() { clang_analyzer_dump(viaReference.rx); // expected-warning-re {{&lifetime_extended_object{int, viaReference, S{{[0-9]+}}} }} clang_analyzer_dump(viaReference.ry); // expected-warning-re {{&lifetime_extended_object{Composite, viaReference, S{{[0-9]+}}} }} - // The lifetime lifetime of object bound to reference members of aggregates, - // that are created from default member initializer was extended. + // FIXME: clang currently support extending lifetime of object bound to reference members of aggregates, + // that are created from default member initializer. But CFG and ExprEngine need to be updated to address this change. + // The following expect warning: {{&lifetime_extended_object{Composite, defaultInitExtended, S{{[0-9]+}}} }} RefAggregate defaultInitExtended{i}; - clang_analyzer_dump(defaultInitExtended.ry); // expected-warning-re {{&lifetime_extended_object{Composite, defaultInitExtended, S{{[0-9]+}}} }} + clang_analyzer_dump(defaultInitExtended.ry); // expected-warning {{Unknown }} } void lambda() { -- GitLab From 026fbdf934d4adc8b6abe544ba1bcfa3b64293ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Thu, 6 Jun 2024 11:20:13 +0200 Subject: [PATCH 070/462] [clang][Interp] Handle one-past-the-end pointers in SubPtr --- clang/lib/AST/Interp/Interp.h | 6 ++++-- clang/test/AST/Interp/arrays.cpp | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h index 17f05548a190..1248eeb79cbf 100644 --- a/clang/lib/AST/Interp/Interp.h +++ b/clang/lib/AST/Interp/Interp.h @@ -1773,8 +1773,10 @@ inline bool SubPtr(InterpState &S, CodePtr OpPC) { return true; } - T A = T::from(LHS.getIndex()); - T B = T::from(RHS.getIndex()); + T A = LHS.isElementPastEnd() ? T::from(LHS.getNumElems()) + : T::from(LHS.getIndex()); + T B = RHS.isElementPastEnd() ? T::from(RHS.getNumElems()) + : T::from(RHS.getIndex()); return AddSubMulHelper(S, OpPC, A.bitWidth(), A, B); } diff --git a/clang/test/AST/Interp/arrays.cpp b/clang/test/AST/Interp/arrays.cpp index dd5064d993e6..6146d41c5ff5 100644 --- a/clang/test/AST/Interp/arrays.cpp +++ b/clang/test/AST/Interp/arrays.cpp @@ -609,3 +609,9 @@ namespace ArrayMemberAccess { bool cond = a->x; } } + +namespace OnePastEndSub { + struct A {}; + constexpr A a[3][3]; + constexpr int diff2 = &a[1][3] - &a[1][0]; /// Used to crash. +} -- GitLab From e2858189bd99e6914dc2f63ab55b053a74b4e58b Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Thu, 6 Jun 2024 17:40:35 +0800 Subject: [PATCH 071/462] Revert "[serialization] no transitive decl change (#92083)" This reverts commit 97c866f6c86456b3316006e6beff47e68a81c00a. This fails on 32bit machines. See https://github.com/llvm/llvm-project/pull/92083 --- clang/include/clang/AST/DeclBase.h | 17 +- clang/include/clang/AST/DeclID.h | 18 +- .../include/clang/Serialization/ASTBitCodes.h | 31 +--- clang/include/clang/Serialization/ASTReader.h | 36 ++-- .../include/clang/Serialization/ModuleFile.h | 18 +- .../clang/Serialization/ModuleManager.h | 2 +- clang/lib/AST/DeclBase.cpp | 40 +---- clang/lib/Serialization/ASTReader.cpp | 161 ++++++++---------- clang/lib/Serialization/ASTReaderDecl.cpp | 16 +- clang/lib/Serialization/ASTWriter.cpp | 7 +- clang/lib/Serialization/ModuleFile.cpp | 3 +- .../Modules/no-transitive-decls-change.cppm | 112 ------------ 12 files changed, 156 insertions(+), 305 deletions(-) delete mode 100644 clang/test/Modules/no-transitive-decls-change.cppm diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h index 5f19af1891b7..600ce73c7f01 100644 --- a/clang/include/clang/AST/DeclBase.h +++ b/clang/include/clang/AST/DeclBase.h @@ -708,7 +708,10 @@ public: /// Set the owning module ID. This may only be called for /// deserialized Decls. - void setOwningModuleID(unsigned ID); + void setOwningModuleID(unsigned ID) { + assert(isFromASTFile() && "Only works on a deserialized declaration"); + *((unsigned*)this - 2) = ID; + } public: /// Determine the availability of the given declaration. @@ -781,11 +784,19 @@ public: /// Retrieve the global declaration ID associated with this /// declaration, which specifies where this Decl was loaded from. - GlobalDeclID getGlobalID() const; + GlobalDeclID getGlobalID() const { + if (isFromASTFile()) + return (*((const GlobalDeclID *)this - 1)); + return GlobalDeclID(); + } /// Retrieve the global ID of the module that owns this particular /// declaration. - unsigned getOwningModuleID() const; + unsigned getOwningModuleID() const { + if (isFromASTFile()) + return *((const unsigned*)this - 2); + return 0; + } private: Module *getOwningModuleSlow() const; diff --git a/clang/include/clang/AST/DeclID.h b/clang/include/clang/AST/DeclID.h index 32d2ed41a374..614ba06b6386 100644 --- a/clang/include/clang/AST/DeclID.h +++ b/clang/include/clang/AST/DeclID.h @@ -19,8 +19,6 @@ #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/iterator.h" -#include - namespace clang { /// Predefined declaration IDs. @@ -109,16 +107,12 @@ public: /// /// DeclID should only be used directly in serialization. All other users /// should use LocalDeclID or GlobalDeclID. - using DeclID = uint64_t; + using DeclID = uint32_t; protected: DeclIDBase() : ID(PREDEF_DECL_NULL_ID) {} explicit DeclIDBase(DeclID ID) : ID(ID) {} - explicit DeclIDBase(unsigned LocalID, unsigned ModuleFileIndex) { - ID = (DeclID)LocalID | ((DeclID)ModuleFileIndex << 32); - } - public: DeclID get() const { return ID; } @@ -130,10 +124,6 @@ public: bool isInvalid() const { return ID == PREDEF_DECL_NULL_ID; } - unsigned getModuleFileIndex() const { return ID >> 32; } - - unsigned getLocalDeclIndex() const; - friend bool operator==(const DeclIDBase &LHS, const DeclIDBase &RHS) { return LHS.ID == RHS.ID; } @@ -166,9 +156,6 @@ public: LocalDeclID(PredefinedDeclIDs ID) : Base(ID) {} explicit LocalDeclID(DeclID ID) : Base(ID) {} - explicit LocalDeclID(unsigned LocalID, unsigned ModuleFileIndex) - : Base(LocalID, ModuleFileIndex) {} - LocalDeclID &operator++() { ++ID; return *this; @@ -188,9 +175,6 @@ public: GlobalDeclID() : Base() {} explicit GlobalDeclID(DeclID ID) : Base(ID) {} - explicit GlobalDeclID(unsigned LocalID, unsigned ModuleFileIndex) - : Base(LocalID, ModuleFileIndex) {} - // For DeclIDIterator to be able to convert a GlobalDeclID // to a LocalDeclID. explicit operator LocalDeclID() const { return LocalDeclID(this->ID); } diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h index 3f3fb8869f5f..f59ff6af4c76 100644 --- a/clang/include/clang/Serialization/ASTBitCodes.h +++ b/clang/include/clang/Serialization/ASTBitCodes.h @@ -255,12 +255,6 @@ public: } }; -// The unaligned decl ID used in the Blobs of bistreams. -using unaligned_decl_id_t = - llvm::support::detail::packed_endian_specific_integral< - serialization::DeclID, llvm::endianness::native, - llvm::support::unaligned>; - /// The number of predefined preprocessed entity IDs. const unsigned int NUM_PREDEF_PP_ENTITY_IDS = 1; @@ -1986,46 +1980,33 @@ enum CleanupObjectKind { COK_Block, COK_CompoundLiteral }; /// Describes the categories of an Objective-C class. struct ObjCCategoriesInfo { - // The ID of the definition. Use unaligned_decl_id_t to keep - // ObjCCategoriesInfo 32-bit aligned. - unaligned_decl_id_t DefinitionID; + // The ID of the definition + LocalDeclID DefinitionID; // Offset into the array of category lists. unsigned Offset; - ObjCCategoriesInfo() = default; - ObjCCategoriesInfo(LocalDeclID ID, unsigned Offset) - : DefinitionID(ID.get()), Offset(Offset) {} - - LocalDeclID getDefinitionID() const { - return LocalDeclID(DefinitionID); - } - friend bool operator<(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.getDefinitionID() < Y.getDefinitionID(); + return X.DefinitionID < Y.DefinitionID; } friend bool operator>(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.getDefinitionID() > Y.getDefinitionID(); + return X.DefinitionID > Y.DefinitionID; } friend bool operator<=(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.getDefinitionID() <= Y.getDefinitionID(); + return X.DefinitionID <= Y.DefinitionID; } friend bool operator>=(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.getDefinitionID() >= Y.getDefinitionID(); + return X.DefinitionID >= Y.DefinitionID; } }; -static_assert(alignof(ObjCCategoriesInfo) <= 4); -static_assert(std::is_standard_layout_v && - std::is_trivial_v); - /// A key used when looking up entities by \ref DeclarationName. /// /// Different \ref DeclarationNames are mapped to different keys, but the diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index 0a9006223dcb..bea58d60d36c 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -504,6 +504,12 @@ private: /// = I + 1 has already been loaded. llvm::PagedVector DeclsLoaded; + using GlobalDeclMapType = ContinuousRangeMap; + + /// Mapping from global declaration IDs to the module in which the + /// declaration resides. + GlobalDeclMapType GlobalDeclMap; + using FileOffset = std::pair; using FileOffsetsTy = SmallVector; using DeclUpdateOffsetsMap = llvm::DenseMap; @@ -586,11 +592,10 @@ private: struct FileDeclsInfo { ModuleFile *Mod = nullptr; - ArrayRef Decls; + ArrayRef Decls; FileDeclsInfo() = default; - FileDeclsInfo(ModuleFile *Mod, - ArrayRef Decls) + FileDeclsInfo(ModuleFile *Mod, ArrayRef Decls) : Mod(Mod), Decls(Decls) {} }; @@ -599,7 +604,11 @@ private: /// An array of lexical contents of a declaration context, as a sequence of /// Decl::Kind, DeclID pairs. - using LexicalContents = ArrayRef; + using unaligned_decl_id_t = + llvm::support::detail::packed_endian_specific_integral< + serialization::DeclID, llvm::endianness::native, + llvm::support::unaligned>; + using LexicalContents = ArrayRef; /// Map from a DeclContext to its lexical contents. llvm::DenseMap> @@ -1480,11 +1489,10 @@ private: unsigned ClientLoadCapabilities); public: - class ModuleDeclIterator - : public llvm::iterator_adaptor_base< - ModuleDeclIterator, const serialization::unaligned_decl_id_t *, - std::random_access_iterator_tag, const Decl *, ptrdiff_t, - const Decl *, const Decl *> { + class ModuleDeclIterator : public llvm::iterator_adaptor_base< + ModuleDeclIterator, const LocalDeclID *, + std::random_access_iterator_tag, const Decl *, + ptrdiff_t, const Decl *, const Decl *> { ASTReader *Reader = nullptr; ModuleFile *Mod = nullptr; @@ -1492,11 +1500,11 @@ public: ModuleDeclIterator() : iterator_adaptor_base(nullptr) {} ModuleDeclIterator(ASTReader *Reader, ModuleFile *Mod, - const serialization::unaligned_decl_id_t *Pos) + const LocalDeclID *Pos) : iterator_adaptor_base(Pos), Reader(Reader), Mod(Mod) {} value_type operator*() const { - return Reader->GetDecl(Reader->getGlobalDeclID(*Mod, (LocalDeclID)*I)); + return Reader->GetDecl(Reader->getGlobalDeclID(*Mod, *I)); } value_type operator->() const { return **this; } @@ -1536,9 +1544,6 @@ private: StringRef Arg2 = StringRef(), StringRef Arg3 = StringRef()) const; void Error(llvm::Error &&Err) const; - /// Translate a \param GlobalDeclID to the index of DeclsLoaded array. - unsigned translateGlobalDeclIDToIndex(GlobalDeclID ID) const; - public: /// Load the AST file and validate its contents against the given /// Preprocessor. @@ -1910,8 +1915,7 @@ public: /// Retrieve the module file that owns the given declaration, or NULL /// if the declaration is not from a module file. - ModuleFile *getOwningModuleFile(const Decl *D) const; - ModuleFile *getOwningModuleFile(GlobalDeclID ID) const; + ModuleFile *getOwningModuleFile(const Decl *D); /// Returns the source location for the decl \p ID. SourceLocation getSourceLocationForDeclID(GlobalDeclID ID); diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 56193d44dd6f..992d26a8b88c 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -454,11 +454,23 @@ public: /// by the declaration ID (-1). const DeclOffset *DeclOffsets = nullptr; - /// Base declaration index in ASTReader for declarations local to this module. - unsigned BaseDeclIndex = 0; + /// Base declaration ID for declarations local to this module. + serialization::DeclID BaseDeclID = 0; + + /// Remapping table for declaration IDs in this module. + ContinuousRangeMap DeclRemap; + + /// Mapping from the module files that this module file depends on + /// to the base declaration ID for that module as it is understood within this + /// module. + /// + /// This is effectively a reverse global-to-local mapping for declaration + /// IDs, so that we can interpret a true global ID (for this translation unit) + /// as a local ID (for this module file). + llvm::DenseMap GlobalToLocalDeclIDs; /// Array of file-level DeclIDs sorted by file. - const serialization::unaligned_decl_id_t *FileSortedDecls = nullptr; + const LocalDeclID *FileSortedDecls = nullptr; unsigned NumFileSortedDecls = 0; /// Array of category list location information within this diff --git a/clang/include/clang/Serialization/ModuleManager.h b/clang/include/clang/Serialization/ModuleManager.h index f898dab39f06..d770bc52eaf4 100644 --- a/clang/include/clang/Serialization/ModuleManager.h +++ b/clang/include/clang/Serialization/ModuleManager.h @@ -45,7 +45,7 @@ namespace serialization { /// Manages the set of modules loaded by an AST reader. class ModuleManager { /// The chain of AST files, in the order in which we started to load - /// them. + /// them (this order isn't really useful for anything). SmallVector, 2> Chain; /// The chain of non-module PCH files. The first entry is the one named diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp index 7f1ed9c691e9..1e9c879e371b 100644 --- a/clang/lib/AST/DeclBase.cpp +++ b/clang/lib/AST/DeclBase.cpp @@ -74,17 +74,18 @@ void *Decl::operator new(std::size_t Size, const ASTContext &Context, GlobalDeclID ID, std::size_t Extra) { // Allocate an extra 8 bytes worth of storage, which ensures that the // resulting pointer will still be 8-byte aligned. - static_assert(sizeof(uint64_t) >= alignof(Decl), "Decl won't be misaligned"); + static_assert(sizeof(unsigned) * 2 >= alignof(Decl), + "Decl won't be misaligned"); void *Start = Context.Allocate(Size + Extra + 8); void *Result = (char*)Start + 8; - uint64_t *PrefixPtr = (uint64_t *)Result - 1; + unsigned *PrefixPtr = (unsigned *)Result - 2; - *PrefixPtr = ID.get(); + // Zero out the first 4 bytes; this is used to store the owning module ID. + PrefixPtr[0] = 0; - // We leave the upper 16 bits to store the module IDs. 48 bits should be - // sufficient to store a declaration ID. - assert(*PrefixPtr < llvm::maskTrailingOnes(48)); + // Store the global declaration ID in the second 4 bytes. + PrefixPtr[1] = ID.get(); return Result; } @@ -110,29 +111,6 @@ void *Decl::operator new(std::size_t Size, const ASTContext &Ctx, return ::operator new(Size + Extra, Ctx); } -GlobalDeclID Decl::getGlobalID() const { - if (!isFromASTFile()) - return GlobalDeclID(); - // See the comments in `Decl::operator new` for details. - uint64_t ID = *((const uint64_t *)this - 1); - return GlobalDeclID(ID & llvm::maskTrailingOnes(48)); -} - -unsigned Decl::getOwningModuleID() const { - if (!isFromASTFile()) - return 0; - - uint64_t ID = *((const uint64_t *)this - 1); - return ID >> 48; -} - -void Decl::setOwningModuleID(unsigned ID) { - assert(isFromASTFile() && "Only works on a deserialized declaration"); - uint64_t *IDAddress = (uint64_t *)this - 1; - *IDAddress &= llvm::maskTrailingOnes(48); - *IDAddress |= (uint64_t)ID << 48; -} - Module *Decl::getOwningModuleSlow() const { assert(isFromASTFile() && "Not from AST file?"); return getASTContext().getExternalSource()->getModule(getOwningModuleID()); @@ -2185,7 +2163,3 @@ DependentDiagnostic *DependentDiagnostic::Create(ASTContext &C, return DD; } - -unsigned DeclIDBase::getLocalDeclIndex() const { - return ID & llvm::maskTrailingOnes(32); -} diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 1ff770b05584..33cea32c3be6 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -1658,7 +1658,7 @@ bool ASTReader::ReadSLocEntry(int ID) { unsigned NumFileDecls = Record[7]; if (NumFileDecls && ContextObj) { - const unaligned_decl_id_t *FirstDecl = F->FileSortedDecls + Record[6]; + const LocalDeclID *FirstDecl = F->FileSortedDecls + Record[6]; assert(F->FileSortedDecls && "FILE_SORTED_DECLS not encountered yet ?"); FileDeclIDs[FID] = FileDeclsInfo(F, llvm::ArrayRef(FirstDecl, NumFileDecls)); @@ -3377,11 +3377,26 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F, "duplicate DECL_OFFSET record in AST file"); F.DeclOffsets = (const DeclOffset *)Blob.data(); F.LocalNumDecls = Record[0]; - F.BaseDeclIndex = getTotalNumDecls(); + unsigned LocalBaseDeclID = Record[1]; + F.BaseDeclID = getTotalNumDecls(); - if (F.LocalNumDecls > 0) - DeclsLoaded.resize(DeclsLoaded.size() + F.LocalNumDecls); + if (F.LocalNumDecls > 0) { + // Introduce the global -> local mapping for declarations within this + // module. + GlobalDeclMap.insert(std::make_pair( + GlobalDeclID(getTotalNumDecls() + NUM_PREDEF_DECL_IDS), &F)); + + // Introduce the local -> global mapping for declarations within this + // module. + F.DeclRemap.insertOrReplace( + std::make_pair(LocalBaseDeclID, F.BaseDeclID - LocalBaseDeclID)); + + // Introduce the global -> local mapping for declarations within this + // module. + F.GlobalToLocalDeclIDs[&F] = LocalBaseDeclID; + DeclsLoaded.resize(DeclsLoaded.size() + F.LocalNumDecls); + } break; } @@ -3616,7 +3631,7 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F, break; case FILE_SORTED_DECLS: - F.FileSortedDecls = (const unaligned_decl_id_t *)Blob.data(); + F.FileSortedDecls = (const LocalDeclID *)Blob.data(); F.NumFileSortedDecls = Record[0]; break; @@ -4043,6 +4058,7 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { RemapBuilder PreprocessedEntityRemap(F.PreprocessedEntityRemap); RemapBuilder SubmoduleRemap(F.SubmoduleRemap); RemapBuilder SelectorRemap(F.SelectorRemap); + RemapBuilder DeclRemap(F.DeclRemap); RemapBuilder TypeRemap(F.TypeRemap); auto &ImportedModuleVector = F.TransitiveImports; @@ -4081,6 +4097,8 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { endian::readNext(Data); uint32_t SelectorIDOffset = endian::readNext(Data); + uint32_t DeclIDOffset = + endian::readNext(Data); uint32_t TypeIndexOffset = endian::readNext(Data); @@ -4098,7 +4116,11 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { PreprocessedEntityRemap); mapOffset(SubmoduleIDOffset, OM->BaseSubmoduleID, SubmoduleRemap); mapOffset(SelectorIDOffset, OM->BaseSelectorID, SelectorRemap); + mapOffset(DeclIDOffset, OM->BaseDeclID, DeclRemap); mapOffset(TypeIndexOffset, OM->BaseTypeIndex, TypeRemap); + + // Global -> local mappings. + F.GlobalToLocalDeclIDs[OM] = DeclIDOffset; } } @@ -4623,7 +4645,7 @@ ASTReader::ASTReadResult ASTReader::ReadAST(StringRef FileName, ModuleKind Type, // that we load any additional categories. if (ContextObj) { for (unsigned I = 0, N = ObjCClassesLoaded.size(); I != N; ++I) { - loadObjCCategories(ObjCClassesLoaded[I]->getGlobalID(), + loadObjCCategories(GlobalDeclID(ObjCClassesLoaded[I]->getGlobalID()), ObjCClassesLoaded[I], PreviousGeneration); } } @@ -7622,25 +7644,18 @@ CXXBaseSpecifier *ASTReader::GetExternalCXXBaseSpecifiers(uint64_t Offset) { GlobalDeclID ASTReader::getGlobalDeclID(ModuleFile &F, LocalDeclID LocalID) const { - if (LocalID.get() < NUM_PREDEF_DECL_IDS) - return GlobalDeclID(LocalID.get()); - - unsigned OwningModuleFileIndex = LocalID.getModuleFileIndex(); - DeclID ID = LocalID.getLocalDeclIndex(); + DeclID ID = LocalID.get(); + if (ID < NUM_PREDEF_DECL_IDS) + return GlobalDeclID(ID); if (!F.ModuleOffsetMap.empty()) ReadModuleOffsetMap(F); - ModuleFile *OwningModuleFile = - OwningModuleFileIndex == 0 - ? &F - : F.TransitiveImports[OwningModuleFileIndex - 1]; - - if (OwningModuleFileIndex == 0) - ID -= NUM_PREDEF_DECL_IDS; + ContinuousRangeMap::iterator I = + F.DeclRemap.find(ID - NUM_PREDEF_DECL_IDS); + assert(I != F.DeclRemap.end() && "Invalid index into decl index remap"); - uint64_t NewModuleFileIndex = OwningModuleFile->Index + 1; - return GlobalDeclID(ID, NewModuleFileIndex); + return GlobalDeclID(ID + I->second); } bool ASTReader::isDeclIDFromModule(GlobalDeclID ID, ModuleFile &M) const { @@ -7648,33 +7663,31 @@ bool ASTReader::isDeclIDFromModule(GlobalDeclID ID, ModuleFile &M) const { if (ID.get() < NUM_PREDEF_DECL_IDS) return false; - unsigned ModuleFileIndex = ID.getModuleFileIndex(); - return M.Index == ModuleFileIndex - 1; -} - -ModuleFile *ASTReader::getOwningModuleFile(GlobalDeclID ID) const { - // Predefined decls aren't from any module. - if (ID.get() < NUM_PREDEF_DECL_IDS) - return nullptr; - - uint64_t ModuleFileIndex = ID.getModuleFileIndex(); - assert(ModuleFileIndex && "Untranslated Local Decl?"); - - return &getModuleManager()[ModuleFileIndex - 1]; + return ID.get() - NUM_PREDEF_DECL_IDS >= M.BaseDeclID && + ID.get() - NUM_PREDEF_DECL_IDS < M.BaseDeclID + M.LocalNumDecls; } -ModuleFile *ASTReader::getOwningModuleFile(const Decl *D) const { +ModuleFile *ASTReader::getOwningModuleFile(const Decl *D) { if (!D->isFromASTFile()) return nullptr; - - return getOwningModuleFile(D->getGlobalID()); + GlobalDeclMapType::const_iterator I = + GlobalDeclMap.find(GlobalDeclID(D->getGlobalID())); + assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); + return I->second; } SourceLocation ASTReader::getSourceLocationForDeclID(GlobalDeclID ID) { if (ID.get() < NUM_PREDEF_DECL_IDS) return SourceLocation(); - if (Decl *D = GetExistingDecl(ID)) + unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; + + if (Index > DeclsLoaded.size()) { + Error("declaration ID out-of-range for AST file"); + return SourceLocation(); + } + + if (Decl *D = DeclsLoaded[Index]) return D->getLocation(); SourceLocation Loc; @@ -7741,19 +7754,8 @@ static Decl *getPredefinedDecl(ASTContext &Context, PredefinedDeclIDs ID) { llvm_unreachable("PredefinedDeclIDs unknown enum value"); } -unsigned ASTReader::translateGlobalDeclIDToIndex(GlobalDeclID GlobalID) const { - ModuleFile *OwningModuleFile = getOwningModuleFile(GlobalID); - if (!OwningModuleFile) { - assert(GlobalID.get() < NUM_PREDEF_DECL_IDS && "Untransalted Global ID?"); - return GlobalID.get(); - } - - return OwningModuleFile->BaseDeclIndex + GlobalID.getLocalDeclIndex(); -} - Decl *ASTReader::GetExistingDecl(GlobalDeclID ID) { assert(ContextObj && "reading decl with no AST context"); - if (ID.get() < NUM_PREDEF_DECL_IDS) { Decl *D = getPredefinedDecl(*ContextObj, (PredefinedDeclIDs)ID); if (D) { @@ -7766,7 +7768,7 @@ Decl *ASTReader::GetExistingDecl(GlobalDeclID ID) { return D; } - unsigned Index = translateGlobalDeclIDToIndex(ID); + unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; if (Index >= DeclsLoaded.size()) { assert(0 && "declaration ID out-of-range for AST file"); @@ -7781,7 +7783,7 @@ Decl *ASTReader::GetDecl(GlobalDeclID ID) { if (ID.get() < NUM_PREDEF_DECL_IDS) return GetExistingDecl(ID); - unsigned Index = translateGlobalDeclIDToIndex(ID); + unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; if (Index >= DeclsLoaded.size()) { assert(0 && "declaration ID out-of-range for AST file"); @@ -7800,31 +7802,20 @@ Decl *ASTReader::GetDecl(GlobalDeclID ID) { LocalDeclID ASTReader::mapGlobalIDToModuleFileGlobalID(ModuleFile &M, GlobalDeclID GlobalID) { - if (GlobalID.get() < NUM_PREDEF_DECL_IDS) - return LocalDeclID(GlobalID.get()); - - if (!M.ModuleOffsetMap.empty()) - ReadModuleOffsetMap(M); - - ModuleFile *Owner = getOwningModuleFile(GlobalID); - DeclID ID = GlobalID.getLocalDeclIndex(); - - if (Owner == &M) { - ID += NUM_PREDEF_DECL_IDS; + DeclID ID = GlobalID.get(); + if (ID < NUM_PREDEF_DECL_IDS) return LocalDeclID(ID); - } - uint64_t OrignalModuleFileIndex = 0; - for (unsigned I = 0; I < M.TransitiveImports.size(); I++) - if (M.TransitiveImports[I] == Owner) { - OrignalModuleFileIndex = I + 1; - break; - } + GlobalDeclMapType::const_iterator I = GlobalDeclMap.find(GlobalID); + assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); + ModuleFile *Owner = I->second; - if (!OrignalModuleFileIndex) + llvm::DenseMap::iterator Pos = + M.GlobalToLocalDeclIDs.find(Owner); + if (Pos == M.GlobalToLocalDeclIDs.end()) return LocalDeclID(); - return LocalDeclID(ID, OrignalModuleFileIndex); + return LocalDeclID(ID - Owner->BaseDeclID + Pos->second); } GlobalDeclID ASTReader::ReadDeclID(ModuleFile &F, const RecordData &Record, @@ -7903,34 +7894,32 @@ void ASTReader::FindExternalLexicalDecls( namespace { -class UnalignedDeclIDComp { +class DeclIDComp { ASTReader &Reader; ModuleFile &Mod; public: - UnalignedDeclIDComp(ASTReader &Reader, ModuleFile &M) - : Reader(Reader), Mod(M) {} + DeclIDComp(ASTReader &Reader, ModuleFile &M) : Reader(Reader), Mod(M) {} - bool operator()(unaligned_decl_id_t L, unaligned_decl_id_t R) const { + bool operator()(LocalDeclID L, LocalDeclID R) const { SourceLocation LHS = getLocation(L); SourceLocation RHS = getLocation(R); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - bool operator()(SourceLocation LHS, unaligned_decl_id_t R) const { + bool operator()(SourceLocation LHS, LocalDeclID R) const { SourceLocation RHS = getLocation(R); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - bool operator()(unaligned_decl_id_t L, SourceLocation RHS) const { + bool operator()(LocalDeclID L, SourceLocation RHS) const { SourceLocation LHS = getLocation(L); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - SourceLocation getLocation(unaligned_decl_id_t ID) const { + SourceLocation getLocation(LocalDeclID ID) const { return Reader.getSourceManager().getFileLoc( - Reader.getSourceLocationForDeclID( - Reader.getGlobalDeclID(Mod, (LocalDeclID)ID))); + Reader.getSourceLocationForDeclID(Reader.getGlobalDeclID(Mod, ID))); } }; @@ -7953,8 +7942,8 @@ void ASTReader::FindFileRegionDecls(FileID File, BeginLoc = SM.getLocForStartOfFile(File).getLocWithOffset(Offset); SourceLocation EndLoc = BeginLoc.getLocWithOffset(Length); - UnalignedDeclIDComp DIDComp(*this, *DInfo.Mod); - ArrayRef::iterator BeginIt = + DeclIDComp DIDComp(*this, *DInfo.Mod); + ArrayRef::iterator BeginIt = llvm::lower_bound(DInfo.Decls, BeginLoc, DIDComp); if (BeginIt != DInfo.Decls.begin()) --BeginIt; @@ -7963,18 +7952,17 @@ void ASTReader::FindFileRegionDecls(FileID File, // to backtrack until we find it otherwise we will fail to report that the // region overlaps with an objc container. while (BeginIt != DInfo.Decls.begin() && - GetDecl(getGlobalDeclID(*DInfo.Mod, (LocalDeclID)(*BeginIt))) + GetDecl(getGlobalDeclID(*DInfo.Mod, *BeginIt)) ->isTopLevelDeclInObjCContainer()) --BeginIt; - ArrayRef::iterator EndIt = + ArrayRef::iterator EndIt = llvm::upper_bound(DInfo.Decls, EndLoc, DIDComp); if (EndIt != DInfo.Decls.end()) ++EndIt; - for (ArrayRef::iterator DIt = BeginIt; DIt != EndIt; - ++DIt) - Decls.push_back(GetDecl(getGlobalDeclID(*DInfo.Mod, (LocalDeclID)(*DIt)))); + for (ArrayRef::iterator DIt = BeginIt; DIt != EndIt; ++DIt) + Decls.push_back(GetDecl(getGlobalDeclID(*DInfo.Mod, *DIt))); } bool @@ -8181,6 +8169,7 @@ LLVM_DUMP_METHOD void ASTReader::dump() { dumpModuleIDMap("Global bit offset map", GlobalBitOffsetsMap); dumpModuleIDMap("Global source location entry map", GlobalSLocEntryMap); dumpModuleIDMap("Global type map", GlobalTypeMap); + dumpModuleIDMap("Global declaration map", GlobalDeclMap); dumpModuleIDMap("Global identifier map", GlobalIdentifierMap); dumpModuleIDMap("Global macro map", GlobalMacroMap); dumpModuleIDMap("Global submodule map", GlobalSubmoduleMap); diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp index cf2dc32e30b9..765c083ce8df 100644 --- a/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/clang/lib/Serialization/ASTReaderDecl.cpp @@ -2924,7 +2924,7 @@ void ASTDeclReader::mergeTemplatePattern(RedeclarableTemplateDecl *D, auto *ExistingPattern = Existing->getTemplatedDecl(); RedeclarableResult Result( /*MergeWith*/ ExistingPattern, - DPattern->getCanonicalDecl()->getGlobalID(), IsKeyDecl); + GlobalDeclID(DPattern->getCanonicalDecl()->getGlobalID()), IsKeyDecl); if (auto *DClass = dyn_cast(DPattern)) { // Merge with any existing definition. @@ -3245,10 +3245,11 @@ bool ASTReader::isConsumerInterestedIn(Decl *D) { /// Get the correct cursor and offset for loading a declaration. ASTReader::RecordLocation ASTReader::DeclCursorForID(GlobalDeclID ID, SourceLocation &Loc) { - ModuleFile *M = getOwningModuleFile(ID); - assert(M); - unsigned LocalDeclIndex = ID.getLocalDeclIndex(); - const DeclOffset &DOffs = M->DeclOffsets[LocalDeclIndex]; + GlobalDeclMapType::iterator I = GlobalDeclMap.find(ID); + assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); + ModuleFile *M = I->second; + const DeclOffset &DOffs = + M->DeclOffsets[ID.get() - M->BaseDeclID - NUM_PREDEF_DECL_IDS]; Loc = ReadSourceLocation(*M, DOffs.getRawLoc()); return RecordLocation(M, DOffs.getBitOffset(M->DeclsBlockStartOffset)); } @@ -3791,6 +3792,7 @@ void ASTReader::markIncompleteDeclChain(Decl *D) { /// Read the declaration at the given offset from the AST file. Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) { + unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; SourceLocation DeclLoc; RecordLocation Loc = DeclCursorForID(ID, DeclLoc); llvm::BitstreamCursor &DeclsCursor = Loc.F->DeclsCursor; @@ -4121,7 +4123,7 @@ Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) { } assert(D && "Unknown declaration reading AST file"); - LoadedDecl(translateGlobalDeclIDToIndex(ID), D); + LoadedDecl(Index, D); // Set the DeclContext before doing any deserialization, to make sure internal // calls to Decl::getASTContext() by Decl's methods will find the // TranslationUnitDecl without crashing. @@ -4447,7 +4449,7 @@ namespace { M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap, Compare); if (Result == M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap || - Result->getDefinitionID() != LocalID) { + Result->DefinitionID != LocalID) { // We didn't find anything. If the class definition is in this module // file, then the module files it depends on cannot have any categories, // so suppress further lookup. diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index ee3e687636e6..953990a4aca6 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -3357,10 +3357,12 @@ void ASTWriter::WriteTypeDeclOffsets() { Abbrev = std::make_shared(); Abbrev->Add(BitCodeAbbrevOp(DECL_OFFSET)); Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of declarations + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // base decl ID Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // declarations block unsigned DeclOffsetAbbrev = Stream.EmitAbbrev(std::move(Abbrev)); { - RecordData::value_type Record[] = {DECL_OFFSET, DeclOffsets.size()}; + RecordData::value_type Record[] = {DECL_OFFSET, DeclOffsets.size(), + FirstDeclID.get() - NUM_PREDEF_DECL_IDS}; Stream.EmitRecordWithBlob(DeclOffsetAbbrev, Record, bytes(DeclOffsets)); } } @@ -5421,6 +5423,7 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, M.NumPreprocessedEntities); writeBaseIDOrNone(M.BaseSubmoduleID, M.LocalNumSubmodules); writeBaseIDOrNone(M.BaseSelectorID, M.LocalNumSelectors); + writeBaseIDOrNone(M.BaseDeclID, M.LocalNumDecls); writeBaseIDOrNone(M.BaseTypeIndex, M.LocalNumTypes); } } @@ -6615,11 +6618,13 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) { // Note, this will get called multiple times, once one the reader starts up // and again each time it's done reading a PCH or module. + FirstDeclID = LocalDeclID(NUM_PREDEF_DECL_IDS + Chain->getTotalNumDecls()); FirstTypeID = NUM_PREDEF_TYPE_IDS + Chain->getTotalNumTypes(); FirstIdentID = NUM_PREDEF_IDENT_IDS + Chain->getTotalNumIdentifiers(); FirstMacroID = NUM_PREDEF_MACRO_IDS + Chain->getTotalNumMacros(); FirstSubmoduleID = NUM_PREDEF_SUBMODULE_IDS + Chain->getTotalNumSubmodules(); FirstSelectorID = NUM_PREDEF_SELECTOR_IDS + Chain->getTotalNumSelectors(); + NextDeclID = FirstDeclID; NextTypeID = FirstTypeID; NextIdentID = FirstIdentID; NextMacroID = FirstMacroID; diff --git a/clang/lib/Serialization/ModuleFile.cpp b/clang/lib/Serialization/ModuleFile.cpp index f64a59bd9489..2c42d33a8f5d 100644 --- a/clang/lib/Serialization/ModuleFile.cpp +++ b/clang/lib/Serialization/ModuleFile.cpp @@ -87,6 +87,7 @@ LLVM_DUMP_METHOD void ModuleFile::dump() { << " Number of types: " << LocalNumTypes << '\n'; dumpLocalRemap("Type index local -> global map", TypeRemap); - llvm::errs() << " Base decl index: " << BaseDeclIndex << '\n' + llvm::errs() << " Base decl ID: " << BaseDeclID << '\n' << " Number of decls: " << LocalNumDecls << '\n'; + dumpLocalRemap("Decl ID local -> global map", DeclRemap); } diff --git a/clang/test/Modules/no-transitive-decls-change.cppm b/clang/test/Modules/no-transitive-decls-change.cppm deleted file mode 100644 index 42ac061bc90b..000000000000 --- a/clang/test/Modules/no-transitive-decls-change.cppm +++ /dev/null @@ -1,112 +0,0 @@ -// Testing that changing a declaration in an unused module file won't change -// the BMI of the current module file. -// -// RUN: rm -rf %t -// RUN: split-file %s %t -// -// RUN: %clang_cc1 -std=c++20 %t/m-partA.cppm -emit-reduced-module-interface -o %t/m-partA.pcm -// RUN: %clang_cc1 -std=c++20 %t/m-partA.v1.cppm -emit-reduced-module-interface -o \ -// RUN: %t/m-partA.v1.pcm -// RUN: %clang_cc1 -std=c++20 %t/m-partB.cppm -emit-reduced-module-interface -o %t/m-partB.pcm -// RUN: %clang_cc1 -std=c++20 %t/m.cppm -emit-reduced-module-interface -o %t/m.pcm \ -// RUN: -fmodule-file=m:partA=%t/m-partA.pcm -fmodule-file=m:partB=%t/m-partB.pcm -// RUN: %clang_cc1 -std=c++20 %t/m.cppm -emit-reduced-module-interface -o %t/m.v1.pcm \ -// RUN: -fmodule-file=m:partA=%t/m-partA.v1.pcm -fmodule-file=m:partB=%t/m-partB.pcm -// -// RUN: %clang_cc1 -std=c++20 %t/useBOnly.cppm -emit-reduced-module-interface -o %t/useBOnly.pcm \ -// RUN: -fmodule-file=m=%t/m.pcm -fmodule-file=m:partA=%t/m-partA.pcm \ -// RUN: -fmodule-file=m:partB=%t/m-partB.pcm -// RUN: %clang_cc1 -std=c++20 %t/useBOnly.cppm -emit-reduced-module-interface -o %t/useBOnly.v1.pcm \ -// RUN: -fmodule-file=m=%t/m.v1.pcm -fmodule-file=m:partA=%t/m-partA.v1.pcm \ -// RUN: -fmodule-file=m:partB=%t/m-partB.pcm -// Since useBOnly only uses partB from module M, the change in partA shouldn't affect -// useBOnly. -// RUN: diff %t/useBOnly.pcm %t/useBOnly.v1.pcm &> /dev/null - -//--- m-partA.cppm -export module m:partA; - -namespace A_Impl { - inline int getAImpl() { - return 43; - } - - inline int getA2Impl() { - return 43; - } -} - -namespace A { - using A_Impl::getAImpl; -} - -export inline int getA() { - return 43; -} - -export inline int getA2(int) { - return 88; -} - -//--- m-partA.v1.cppm -export module m:partA; - -namespace A_Impl { - inline int getAImpl() { - return 43; - } - - inline int getA2Impl() { - return 43; - } -} - -namespace A { - using A_Impl::getAImpl; - // Adding a new declaration without introducing a new declaration name. - using A_Impl::getA2Impl; -} - -inline int getA() { - return 43; -} - -inline int getA2(int) { - return 88; -} - -// Now we add a new declaration without introducing new identifier and new types. -// The consuming module which didn't use m:partA completely is expected to be -// not changed. -inline int getA(int) { - return 88; -} - -//--- m-partB.cppm -export module m:partB; - -export inline int getB() { - return 430; -} - -//--- m.cppm -export module m; -export import :partA; -export import :partB; - -//--- useBOnly.cppm -export module useBOnly; -import m; - -export inline int get() { - return getB(); -} - -//--- useAOnly.cppm -export module useAOnly; -import m; - -export inline int get() { - A a; - return a.getValue(); -} -- GitLab From 7f52e4c755fcd02232964f19bb0226878255f274 Mon Sep 17 00:00:00 2001 From: bgra8 <32298056+bgra8@users.noreply.github.com> Date: Thu, 6 Jun 2024 09:59:52 +0000 Subject: [PATCH 072/462] Revert "Reapply "[Clang][CWG1815] Support lifetime extension of temporary created by aggregate initialization using a default member initializer" (#92527)" (#94600) Reverting due to https://github.com/llvm/llvm-project/pull/92527#issuecomment-2149120420. This reverts commit f049d72ac2bcc40fd91d4e95148658021fb24bf1. Co-authored-by: Bogdan Graur --- .../clang/Basic/DiagnosticSemaKinds.td | 6 ++ clang/lib/Sema/SemaExpr.cpp | 31 ++------ clang/lib/Sema/SemaExprCXX.cpp | 3 + clang/lib/Sema/SemaInit.cpp | 19 ++++- clang/lib/Sema/TreeTransform.h | 12 +-- clang/test/AST/ast-dump-default-init-json.cpp | 6 +- clang/test/AST/ast-dump-default-init.cpp | 2 +- .../Analysis/lifetime-extended-regions.cpp | 9 +-- clang/test/CXX/drs/cwg16xx.cpp | 2 + clang/test/CXX/drs/cwg18xx.cpp | 19 ++--- clang/test/CXX/special/class.temporary/p6.cpp | 34 --------- clang/test/SemaCXX/constexpr-default-arg.cpp | 4 +- .../cxx11-default-member-initializers.cpp | 74 ------------------- clang/test/SemaCXX/eval-crashes.cpp | 6 +- clang/www/cxx_dr_status.html | 2 +- 15 files changed, 61 insertions(+), 168 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 87745140cb0e..9f0b6f5a3638 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -10082,6 +10082,12 @@ def warn_new_dangling_initializer_list : Warning< "the allocated initializer list}0 " "will be destroyed at the end of the full-expression">, InGroup; +def warn_unsupported_lifetime_extension : Warning< + "lifetime extension of " + "%select{temporary|backing array of initializer list}0 created " + "by aggregate initialization using a default member initializer " + "is not yet supported; lifetime of %select{temporary|backing array}0 " + "will end at the end of the full-expression">, InGroup; // For non-floating point, expressions of the form x == x or x != x // should result in a warning, since these always evaluate to a constant. diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp index d8c77e3e0a5c..76145f291887 100644 --- a/clang/lib/Sema/SemaExpr.cpp +++ b/clang/lib/Sema/SemaExpr.cpp @@ -5572,9 +5572,10 @@ ExprResult Sema::BuildCXXDefaultArgExpr(SourceLocation CallLoc, Res = Immediate.TransformInitializer(Param->getInit(), /*NotCopy=*/false); }); - if (Res.isUsable()) - Res = ConvertParamDefaultArgument(Param, Res.get(), - Res.get()->getBeginLoc()); + if (Res.isInvalid()) + return ExprError(); + Res = ConvertParamDefaultArgument(Param, Res.get(), + Res.get()->getBeginLoc()); if (Res.isInvalid()) return ExprError(); Init = Res.get(); @@ -5610,7 +5611,7 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) { Expr *Init = nullptr; bool NestedDefaultChecking = isCheckingDefaultArgumentOrInitializer(); - bool InLifetimeExtendingContext = isInLifetimeExtendingContext(); + EnterExpressionEvaluationContext EvalContext( *this, ExpressionEvaluationContext::PotentiallyEvaluated, Field); @@ -5645,35 +5646,19 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) { ImmediateCallVisitor V(getASTContext()); if (!NestedDefaultChecking) V.TraverseDecl(Field); - - // CWG1815 - // Support lifetime extension of temporary created by aggregate - // initialization using a default member initializer. We should always rebuild - // the initializer if it contains any temporaries (if the initializer - // expression is an ExprWithCleanups). Then make sure the normal lifetime - // extension code recurses into the default initializer and does lifetime - // extension when warranted. - bool ContainsAnyTemporaries = - isa_and_present(Field->getInClassInitializer()); - if (V.HasImmediateCalls || InLifetimeExtendingContext || - ContainsAnyTemporaries) { + if (V.HasImmediateCalls) { ExprEvalContexts.back().DelayedDefaultInitializationContext = {Loc, Field, CurContext}; ExprEvalContexts.back().IsCurrentlyCheckingDefaultArgumentOrInitializer = NestedDefaultChecking; - // Pass down lifetime extending flag, and collect temporaries in - // CreateMaterializeTemporaryExpr when we rewrite the call argument. - keepInLifetimeExtendingContext(); + EnsureImmediateInvocationInDefaultArgs Immediate(*this); ExprResult Res; - - // Rebuild CXXDefaultInitExpr might cause diagnostics. - SFINAETrap Trap(*this); runWithSufficientStackSpace(Loc, [&] { Res = Immediate.TransformInitializer(Field->getInClassInitializer(), /*CXXDirectInit=*/false); }); - if (Res.isUsable()) + if (!Res.isInvalid()) Res = ConvertMemberDefaultInitExpression(Field, Res.get(), Loc); if (Res.isInvalid()) { Field->setInvalidDecl(); diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp index eb0d987f63da..cf461a68d552 100644 --- a/clang/lib/Sema/SemaExprCXX.cpp +++ b/clang/lib/Sema/SemaExprCXX.cpp @@ -1555,6 +1555,9 @@ Sema::BuildCXXTypeConstructExpr(TypeSourceInfo *TInfo, bool ListInitialization) { QualType Ty = TInfo->getType(); SourceLocation TyBeginLoc = TInfo->getTypeLoc().getBeginLoc(); + + assert((!ListInitialization || Exprs.size() == 1) && + "List initialization must have exactly one expression."); SourceRange FullRange = SourceRange(TyBeginLoc, RParenOrBraceLoc); InitializedEntity Entity = diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index 9ed3e8a0df02..ed8b226a6b39 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -8063,6 +8063,11 @@ static void visitLocalsRetainedByInitializer(IndirectLocalPath &Path, enum PathLifetimeKind { /// Lifetime-extend along this path. Extend, + /// We should lifetime-extend, but we don't because (due to technical + /// limitations) we can't. This happens for default member initializers, + /// which we don't clone for every use, so we don't have a unique + /// MaterializeTemporaryExpr to update. + ShouldExtend, /// Do not lifetime extend along this path. NoExtend }; @@ -8074,7 +8079,7 @@ shouldLifetimeExtendThroughPath(const IndirectLocalPath &Path) { PathLifetimeKind Kind = PathLifetimeKind::Extend; for (auto Elem : Path) { if (Elem.Kind == IndirectLocalPathEntry::DefaultInit) - Kind = PathLifetimeKind::Extend; + Kind = PathLifetimeKind::ShouldExtend; else if (Elem.Kind != IndirectLocalPathEntry::LambdaCaptureInit) return PathLifetimeKind::NoExtend; } @@ -8194,6 +8199,18 @@ void Sema::checkInitializerLifetime(const InitializedEntity &Entity, ExtendingEntity->allocateManglingNumber()); // Also visit the temporaries lifetime-extended by this initializer. return true; + + case PathLifetimeKind::ShouldExtend: + // We're supposed to lifetime-extend the temporary along this path (per + // the resolution of DR1815), but we don't support that yet. + // + // FIXME: Properly handle this situation. Perhaps the easiest approach + // would be to clone the initializer expression on each use that would + // lifetime extend its temporaries. + Diag(DiagLoc, diag::warn_unsupported_lifetime_extension) + << RK << DiagRange; + break; + case PathLifetimeKind::NoExtend: // If the path goes through the initialization of a variable or field, // it can't possibly reach a temporary created in this full-expression. diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h index 95dd356d48be..3bfda09d5f80 100644 --- a/clang/lib/Sema/TreeTransform.h +++ b/clang/lib/Sema/TreeTransform.h @@ -14172,13 +14172,6 @@ TreeTransform::TransformCXXTemporaryObjectExpr( if (TransformExprs(E->getArgs(), E->getNumArgs(), true, Args, &ArgumentChanged)) return ExprError(); - - if (E->isListInitialization() && !E->isStdInitListInitialization()) { - ExprResult Res = RebuildInitList(E->getBeginLoc(), Args, E->getEndLoc()); - if (Res.isInvalid()) - return ExprError(); - Args = {Res.get()}; - } } if (!getDerived().AlwaysRebuild() && @@ -14190,9 +14183,12 @@ TreeTransform::TransformCXXTemporaryObjectExpr( return SemaRef.MaybeBindToTemporary(E); } + // FIXME: We should just pass E->isListInitialization(), but we're not + // prepared to handle list-initialization without a child InitListExpr. SourceLocation LParenLoc = T->getTypeLoc().getEndLoc(); return getDerived().RebuildCXXTemporaryObjectExpr( - T, LParenLoc, Args, E->getEndLoc(), E->isListInitialization()); + T, LParenLoc, Args, E->getEndLoc(), + /*ListInitialization=*/LParenLoc.isInvalid()); } template diff --git a/clang/test/AST/ast-dump-default-init-json.cpp b/clang/test/AST/ast-dump-default-init-json.cpp index f4949a9c9eed..1058b4e3ea4d 100644 --- a/clang/test/AST/ast-dump-default-init-json.cpp +++ b/clang/test/AST/ast-dump-default-init-json.cpp @@ -789,10 +789,10 @@ void test() { // CHECK-NEXT: "valueCategory": "lvalue", // CHECK-NEXT: "extendingDecl": { // CHECK-NEXT: "id": "0x{{.*}}", -// CHECK-NEXT: "kind": "VarDecl", -// CHECK-NEXT: "name": "b", +// CHECK-NEXT: "kind": "FieldDecl", +// CHECK-NEXT: "name": "a", // CHECK-NEXT: "type": { -// CHECK-NEXT: "qualType": "B" +// CHECK-NEXT: "qualType": "const A &" // CHECK-NEXT: } // CHECK-NEXT: }, // CHECK-NEXT: "storageDuration": "automatic", diff --git a/clang/test/AST/ast-dump-default-init.cpp b/clang/test/AST/ast-dump-default-init.cpp index 26864fbf1542..15b29f04bf21 100644 --- a/clang/test/AST/ast-dump-default-init.cpp +++ b/clang/test/AST/ast-dump-default-init.cpp @@ -13,7 +13,7 @@ void test() { } // CHECK: -CXXDefaultInitExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue has rewritten init // CHECK-NEXT: `-ExprWithCleanups 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue -// CHECK-NEXT: `-MaterializeTemporaryExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue extended by Var 0x{{[^ ]*}} 'b' 'B' +// CHECK-NEXT: `-MaterializeTemporaryExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue extended by Field 0x{{[^ ]*}} 'a' 'const A &' // CHECK-NEXT: `-ImplicitCastExpr 0x{{[^ ]*}} <{{.*}}> 'const A' // CHECK-NEXT: `-CXXFunctionalCastExpr 0x{{[^ ]*}} <{{.*}}> 'A' functional cast to A // CHECK-NEXT: `-InitListExpr 0x{{[^ ]*}} <{{.*}}> 'A' diff --git a/clang/test/Analysis/lifetime-extended-regions.cpp b/clang/test/Analysis/lifetime-extended-regions.cpp index 4458ad294af7..4e98bd4b0403 100644 --- a/clang/test/Analysis/lifetime-extended-regions.cpp +++ b/clang/test/Analysis/lifetime-extended-regions.cpp @@ -120,11 +120,10 @@ void aggregateWithReferences() { clang_analyzer_dump(viaReference); // expected-warning-re {{&lifetime_extended_object{RefAggregate, viaReference, S{{[0-9]+}}} }} clang_analyzer_dump(viaReference.rx); // expected-warning-re {{&lifetime_extended_object{int, viaReference, S{{[0-9]+}}} }} clang_analyzer_dump(viaReference.ry); // expected-warning-re {{&lifetime_extended_object{Composite, viaReference, S{{[0-9]+}}} }} - - // FIXME: clang currently support extending lifetime of object bound to reference members of aggregates, - // that are created from default member initializer. But CFG and ExprEngine need to be updated to address this change. - // The following expect warning: {{&lifetime_extended_object{Composite, defaultInitExtended, S{{[0-9]+}}} }} - RefAggregate defaultInitExtended{i}; + + // clang does not currently implement extending lifetime of object bound to reference members of aggregates, + // that are created from default member initializer (see `warn_unsupported_lifetime_extension` from `-Wdangling`) + RefAggregate defaultInitExtended{i}; // clang-bug does not extend `Composite` clang_analyzer_dump(defaultInitExtended.ry); // expected-warning {{Unknown }} } diff --git a/clang/test/CXX/drs/cwg16xx.cpp b/clang/test/CXX/drs/cwg16xx.cpp index 82ef871939d2..cf6b45ceabf2 100644 --- a/clang/test/CXX/drs/cwg16xx.cpp +++ b/clang/test/CXX/drs/cwg16xx.cpp @@ -483,6 +483,8 @@ namespace cwg1696 { // cwg1696: 7 const A &a = A(); // #cwg1696-D1-a }; D1 d1 = {}; // #cwg1696-d1 + // since-cxx14-warning@-1 {{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported; lifetime of temporary will end at the end of the full-expression}} + // since-cxx14-note@#cwg1696-D1-a {{initializing field 'a' with default member initializer}} struct D2 { const A &a = A(); // #cwg1696-D2-a diff --git a/clang/test/CXX/drs/cwg18xx.cpp b/clang/test/CXX/drs/cwg18xx.cpp index 054ce5a4f4b7..323e56f9c527 100644 --- a/clang/test/CXX/drs/cwg18xx.cpp +++ b/clang/test/CXX/drs/cwg18xx.cpp @@ -206,28 +206,19 @@ namespace cwg1814 { // cwg1814: yes #endif } -namespace cwg1815 { // cwg1815: 19 +namespace cwg1815 { // cwg1815: no #if __cplusplus >= 201402L - struct A { int &&r = 0; }; + // FIXME: needs codegen test + struct A { int &&r = 0; }; // #cwg1815-A A a = {}; + // since-cxx14-warning@-1 {{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported; lifetime of temporary will end at the end of the full-expression}} FIXME + // since-cxx14-note@#cwg1815-A {{initializing field 'r' with default member initializer}} struct B { int &&r = 0; }; // #cwg1815-B // since-cxx14-error@-1 {{reference member 'r' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} // since-cxx14-note@#cwg1815-B {{initializing field 'r' with default member initializer}} // since-cxx14-note@#cwg1815-b {{in implicit default constructor for 'cwg1815::B' first required here}} B b; // #cwg1815-b - -#if __cplusplus >= 201703L - struct C { const int &r = 0; }; - constexpr C c = {}; // OK, since cwg1815 - static_assert(c.r == 0); - - constexpr int f() { - A a = {}; // OK, since cwg1815 - return a.r; - } - static_assert(f() == 0); -#endif #endif } diff --git a/clang/test/CXX/special/class.temporary/p6.cpp b/clang/test/CXX/special/class.temporary/p6.cpp index a6d2adfd1fd2..5554363cc69a 100644 --- a/clang/test/CXX/special/class.temporary/p6.cpp +++ b/clang/test/CXX/special/class.temporary/p6.cpp @@ -269,40 +269,6 @@ void init_capture_init_list() { // CHECK: } } -void check_dr1815() { // dr1815: yes -#if __cplusplus >= 201402L - - struct A { - int &&r = 0; - ~A() {} - }; - - struct B { - A &&a = A{}; - ~B() {} - }; - B a = {}; - - // CHECK: call {{.*}}block_scope_begin_function - extern void block_scope_begin_function(); - extern void block_scope_end_function(); - block_scope_begin_function(); - { - // CHECK: call void @_ZZ12check_dr1815vEN1BD1Ev - // CHECK: call void @_ZZ12check_dr1815vEN1AD1Ev - B b = {}; - } - // CHECK: call {{.*}}block_scope_end_function - block_scope_end_function(); - - // CHECK: call {{.*}}some_other_function - extern void some_other_function(); - some_other_function(); - // CHECK: call void @_ZZ12check_dr1815vEN1BD1Ev - // CHECK: call void @_ZZ12check_dr1815vEN1AD1Ev -#endif -} - namespace P2718R0 { namespace basic { template using T2 = std::list; diff --git a/clang/test/SemaCXX/constexpr-default-arg.cpp b/clang/test/SemaCXX/constexpr-default-arg.cpp index 901123bfb359..ec9b2927880b 100644 --- a/clang/test/SemaCXX/constexpr-default-arg.cpp +++ b/clang/test/SemaCXX/constexpr-default-arg.cpp @@ -32,8 +32,8 @@ void test_default_arg2() { } // Check that multiple CXXDefaultInitExprs don't cause an assertion failure. -struct A { int &&r = 0; }; +struct A { int &&r = 0; }; // expected-note 2{{default member initializer}} struct B { A x, y; }; -B b = {}; // expected-no-diagnostics +B b = {}; // expected-warning 2{{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported}} } diff --git a/clang/test/SemaCXX/cxx11-default-member-initializers.cpp b/clang/test/SemaCXX/cxx11-default-member-initializers.cpp index 1ea8b98cd863..dd8e9c6b7fc1 100644 --- a/clang/test/SemaCXX/cxx11-default-member-initializers.cpp +++ b/clang/test/SemaCXX/cxx11-default-member-initializers.cpp @@ -27,80 +27,6 @@ class MemInit { C m = s; }; -namespace std { -typedef decltype(sizeof(int)) size_t; - -// libc++'s implementation -template class initializer_list { - const _E *__begin_; - size_t __size_; - - initializer_list(const _E *__b, size_t __s) : __begin_(__b), __size_(__s) {} - -public: - typedef _E value_type; - typedef const _E &reference; - typedef const _E &const_reference; - typedef size_t size_type; - - typedef const _E *iterator; - typedef const _E *const_iterator; - - initializer_list() : __begin_(nullptr), __size_(0) {} - - size_t size() const { return __size_; } - const _E *begin() const { return __begin_; } - const _E *end() const { return __begin_ + __size_; } -}; -} // namespace std - -#if __cplusplus >= 201703L -namespace test_rebuild { -template class C { -public: - C(std::initializer_list); -}; - -template using Ptr = __remove_pointer(T) *; -template C(T) -> C, sizeof(T)>; - -class A { -public: - template T1 *some_func(T2 &&); -}; - -struct B : A { - // Test CXXDefaultInitExpr rebuild issue in - // https://github.com/llvm/llvm-project/pull/87933 - int *ar = some_func(C{some_func(0)}); - B() {} -}; - -int TestBody_got; -template class Vector { -public: - Vector(std::initializer_list); -}; -template Vector(Ts...) -> Vector; -class ProgramBuilder { -public: - template int *create(ARGS); -}; - -struct TypeTest : ProgramBuilder { - int *str_f16 = create(Vector{0}); - TypeTest() {} -}; -class TypeTest_Element_Test : TypeTest { - void TestBody(); -}; -void TypeTest_Element_Test::TestBody() { - int *expect = str_f16; - &TestBody_got != expect; // expected-warning {{inequality comparison result unused}} -} -} // namespace test_rebuild -#endif // __cplusplus >= 201703L - #if __cplusplus >= 202002L // This test ensures cleanup expressions are correctly produced // in the presence of default member initializers. diff --git a/clang/test/SemaCXX/eval-crashes.cpp b/clang/test/SemaCXX/eval-crashes.cpp index a06f60f71e9c..017df977b26b 100644 --- a/clang/test/SemaCXX/eval-crashes.cpp +++ b/clang/test/SemaCXX/eval-crashes.cpp @@ -25,9 +25,11 @@ namespace pr33140_0b { } namespace pr33140_2 { - struct A { int &&r = 0; }; + // FIXME: The declaration of 'b' below should lifetime-extend two int + // temporaries. + struct A { int &&r = 0; }; // expected-note 2{{initializing field 'r' with default member initializer}} struct B { A x, y; }; - B b = {}; + B b = {}; // expected-warning 2{{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported}} } namespace pr33140_3 { diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html index b046468c8531..43857447d83b 100755 --- a/clang/www/cxx_dr_status.html +++ b/clang/www/cxx_dr_status.html @@ -10698,7 +10698,7 @@ and POD class 1815 CD4 Lifetime extension in aggregate initialization - Clang 19 + No 1816 -- GitLab From 64c8b66cc9972123c5f4aefe692c275898221aeb Mon Sep 17 00:00:00 2001 From: Kerry McLaughlin Date: Thu, 6 Jun 2024 11:02:19 +0100 Subject: [PATCH 073/462] [AArch64][SME] Add calling convention for __arm_get_current_vg (#93963) Adds a calling convention for calls to the `__arm_get_current_vg` support routine, which preserves X1-X15, X19-X29, SP, Z0-Z31 & P0-P15. See https://github.com/ARM-software/abi-aa/pull/263 --- llvm/include/llvm/AsmParser/LLToken.h | 1 + llvm/include/llvm/IR/CallingConv.h | 3 ++ llvm/lib/AsmParser/LLLexer.cpp | 1 + llvm/lib/AsmParser/LLParser.cpp | 4 +++ llvm/lib/IR/AsmWriter.cpp | 3 ++ .../AArch64/AArch64CallingConvention.td | 8 +++++ .../Target/AArch64/AArch64RegisterInfo.cpp | 33 ++++++++++++++++--- ...sme-support-routines-calling-convention.ll | 20 +++++++++++ 8 files changed, 68 insertions(+), 5 deletions(-) diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h index 69821c22dcd6..db6780b70ca5 100644 --- a/llvm/include/llvm/AsmParser/LLToken.h +++ b/llvm/include/llvm/AsmParser/LLToken.h @@ -147,6 +147,7 @@ enum Kind { kw_aarch64_vector_pcs, kw_aarch64_sve_vector_pcs, kw_aarch64_sme_preservemost_from_x0, + kw_aarch64_sme_preservemost_from_x1, kw_aarch64_sme_preservemost_from_x2, kw_msp430_intrcc, kw_avr_intrcc, diff --git a/llvm/include/llvm/IR/CallingConv.h b/llvm/include/llvm/IR/CallingConv.h index a05d1a4d5878..55e32028e3ed 100644 --- a/llvm/include/llvm/IR/CallingConv.h +++ b/llvm/include/llvm/IR/CallingConv.h @@ -267,6 +267,9 @@ namespace CallingConv { /// Calling convention used for RISC-V V-extension. RISCV_VectorCall = 110, + /// Preserve X1-X15, X19-X29, SP, Z0-Z31, P0-P15. + AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 = 111, + /// The highest possible ID. Must be some 2^k - 1. MaxID = 1023 }; diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp index d3ab306904da..7d7fe19568e8 100644 --- a/llvm/lib/AsmParser/LLLexer.cpp +++ b/llvm/lib/AsmParser/LLLexer.cpp @@ -604,6 +604,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(aarch64_vector_pcs); KEYWORD(aarch64_sve_vector_pcs); KEYWORD(aarch64_sme_preservemost_from_x0); + KEYWORD(aarch64_sme_preservemost_from_x1); KEYWORD(aarch64_sme_preservemost_from_x2); KEYWORD(msp430_intrcc); KEYWORD(avr_intrcc); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 07c8aa23fc5e..f0fde9ae4df5 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -2153,6 +2153,7 @@ void LLParser::parseOptionalDLLStorageClass(unsigned &Res) { /// ::= 'aarch64_vector_pcs' /// ::= 'aarch64_sve_vector_pcs' /// ::= 'aarch64_sme_preservemost_from_x0' +/// ::= 'aarch64_sme_preservemost_from_x1' /// ::= 'aarch64_sme_preservemost_from_x2' /// ::= 'msp430_intrcc' /// ::= 'avr_intrcc' @@ -2212,6 +2213,9 @@ bool LLParser::parseOptionalCallingConv(unsigned &CC) { case lltok::kw_aarch64_sme_preservemost_from_x0: CC = CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0; break; + case lltok::kw_aarch64_sme_preservemost_from_x1: + CC = CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1; + break; case lltok::kw_aarch64_sme_preservemost_from_x2: CC = CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2; break; diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 7a5f18fe2cbd..0bf8be9ac55f 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -326,6 +326,9 @@ static void PrintCallingConv(unsigned cc, raw_ostream &Out) { case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0: Out << "aarch64_sme_preservemost_from_x0"; break; + case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1: + Out << "aarch64_sme_preservemost_from_x1"; + break; case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2: Out << "aarch64_sme_preservemost_from_x2"; break; diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.td b/llvm/lib/Target/AArch64/AArch64CallingConvention.td index 32646c6ee689..941990c53c4a 100644 --- a/llvm/lib/Target/AArch64/AArch64CallingConvention.td +++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.td @@ -589,6 +589,14 @@ def CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 (sequence "X%u",19, 28), LR, FP)>; +// SME ABI support routines such as __arm_get_current_vg preserve most registers. +def CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 + : CalleeSavedRegs<(add (sequence "Z%u", 0, 31), + (sequence "P%u", 0, 15), + (sequence "X%u", 1, 15), + (sequence "X%u",19, 28), + LR, FP)>; + // SME ABI support routines __arm_sme_state preserves most registers. def CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 : CalleeSavedRegs<(add (sequence "Z%u", 0, 31), diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp index e97d7e3b6ed8..cc50b59dd8d7 100644 --- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp @@ -107,13 +107,22 @@ AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const { if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) report_fatal_error( - "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is " - "only supported to improve calls to SME ACLE save/restore/disable-za " + "Calling convention " + "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is only " + "supported to improve calls to SME ACLE save/restore/disable-za " "functions, and is not intended to be used beyond that scope."); + if (MF->getFunction().getCallingConv() == + CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1) + report_fatal_error( + "Calling convention " + "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 is " + "only supported to improve calls to SME ACLE __arm_get_current_vg " + "function, and is not intended to be used beyond that scope."); if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) report_fatal_error( - "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is " + "Calling convention " + "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is " "only supported to improve calls to SME ACLE __arm_sme_state " "and is not intended to be used beyond that scope."); if (MF->getSubtarget().getTargetLowering() @@ -153,13 +162,22 @@ AArch64RegisterInfo::getDarwinCalleeSavedRegs(const MachineFunction *MF) const { if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) report_fatal_error( - "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is " + "Calling convention " + "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is " "only supported to improve calls to SME ACLE save/restore/disable-za " "functions, and is not intended to be used beyond that scope."); + if (MF->getFunction().getCallingConv() == + CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1) + report_fatal_error( + "Calling convention " + "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 is " + "only supported to improve calls to SME ACLE __arm_get_current_vg " + "function, and is not intended to be used beyond that scope."); if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) report_fatal_error( - "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is " + "Calling convention " + "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is " "only supported to improve calls to SME ACLE __arm_sme_state " "and is not intended to be used beyond that scope."); if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS) @@ -236,6 +254,8 @@ AArch64RegisterInfo::getDarwinCallPreservedMask(const MachineFunction &MF, "Calling convention SVE_VectorCall is unsupported on Darwin."); if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask; + if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1) + return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask; if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask; if (CC == CallingConv::CFGuard_Check) @@ -282,6 +302,8 @@ AArch64RegisterInfo::getCallPreservedMask(const MachineFunction &MF, : CSR_AArch64_SVE_AAPCS_RegMask; if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask; + if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1) + return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask; if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask; if (CC == CallingConv::CFGuard_Check) @@ -643,6 +665,7 @@ bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF, case CallingConv::AArch64_VectorCall: case CallingConv::AArch64_SVE_VectorCall: case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0: + case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1: case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2: if (STI.isTargetWindows()) return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg); diff --git a/llvm/test/CodeGen/AArch64/sme-support-routines-calling-convention.ll b/llvm/test/CodeGen/AArch64/sme-support-routines-calling-convention.ll index 7535638137ca..63c65334afe1 100644 --- a/llvm/test/CodeGen/AArch64/sme-support-routines-calling-convention.ll +++ b/llvm/test/CodeGen/AArch64/sme-support-routines-calling-convention.ll @@ -25,6 +25,25 @@ define void @test_sme_calling_convention_x0() nounwind { ret void } +define i64 @test_sme_calling_convention_x1() nounwind { +; CHECK-LABEL: test_sme_calling_convention_x1: +; CHECK: // %bb.0: +; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: bl __arm_get_current_vg +; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret +; DARWIN-LABEL: test_sme_calling_convention_x1: +; DARWIN: stp x29, x30, [sp, #-16]! +; DARWIN: bl ___arm_get_current_vg +; DARWIN: ldp x29, x30, [sp], #16 +; DARWIN: ret +; +; CHECK-CSRMASK-LABEL: name: test_sme_calling_convention_x1 +; CHECK-CSRMASK: BL @__arm_get_current_vg, csr_aarch64_sme_abi_support_routines_preservemost_from_x1 + %vg = call aarch64_sme_preservemost_from_x1 i64 @__arm_get_current_vg() + ret i64 %vg +} + define i64 @test_sme_calling_convention_x2() nounwind { ; CHECK-LABEL: test_sme_calling_convention_x2: ; CHECK: // %bb.0: @@ -46,4 +65,5 @@ define i64 @test_sme_calling_convention_x2() nounwind { } declare void @__arm_tpidr2_save() +declare i64 @__arm_get_current_vg() declare {i64, i64} @__arm_sme_state() -- GitLab From 3cdb1df39d1be54c219ef083a4d1bfe3c9ed4e19 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Thu, 6 Jun 2024 10:08:45 +0000 Subject: [PATCH 074/462] [gn build] Port 8516f54e6a98 --- llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn index 0d134c7bdffb..bcf2ea751056 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn @@ -33,6 +33,7 @@ static_library("IPO") { "DeadArgumentElimination.cpp", "ElimAvailExtern.cpp", "EmbedBitcodePass.cpp", + "ExpandVariadics.cpp", "ExtractGV.cpp", "ForceFunctionAttrs.cpp", "FunctionAttrs.cpp", -- GitLab From 9b692e562dbe6a5c9cac3b438f755d83ab8e947b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= Date: Thu, 6 Jun 2024 12:22:52 +0200 Subject: [PATCH 075/462] [GlobalIsel] Combine G_VSCALE (#94096) We need them for scalable address calculation and legal scalable addressing modes. --- .../llvm/CodeGen/GlobalISel/CombinerHelper.h | 8 ++ .../CodeGen/GlobalISel/GenericMachineInstrs.h | 41 ++++++- .../include/llvm/Target/GlobalISel/Combine.td | 37 +++++- .../GlobalISel/CombinerHelperVectorOps.cpp | 86 ++++++++++++- .../AArch64/GlobalISel/combine-vscale.mir | 113 ++++++++++++++++++ 5 files changed, 282 insertions(+), 3 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/combine-vscale.mir diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 12e5b31e5817..43659564d5ac 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -871,6 +871,14 @@ public: bool matchFreezeOfSingleMaybePoisonOperand(MachineInstr &MI, BuildFnTy &MatchInfo); + bool matchAddOfVScale(const MachineOperand &MO, BuildFnTy &MatchInfo); + + bool matchMulOfVScale(const MachineOperand &MO, BuildFnTy &MatchInfo); + + bool matchSubOfVScale(const MachineOperand &MO, BuildFnTy &MatchInfo); + + bool matchShlOfVScale(const MachineOperand &MO, BuildFnTy &MatchInfo); + private: /// Checks for legality of an indexed variant of \p LdSt. bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h index 2b3efc3b609f..227372563771 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h @@ -14,10 +14,12 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H #define LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H -#include "llvm/IR/Instructions.h" +#include "llvm/ADT/APInt.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/TargetOpcodes.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Instructions.h" #include "llvm/Support/Casting.h" namespace llvm { @@ -856,6 +858,43 @@ public: }; }; +/// Represents a vscale. +class GVScale : public GenericMachineInstr { +public: + APInt getSrc() const { return getOperand(1).getCImm()->getValue(); } + + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_VSCALE; + }; +}; + +/// Represents an integer subtraction. +class GSub : public GIntBinOp { +public: + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_SUB; + }; +}; + +/// Represents an integer multiplication. +class GMul : public GIntBinOp { +public: + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_MUL; + }; +}; + +/// Represents a shift left. +class GShl : public GenericMachineInstr { +public: + Register getSrcReg() const { return getOperand(1).getReg(); } + Register getShiftReg() const { return getOperand(2).getReg(); } + + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_SHL; + }; +}; + } // namespace llvm #endif // LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index 1ea2652871ab..bd43b9589903 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -1605,6 +1605,37 @@ def insert_vector_elt_oob : GICombineRule< [{ return Helper.matchInsertVectorElementOOB(*${root}, ${matchinfo}); }]), (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>; +def add_of_vscale : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_VSCALE $left, $imm1), + (G_VSCALE $right, $imm2), + (G_ADD $root, $left, $right, (MIFlags NoSWrap)), + [{ return Helper.matchAddOfVScale(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def mul_of_vscale : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_VSCALE $left, $scale), + (G_CONSTANT $x, $imm1), + (G_MUL $root, $left, $x, (MIFlags NoSWrap)), + [{ return Helper.matchMulOfVScale(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def shl_of_vscale : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_VSCALE $left, $imm), + (G_CONSTANT $x, $imm1), + (G_SHL $root, $left, $x, (MIFlags NoSWrap)), + [{ return Helper.matchShlOfVScale(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + +def sub_of_vscale : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_VSCALE $right, $imm), + (G_SUB $root, $x, $right, (MIFlags NoSWrap)), + [{ return Helper.matchSubOfVScale(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + // match_extract_of_element and insert_vector_elt_oob must be the first! def vector_ops_combines: GICombineGroup<[ match_extract_of_element_undef_vector, @@ -1637,7 +1668,11 @@ extract_vector_element_build_vector_trunc6, extract_vector_element_build_vector_trunc7, extract_vector_element_build_vector_trunc8, extract_vector_element_shuffle_vector, -insert_vector_element_extract_vector_element +insert_vector_element_extract_vector_element, +add_of_vscale, +mul_of_vscale, +shl_of_vscale, +sub_of_vscale, ]>; diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp index b4765fb280f9..66b1c5f8ca82 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp @@ -6,7 +6,8 @@ // //===----------------------------------------------------------------------===// // -// This file implements CombinerHelper for G_EXTRACT_VECTOR_ELT. +// This file implements CombinerHelper for G_EXTRACT_VECTOR_ELT, +// G_INSERT_VECTOR_ELT, and G_VSCALE // //===----------------------------------------------------------------------===// #include "llvm/CodeGen/GlobalISel/CombinerHelper.h" @@ -400,3 +401,86 @@ bool CombinerHelper::matchInsertVectorElementOOB(MachineInstr &MI, return false; } + +bool CombinerHelper::matchAddOfVScale(const MachineOperand &MO, + BuildFnTy &MatchInfo) { + GAdd *Add = cast(MRI.getVRegDef(MO.getReg())); + GVScale *LHSVScale = cast(MRI.getVRegDef(Add->getLHSReg())); + GVScale *RHSVScale = cast(MRI.getVRegDef(Add->getRHSReg())); + + Register Dst = Add->getReg(0); + + if (!MRI.hasOneNonDBGUse(LHSVScale->getReg(0)) || + !MRI.hasOneNonDBGUse(RHSVScale->getReg(0))) + return false; + + MatchInfo = [=](MachineIRBuilder &B) { + B.buildVScale(Dst, LHSVScale->getSrc() + RHSVScale->getSrc()); + }; + + return true; +} + +bool CombinerHelper::matchMulOfVScale(const MachineOperand &MO, + BuildFnTy &MatchInfo) { + GMul *Mul = cast(MRI.getVRegDef(MO.getReg())); + GVScale *LHSVScale = cast(MRI.getVRegDef(Mul->getLHSReg())); + + std::optional MaybeRHS = getIConstantVRegVal(Mul->getRHSReg(), MRI); + if (!MaybeRHS) + return false; + + Register Dst = MO.getReg(); + + if (!MRI.hasOneNonDBGUse(LHSVScale->getReg(0))) + return false; + + MatchInfo = [=](MachineIRBuilder &B) { + B.buildVScale(Dst, LHSVScale->getSrc() * *MaybeRHS); + }; + + return true; +} + +bool CombinerHelper::matchSubOfVScale(const MachineOperand &MO, + BuildFnTy &MatchInfo) { + GSub *Sub = cast(MRI.getVRegDef(MO.getReg())); + GVScale *RHSVScale = cast(MRI.getVRegDef(Sub->getRHSReg())); + + Register Dst = MO.getReg(); + LLT DstTy = MRI.getType(Dst); + + if (!MRI.hasOneNonDBGUse(RHSVScale->getReg(0)) || + !isLegalOrBeforeLegalizer({TargetOpcode::G_ADD, DstTy})) + return false; + + MatchInfo = [=](MachineIRBuilder &B) { + auto VScale = B.buildVScale(DstTy, -RHSVScale->getSrc()); + B.buildAdd(Dst, Sub->getLHSReg(), VScale, Sub->getFlags()); + }; + + return true; +} + +bool CombinerHelper::matchShlOfVScale(const MachineOperand &MO, + BuildFnTy &MatchInfo) { + GShl *Shl = cast(MRI.getVRegDef(MO.getReg())); + GVScale *LHSVScale = cast(MRI.getVRegDef(Shl->getSrcReg())); + + std::optional MaybeRHS = getIConstantVRegVal(Shl->getShiftReg(), MRI); + if (!MaybeRHS) + return false; + + Register Dst = MO.getReg(); + LLT DstTy = MRI.getType(Dst); + + if (!MRI.hasOneNonDBGUse(LHSVScale->getReg(0)) || + !isLegalOrBeforeLegalizer({TargetOpcode::G_VSCALE, DstTy})) + return false; + + MatchInfo = [=](MachineIRBuilder &B) { + B.buildVScale(Dst, LHSVScale->getSrc().shl(*MaybeRHS)); + }; + + return true; +} diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-vscale.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-vscale.mir new file mode 100644 index 000000000000..9b7a44954afd --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-vscale.mir @@ -0,0 +1,113 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -o - -mtriple=aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s | FileCheck %s + +... +--- +name: sum_of_vscale +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: sum_of_vscale + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %sum:_(s64) = G_VSCALE i64 20 + ; CHECK-NEXT: $x0 = COPY %sum(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %rhs:_(s64) = G_VSCALE i64 11 + %lhs:_(s64) = G_VSCALE i64 9 + %sum:_(s64) = nsw G_ADD %lhs(s64), %rhs(s64) + $x0 = COPY %sum(s64) + RET_ReallyLR implicit $x0 +... +--- +name: sum_of_vscale_multi_use +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: sum_of_vscale_multi_use + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %rhs:_(s64) = G_VSCALE i64 11 + ; CHECK-NEXT: %lhs:_(s64) = G_VSCALE i64 9 + ; CHECK-NEXT: %sum:_(s64) = nsw G_ADD %lhs, %rhs + ; CHECK-NEXT: $x0 = COPY %sum(s64) + ; CHECK-NEXT: $x1 = COPY %rhs(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %rhs:_(s64) = G_VSCALE i64 11 + %lhs:_(s64) = G_VSCALE i64 9 + %sum:_(s64) = nsw G_ADD %lhs(s64), %rhs(s64) + $x0 = COPY %sum(s64) + $x1 = COPY %rhs(s64) + RET_ReallyLR implicit $x0 +... +--- +name: mul_of_vscale +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: mul_of_vscale + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %mul:_(s64) = G_VSCALE i64 99 + ; CHECK-NEXT: $x0 = COPY %mul(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %rhs:_(s64) = G_CONSTANT i64 11 + %lhs:_(s64) = G_VSCALE i64 9 + %mul:_(s64) = nsw G_MUL %lhs(s64), %rhs(s64) + $x0 = COPY %mul(s64) + RET_ReallyLR implicit $x0 +... +--- +name: sub_of_vscale +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: sub_of_vscale + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %x:_(s64) = COPY $x0 + ; CHECK-NEXT: [[VSCALE:%[0-9]+]]:_(s64) = G_VSCALE i64 -9 + ; CHECK-NEXT: %sub:_(s64) = nsw G_ADD %x, [[VSCALE]] + ; CHECK-NEXT: $x0 = COPY %sub(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %x:_(s64) = COPY $x0 + %rhs:_(s64) = G_VSCALE i64 9 + %sub:_(s64) = nsw G_SUB %x(s64), %rhs(s64) + $x0 = COPY %sub(s64) + RET_ReallyLR implicit $x0 +... +--- +name: shl_of_vscale +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: shl_of_vscale + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %shl:_(s64) = G_VSCALE i64 44 + ; CHECK-NEXT: $x0 = COPY %shl(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %rhs:_(s64) = G_CONSTANT i64 2 + %lhs:_(s64) = G_VSCALE i64 11 + %shl:_(s64) = nsw G_SHL %lhs(s64), %rhs(s64) + $x0 = COPY %shl(s64) + RET_ReallyLR implicit $x0 +... +--- +name: shl_of_vscale_wrong_flag +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: shl_of_vscale_wrong_flag + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %rhs:_(s64) = G_CONSTANT i64 2 + ; CHECK-NEXT: %lhs:_(s64) = G_VSCALE i64 11 + ; CHECK-NEXT: %shl:_(s64) = nuw G_SHL %lhs, %rhs(s64) + ; CHECK-NEXT: $x0 = COPY %shl(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %rhs:_(s64) = G_CONSTANT i64 2 + %lhs:_(s64) = G_VSCALE i64 11 + %shl:_(s64) = nuw G_SHL %lhs(s64), %rhs(s64) + $x0 = COPY %shl(s64) + RET_ReallyLR implicit $x0 -- GitLab From 4559d83b24a839e9309378e070c64623f04f1e49 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Thu, 6 Jun 2024 12:31:19 +0200 Subject: [PATCH 076/462] [bazel] Port for 8516f54e6a984a79d1a988ec66a414b5cfbd7ec7 Remove some #includes in ExpandVariadics.cpp as it will cause layering violations. --- llvm/lib/Transforms/IPO/ExpandVariadics.cpp | 2 -- utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/IPO/ExpandVariadics.cpp b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp index 9a4f39948a30..d29b8372a73c 100644 --- a/llvm/lib/Transforms/IPO/ExpandVariadics.cpp +++ b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp @@ -53,7 +53,6 @@ #include "llvm/Transforms/IPO/ExpandVariadics.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/Passes.h" #include "llvm/IR/Constants.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/IntrinsicInst.h" @@ -61,7 +60,6 @@ #include "llvm/IR/PassManager.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" -#include "llvm/Passes/OptimizationLevel.h" #include "llvm/Support/CommandLine.h" #include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Utils/ModuleUtils.h" diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index 2ffbc783ec97..cfc2ac3ae7fb 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -1755,6 +1755,7 @@ cc_library( ":TransformUtils", ":Vectorize", ":config", + ":ir_headers", ], ) -- GitLab From 574ab7e7b759a400ecf54c1141b1433bb2415e93 Mon Sep 17 00:00:00 2001 From: Jon Chesterfield Date: Thu, 6 Jun 2024 11:33:11 +0100 Subject: [PATCH 077/462] [libc] Disable varargs tests on gpu, nvptx can't build them yet --- libc/config/gpu/entrypoints.txt | 4 ---- libc/test/src/__support/CMakeLists.txt | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt index 7b73b8a22421..b678350e9fcb 100644 --- a/libc/config/gpu/entrypoints.txt +++ b/libc/config/gpu/entrypoints.txt @@ -181,10 +181,6 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdio.fflush libc.src.stdio.ftell libc.src.stdio.clearerr - libc.src.stdio.sprintf - libc.src.stdio.snprintf - libc.src.stdio.vsprintf - libc.src.stdio.vsnprintf libc.src.stdio.puts libc.src.stdio.fopen libc.src.stdio.fclose diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt index 5afc4173f61a..663aa2bb82ca 100644 --- a/libc/test/src/__support/CMakeLists.txt +++ b/libc/test/src/__support/CMakeLists.txt @@ -86,8 +86,8 @@ add_libc_test( libc.src.__support.uint128 ) -# NVPTX does not support varargs currently. -if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX) +# The GPU does not support varargs currently. +if(NOT LIBC_TARGET_OS_IS_GPU) add_libc_test( arg_list_test SUITE -- GitLab From b3b9f8dd4cad55cf4106570af518c5075c2cdcee Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Thu, 6 Jun 2024 18:41:07 +0800 Subject: [PATCH 078/462] [Transforms] Fix -Wunused-variable in ExpandVariadics.cpp (NFC) /llvm-project/llvm/lib/Transforms/IPO/ExpandVariadics.cpp:426:14: error: unused variable 'OriginalFunctionIsDeclaration' [-Werror,-Wunused-variable] const bool OriginalFunctionIsDeclaration = OriginalFunction->isDeclaration(); ^ /llvm-project/llvm/lib/Transforms/IPO/ExpandVariadics.cpp:445:13: error: unused variable 'VariadicWrapperDefine' [-Werror,-Wunused-variable] Function *VariadicWrapperDefine = ^ 2 errors generated. --- llvm/lib/Transforms/IPO/ExpandVariadics.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/IPO/ExpandVariadics.cpp b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp index d29b8372a73c..d340bc041ccd 100644 --- a/llvm/lib/Transforms/IPO/ExpandVariadics.cpp +++ b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp @@ -423,7 +423,8 @@ bool ExpandVariadics::runOnFunction(Module &M, IRBuilder<> &Builder, if (!expansionApplicableToFunction(M, OriginalFunction)) return Changed; - const bool OriginalFunctionIsDeclaration = OriginalFunction->isDeclaration(); + [[maybe_unused]] const bool OriginalFunctionIsDeclaration = + OriginalFunction->isDeclaration(); assert(rewriteABI() || !OriginalFunctionIsDeclaration); // Declare a new function and redirect every use to that new function @@ -442,7 +443,7 @@ bool ExpandVariadics::runOnFunction(Module &M, IRBuilder<> &Builder, assert(VariadicWrapper->isDeclaration()); // Create a single block forwarding wrapper that turns a ... into a va_list - Function *VariadicWrapperDefine = + [[maybe_unused]] Function *VariadicWrapperDefine = defineVariadicWrapper(M, Builder, VariadicWrapper, FixedArityReplacement); assert(VariadicWrapperDefine == VariadicWrapper); assert(!VariadicWrapper->isDeclaration()); -- GitLab From 5fe7307146004c8e841b2340f075519be4a215ed Mon Sep 17 00:00:00 2001 From: David Green Date: Thu, 6 Jun 2024 11:43:39 +0100 Subject: [PATCH 079/462] [ARM] Don't block tail-predication from unrelated VPT blocks. (#94239) VPT blocks that do not produce an interesting 'output' (like a stored value or reduction result), do not need to be predicated on vctp for the whole loop to be tail-predicated. Just producing results for the valid tail predication lanes should be enough. --- llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp | 22 +++++++++++----- .../CodeGen/Thumb2/mve-tailpred-vptblock.ll | 25 ++++--------------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp index a46c383115e2..919828753f45 100644 --- a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp +++ b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp @@ -115,6 +115,12 @@ static bool shouldInspect(MachineInstr &MI) { return isDomainMVE(&MI) || isVectorPredicate(&MI) || hasVPRUse(MI); } +static bool isHorizontalReduction(const MachineInstr &MI) { + const MCInstrDesc &MCID = MI.getDesc(); + uint64_t Flags = MCID.TSFlags; + return (Flags & ARMII::HorizontalReduction) != 0; +} + namespace { using InstSet = SmallPtrSetImpl; @@ -275,6 +281,16 @@ namespace { if (VPT->getOpcode() == ARM::MVE_VPST) return false; + // If the VPT block does not define something that is an "output", then + // the tail-predicated version will just perform a subset of the original + // vpt block, where the last lanes should not be used. + if (isVPTOpcode(VPT->getOpcode()) && + all_of(Block.getInsts(), [](const MachineInstr *MI) { + return !MI->mayStore() && !MI->mayLoad() && + !isHorizontalReduction(*MI) && !isVCTP(MI); + })) + return true; + auto IsOperandPredicated = [&](MachineInstr *MI, unsigned Idx) { MachineInstr *Op = RDA.getMIOperand(MI, MI->getOperand(Idx)); return Op && PredicatedInsts.count(Op) && isPredicatedOnVCTP(Op); @@ -813,12 +829,6 @@ static bool producesDoubleWidthResult(const MachineInstr &MI) { return (Flags & ARMII::DoubleWidthResult) != 0; } -static bool isHorizontalReduction(const MachineInstr &MI) { - const MCInstrDesc &MCID = MI.getDesc(); - uint64_t Flags = MCID.TSFlags; - return (Flags & ARMII::HorizontalReduction) != 0; -} - // Can this instruction generate a non-zero result when given only zeroed // operands? This allows us to know that, given operands with false bytes // zeroed by masked loads, that the result will also contain zeros in those diff --git a/llvm/test/CodeGen/Thumb2/mve-tailpred-vptblock.ll b/llvm/test/CodeGen/Thumb2/mve-tailpred-vptblock.ll index f9b3757bb6d2..6392452e06cd 100644 --- a/llvm/test/CodeGen/Thumb2/mve-tailpred-vptblock.ll +++ b/llvm/test/CodeGen/Thumb2/mve-tailpred-vptblock.ll @@ -20,50 +20,35 @@ define void @convert_vptblock(ptr %pchTarget, i16 signext %iTargetStride, ptr %p ; CHECK-NEXT: mov.w r8, #0 ; CHECK-NEXT: ldrd r4, r5, [sp, #88] ; CHECK-NEXT: mov r7, r0 -; CHECK-NEXT: cmp.w r10, #8 -; CHECK-NEXT: mov.w r0, #1 -; CHECK-NEXT: mov r3, r10 ; CHECK-NEXT: mov.w r11, #0 -; CHECK-NEXT: it ge -; CHECK-NEXT: movge r3, #8 ; CHECK-NEXT: vidup.u16 q0, r8, #4 -; CHECK-NEXT: sub.w r3, r10, r3 ; CHECK-NEXT: vmov.i32 q1, #0x0 -; CHECK-NEXT: adds r3, #7 ; CHECK-NEXT: vmov.i16 q2, #0x100 ; CHECK-NEXT: vmov.i16 q3, #0xff -; CHECK-NEXT: add.w r9, r0, r3, lsr #3 ; CHECK-NEXT: .LBB0_2: @ %for.body ; CHECK-NEXT: @ =>This Loop Header: Depth=1 ; CHECK-NEXT: @ Child Loop BB0_3 Depth 2 -; CHECK-NEXT: mov r3, r10 ; CHECK-NEXT: vmov q4, q0 ; CHECK-NEXT: mov r6, r8 ; CHECK-NEXT: mov r0, r7 -; CHECK-NEXT: dls lr, r9 +; CHECK-NEXT: dlstp.16 lr, r10 ; CHECK-NEXT: .LBB0_3: @ %do.body ; CHECK-NEXT: @ Parent Loop BB0_2 Depth=1 ; CHECK-NEXT: @ => This Inner Loop Header: Depth=2 -; CHECK-NEXT: vctp.16 r3 -; CHECK-NEXT: vpst -; CHECK-NEXT: vldrbt.u16 q5, [r2, q4] +; CHECK-NEXT: vldrb.u16 q5, [r2, q4] ; CHECK-NEXT: vmul.i16 q4, q5, r5 ; CHECK-NEXT: vshr.u16 q4, q4, #8 ; CHECK-NEXT: vsub.i16 q5, q2, q4 ; CHECK-NEXT: vpt.i16 eq, q4, q3 ; CHECK-NEXT: vmovt q5, q1 -; CHECK-NEXT: vctp.16 r3 -; CHECK-NEXT: vpst -; CHECK-NEXT: vldrbt.u16 q6, [r0] +; CHECK-NEXT: vldrb.u16 q6, [r0] ; CHECK-NEXT: vsub.i16 q4, q2, q5 -; CHECK-NEXT: subs r3, #8 ; CHECK-NEXT: vmul.i16 q5, q5, q6 ; CHECK-NEXT: vmla.i16 q5, q4, r4 ; CHECK-NEXT: vshr.u16 q4, q5, #8 -; CHECK-NEXT: vpst -; CHECK-NEXT: vstrbt.16 q4, [r0], #8 +; CHECK-NEXT: vstrb.16 q4, [r0], #8 ; CHECK-NEXT: vidup.u16 q4, r6, #4 -; CHECK-NEXT: le lr, .LBB0_3 +; CHECK-NEXT: letp lr, .LBB0_3 ; CHECK-NEXT: @ %bb.4: @ %do.end ; CHECK-NEXT: @ in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: add.w r0, r11, #1 -- GitLab From 10dcba76545b8b8c58456ba99a572d53183c9d76 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 6 Jun 2024 13:55:33 +0200 Subject: [PATCH 080/462] [clang-tidy] Remove redundant LINK_LIBS (#94588) clangAnalysis is already being pulled in via clang_target_link_libraries(). Also listing it in LINK_LIBS means that we'll link both against the static libraries and the shared libclang-cpp.so library if CLANG_LINK_CLANG_DYLIB is enabled, and waste time on unnecessary LTO. --- clang-tools-extra/clang-tidy/misc/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/clang-tools-extra/clang-tidy/misc/CMakeLists.txt b/clang-tools-extra/clang-tidy/misc/CMakeLists.txt index 35e29b9a7d13..36fcd8fc1b27 100644 --- a/clang-tools-extra/clang-tidy/misc/CMakeLists.txt +++ b/clang-tools-extra/clang-tidy/misc/CMakeLists.txt @@ -43,7 +43,6 @@ add_clang_library(clangTidyMiscModule UseAnonymousNamespaceCheck.cpp LINK_LIBS - clangAnalysis clangTidy clangTidyUtils -- GitLab From e040474f493d3392af3fbd4284bb1448a7eb4341 Mon Sep 17 00:00:00 2001 From: lntue <35648136+lntue@users.noreply.github.com> Date: Thu, 6 Jun 2024 07:59:02 -0400 Subject: [PATCH 081/462] [libc][math] Temporarily disable nexttowardf16 on aarch64 due to clang-11 bug. (#94569) The conversion between _Float16 and long double will crash clang-11 on aarch64. This is fixed in clang-12: https://godbolt.org/z/8ceT9454c --- libc/config/linux/aarch64/entrypoints.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index a8cb468b3cba..13a4445aca76 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -516,7 +516,10 @@ if(LIBC_TYPES_HAS_FLOAT16) libc.src.math.nearbyintf16 libc.src.math.nextafterf16 libc.src.math.nextdownf16 - libc.src.math.nexttowardf16 + # Temporarily disable nexttowardf16 on aarch64 because the conversion + # between _Float16 and long double will crash clang-11. This is fixed in + # clang-12 and after: https://godbolt.org/z/8ceT9454c + # libc.src.math.nexttowardf16 libc.src.math.nextupf16 libc.src.math.rintf16 libc.src.math.roundf16 -- GitLab From 8725b672071f24721bee11caa767eed6e773fce7 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 6 Jun 2024 13:26:11 +0100 Subject: [PATCH 082/462] [DAG] expandABS - add missing FREEZE in abs(x) -> smax(x,sub(0,x)) expansion Noticed while working on #94601 --- llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index f856c8a51984..ee0bce344a01 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -9159,6 +9159,7 @@ SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG, if (!IsNegative && isOperationLegal(ISD::SUB, VT) && isOperationLegal(ISD::SMAX, VT)) { SDValue Zero = DAG.getConstant(0, dl, VT); + Op = DAG.getFreeze(Op); return DAG.getNode(ISD::SMAX, dl, VT, Op, DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); } @@ -9175,8 +9176,8 @@ SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG, // 0 - abs(x) -> smin(x, sub(0,x)) if (IsNegative && isOperationLegal(ISD::SUB, VT) && isOperationLegal(ISD::SMIN, VT)) { - Op = DAG.getFreeze(Op); SDValue Zero = DAG.getConstant(0, dl, VT); + Op = DAG.getFreeze(Op); return DAG.getNode(ISD::SMIN, dl, VT, Op, DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); } -- GitLab From b025d6913ef7160699e96f0d00820873807310c8 Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Thu, 6 Jun 2024 07:28:41 -0500 Subject: [PATCH 083/462] [flang][OpenMP] Make object identity more precise (#94495) Derived type components may use a given `Symbol` regardless of what parent objects they are a part of. Because of that, simply using a symbol address is not sufficient to determine object identity. Make the designator a part of the IdTy. To compare identities, when symbols are equal (and non-null), compare the designators. --- flang/lib/Lower/OpenMP/Clauses.h | 50 ++++++++++++++++--- flang/lib/Lower/OpenMP/Utils.cpp | 2 +- flang/test/Lower/OpenMP/map-component-ref.f90 | 41 +++++++++++---- 3 files changed, 75 insertions(+), 18 deletions(-) diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h index f7cd0ea83ad1..98fb5dcf7722 100644 --- a/flang/lib/Lower/OpenMP/Clauses.h +++ b/flang/lib/Lower/OpenMP/Clauses.h @@ -36,30 +36,64 @@ struct TypeTy : public evaluate::SomeType { bool operator==(const TypeTy &t) const { return true; } }; -using IdTy = semantics::Symbol *; +template +struct IdTyTemplate { + // "symbol" is always non-null for id's of actual objects. + Fortran::semantics::Symbol *symbol; + std::optional designator; + + bool operator==(const IdTyTemplate &other) const { + // If symbols are different, then the objects are different. + if (symbol != other.symbol) + return false; + if (symbol == nullptr) + return true; + // Equal symbols don't necessarily indicate identical objects, + // for example, a derived object component may use a single symbol, + // which will refer to different objects for different designators, + // e.g. a%c and b%c. + return designator == other.designator; + } + + operator bool() const { return symbol != nullptr; } +}; + using ExprTy = SomeExpr; template using List = tomp::ListT; } // namespace Fortran::lower::omp +// Specialization of the ObjectT template namespace tomp::type { template <> -struct ObjectT { - using IdTy = Fortran::lower::omp::IdTy; +struct ObjectT, + Fortran::lower::omp::ExprTy> { + using IdTy = Fortran::lower::omp::IdTyTemplate; using ExprTy = Fortran::lower::omp::ExprTy; - IdTy id() const { return symbol; } - Fortran::semantics::Symbol *sym() const { return symbol; } - const std::optional &ref() const { return designator; } + IdTy id() const { return identity; } + Fortran::semantics::Symbol *sym() const { return identity.symbol; } + const std::optional &ref() const { return identity.designator; } - IdTy symbol; - std::optional designator; + IdTy identity; }; } // namespace tomp::type namespace Fortran::lower::omp { +using IdTy = IdTyTemplate; +} +namespace std { +template <> +struct hash { + size_t operator()(const Fortran::lower::omp::IdTy &id) const { + return static_cast(reinterpret_cast(id.symbol)); + } +}; +} // namespace std + +namespace Fortran::lower::omp { using Object = tomp::ObjectT; using ObjectList = tomp::ObjectListT; diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp index eff915f569f2..da94352a84a7 100644 --- a/flang/lib/Lower/OpenMP/Utils.cpp +++ b/flang/lib/Lower/OpenMP/Utils.cpp @@ -188,7 +188,7 @@ void addChildIndexAndMapToParent( std::map> &parentMemberIndices, mlir::omp::MapInfoOp &mapOp, semantics::SemanticsContext &semaCtx) { - std::optional dataRef = ExtractDataRef(object.designator); + std::optional dataRef = ExtractDataRef(object.ref()); assert(dataRef.has_value() && "DataRef could not be extracted during mapping of derived type " "cannot proceed"); diff --git a/flang/test/Lower/OpenMP/map-component-ref.f90 b/flang/test/Lower/OpenMP/map-component-ref.f90 index 2c582667f38d..21b56ab303ac 100644 --- a/flang/test/Lower/OpenMP/map-component-ref.f90 +++ b/flang/test/Lower/OpenMP/map-component-ref.f90 @@ -1,21 +1,22 @@ ! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s ! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s -! CHECK: %[[V0:[0-9]+]] = fir.alloca !fir.type<_QFfooTt0{a0:i32,a1:i32}> {bindc_name = "a", uniq_name = "_QFfooEa"} -! CHECK: %[[V1:[0-9]+]]:2 = hlfir.declare %[[V0]] {uniq_name = "_QFfooEa"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) -! CHECK: %[[V2:[0-9]+]] = hlfir.designate %[[V1]]#0{"a1"} : (!fir.ref>) -> !fir.ref +! CHECK-LABEL: func.func @_QPfoo1 +! CHECK: %[[V0:[0-9]+]] = fir.alloca !fir.type<_QFfoo1Tt0{a0:i32,a1:i32}> {bindc_name = "a", uniq_name = "_QFfoo1Ea"} +! CHECK: %[[V1:[0-9]+]]:2 = hlfir.declare %[[V0]] {uniq_name = "_QFfoo1Ea"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[V2:[0-9]+]] = hlfir.designate %[[V1]]#0{"a1"} : (!fir.ref>) -> !fir.ref ! CHECK: %[[V3:[0-9]+]] = omp.map.info var_ptr(%[[V2]] : !fir.ref, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref {name = "a%a1"} -! CHECK: %[[V4:[0-9]+]] = omp.map.info var_ptr(%[[V1]]#1 : !fir.ref>, !fir.type<_QFfooTt0{a0:i32,a1:i32}>) map_clauses(tofrom) capture(ByRef) members(%[[V3]] : [1] : !fir.ref) -> !fir.ref> {name = "a", partial_map = true} -! CHECK: omp.target map_entries(%[[V3]] -> %arg0, %[[V4]] -> %arg1 : !fir.ref, !fir.ref>) { -! CHECK: ^bb0(%arg0: !fir.ref, %arg1: !fir.ref>): -! CHECK: %[[V5:[0-9]+]]:2 = hlfir.declare %arg1 {uniq_name = "_QFfooEa"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[V4:[0-9]+]] = omp.map.info var_ptr(%[[V1]]#1 : !fir.ref>, !fir.type<_QFfoo1Tt0{a0:i32,a1:i32}>) map_clauses(tofrom) capture(ByRef) members(%[[V3]] : [1] : !fir.ref) -> !fir.ref> {name = "a", partial_map = true} +! CHECK: omp.target map_entries(%[[V3]] -> %arg0, %[[V4]] -> %arg1 : !fir.ref, !fir.ref>) { +! CHECK: ^bb0(%arg0: !fir.ref, %arg1: !fir.ref>): +! CHECK: %[[V5:[0-9]+]]:2 = hlfir.declare %arg1 {uniq_name = "_QFfoo1Ea"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) ! CHECK: %c0_i32 = arith.constant 0 : i32 -! CHECK: %[[V6:[0-9]+]] = hlfir.designate %[[V5]]#0{"a1"} : (!fir.ref>) -> !fir.ref +! CHECK: %[[V6:[0-9]+]] = hlfir.designate %[[V5]]#0{"a1"} : (!fir.ref>) -> !fir.ref ! CHECK: hlfir.assign %c0_i32 to %[[V6]] : i32, !fir.ref ! CHECK: omp.terminator ! CHECK: } -subroutine foo() +subroutine foo1() implicit none type t0 @@ -29,3 +30,25 @@ subroutine foo() !$omp end target end + +! CHECK-LABEL: func.func @_QPfoo2 +! CHECK-DAG: omp.map.info var_ptr(%{{[0-9]+}} : {{.*}} map_clauses(to) capture(ByRef) bounds(%{{[0-9]+}}) -> {{.*}} {name = "t%b(1_8)%a(1)"} +! CHECK-DAG: omp.map.info var_ptr(%{{[0-9]+}} : {{.*}} map_clauses(from) capture(ByRef) bounds(%{{[0-9]+}}) -> {{.*}} {name = "u%b(1_8)%a(1)"} +subroutine foo2() + implicit none + + type t0 + integer :: a(10) + end type + + type t1 + type(t0) :: b(10) + end type + + type(t1) :: t, u + +!$omp target map(to: t%b(1)%a(1)) map(from: u%b(1)%a(1)) + t%b(1)%a(1) = u%b(1)%a(1) +!$omp end target + +end -- GitLab From c5c530f4f900b5abfbea5c0f9c934161454773e7 Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Thu, 6 Jun 2024 07:29:26 -0500 Subject: [PATCH 084/462] [clang][Sema] Add missing scope flags to Scope::dumpImpl (#94529) There were a handlful of scope flags that were not handled in the dump function, which would then lead to an assert. --- clang/lib/Sema/Scope.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clang/lib/Sema/Scope.cpp b/clang/lib/Sema/Scope.cpp index c08073e80ff3..5bc7e79a6818 100644 --- a/clang/lib/Sema/Scope.cpp +++ b/clang/lib/Sema/Scope.cpp @@ -228,7 +228,11 @@ void Scope::dumpImpl(raw_ostream &OS) const { {CompoundStmtScope, "CompoundStmtScope"}, {ClassInheritanceScope, "ClassInheritanceScope"}, {CatchScope, "CatchScope"}, + {ConditionVarScope, "ConditionVarScope"}, + {OpenMPOrderClauseScope, "OpenMPOrderClauseScope"}, + {LambdaScope, "LambdaScope"}, {OpenACCComputeConstructScope, "OpenACCComputeConstructScope"}, + {TypeAliasScope, "TypeAliasScope"}, {FriendScope, "FriendScope"}, }; -- GitLab From 41d73504c95342410ac954774b9583be73c97bf9 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 6 Jun 2024 13:43:54 +0100 Subject: [PATCH 085/462] [ConstraintElim] Add set of tests where a loop iv is used in exit. Test cases inspired by https://github.com/llvm/llvm-project/issues/90417. --- .../induction-condition-in-loop-exit.ll | 277 ++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit.ll diff --git a/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit.ll b/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit.ll new file mode 100644 index 000000000000..44ce82b51d70 --- /dev/null +++ b/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit.ll @@ -0,0 +1,277 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -p constraint-elimination -S %s | FileCheck %s + +define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_known(ptr %s) { +; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_known( +; CHECK-SAME: ptr [[S:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]] +; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; CHECK-NEXT: br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[T:%.*]] = icmp ult i32 [[IV]], 1235 +; CHECK-NEXT: ret i1 [[T]] +; +entry: + br label %loop.header + +loop.header: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ] + %exitcond.not = icmp eq i32 %iv, 1234 + br i1 %exitcond.not, label %exit, label %loop.latch + +loop.latch: + %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv + %0 = load i8, ptr %arrayidx, align 1 + %latch.c = icmp ult i8 %0, 10 + %iv.next = add nuw nsw i32 %iv, 1 + br i1 %latch.c, label %loop.header, label %exit + +exit: + %t = icmp ult i32 %iv, 1235 + ret i1 %t +} + +define i1 @multi_exiting_loop_eq_same_exit_with_out_loop_preds_const_compare_not_known(ptr %s, i1 %pre.c, i32 %x) { +; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_exit_with_out_loop_preds_const_compare_not_known( +; CHECK-SAME: ptr [[S:%.*]], i1 [[PRE_C:%.*]], i32 [[X:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br i1 [[PRE_C]], label %[[LOOP_HEADER:.*]], label %[[EXIT:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]] +; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; CHECK-NEXT: br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[X]], %[[ENTRY]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ [[IV]], %[[LOOP_LATCH]] ] +; CHECK-NEXT: [[U:%.*]] = icmp ult i32 [[P]], 1235 +; CHECK-NEXT: ret i1 [[U]] +; +entry: + br i1 %pre.c, label %loop.header, label %exit + +loop.header: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ] + %exitcond.not = icmp eq i32 %iv, 1234 + br i1 %exitcond.not, label %exit, label %loop.latch + +loop.latch: + %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv + %0 = load i8, ptr %arrayidx, align 1 + %latch.c = icmp ult i8 %0, 10 + %iv.next = add nuw nsw i32 %iv, 1 + br i1 %latch.c, label %loop.header, label %exit + +exit: + %p = phi i32 [ %x, %entry ], [ %iv, %loop.header ], [ %iv, %loop.latch ] + %u = icmp ult i32 %p, 1235 + ret i1 %u +} + +define i1 @multi_exiting_loop_eq_same_unique_exit_successors_swapped(ptr %s) { +; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_successors_swapped( +; CHECK-SAME: ptr [[S:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[LOOP_LATCH]], label %[[EXIT:.*]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]] +; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; CHECK-NEXT: br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[U:%.*]] = icmp ult i32 [[IV]], 1235 +; CHECK-NEXT: ret i1 [[U]] +; +entry: + br label %loop.header + +loop.header: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ] + %exitcond.not = icmp eq i32 %iv, 1234 + br i1 %exitcond.not, label %loop.latch, label %exit + +loop.latch: + %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv + %0 = load i8, ptr %arrayidx, align 1 + %latch.c = icmp ult i8 %0, 10 + %iv.next = add nuw nsw i32 %iv, 1 + br i1 %latch.c, label %loop.header, label %exit + +exit: + %u = icmp ult i32 %iv, 1235 + ret i1 %u +} + +define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known(ptr %s) { +; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known( +; CHECK-SAME: ptr [[S:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]] +; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; CHECK-NEXT: br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[U:%.*]] = icmp ult i32 [[IV]], 1234 +; CHECK-NEXT: ret i1 [[U]] +; +entry: + br label %loop.header + +loop.header: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ] + %exitcond.not = icmp eq i32 %iv, 1234 + br i1 %exitcond.not, label %exit, label %loop.latch + +loop.latch: + %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv + %0 = load i8, ptr %arrayidx, align 1 + %latch.c = icmp ult i8 %0, 10 + %iv.next = add nuw nsw i32 %iv, 1 + br i1 %latch.c, label %loop.header, label %exit + +exit: + %u = icmp ult i32 %iv, 1234 + ret i1 %u +} + +define i1 @multi_exiting_loop_eq_same_unique_exit_var_compare_known(ptr %s, i32 %N) { +; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_var_compare_known( +; CHECK-SAME: ptr [[S:%.*]], i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], [[N]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]] +; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; CHECK-NEXT: br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[T:%.*]] = icmp ule i32 [[IV]], [[N]] +; CHECK-NEXT: ret i1 [[T]] +; +entry: + br label %loop.header + +loop.header: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ] + %exitcond.not = icmp eq i32 %iv, %N + br i1 %exitcond.not, label %exit, label %loop.latch + +loop.latch: + %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv + %0 = load i8, ptr %arrayidx, align 1 + %latch.c = icmp ult i8 %0, 10 + %iv.next = add nuw nsw i32 %iv, 1 + br i1 %latch.c, label %loop.header, label %exit + +exit: + %t = icmp ule i32 %iv, %N + ret i1 %t +} + +define i1 @multi_exiting_loop_ne_same_unique_exit_const_compare_known(ptr %s) { +; CHECK-LABEL: define i1 @multi_exiting_loop_ne_same_unique_exit_const_compare_known( +; CHECK-SAME: ptr [[S:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp ne i32 [[IV]], 1234 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[LOOP_LATCH]], label %[[EXIT:.*]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]] +; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; CHECK-NEXT: br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[T:%.*]] = icmp ult i32 [[IV]], 1235 +; CHECK-NEXT: ret i1 [[T]] +; +entry: + br label %loop.header + +loop.header: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ] + %exitcond.not = icmp ne i32 %iv, 1234 + br i1 %exitcond.not, label %loop.latch, label %exit + +loop.latch: + %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv + %0 = load i8, ptr %arrayidx, align 1 + %latch.c = icmp ult i8 %0, 10 + %iv.next = add nuw nsw i32 %iv, 1 + br i1 %latch.c, label %loop.header, label %exit + +exit: + %t = icmp ult i32 %iv, 1235 + ret i1 %t +} + +define i1 @multi_exiting_loop_ne_same_unique_exit_successors_swapped(ptr %s) { +; CHECK-LABEL: define i1 @multi_exiting_loop_ne_same_unique_exit_successors_swapped( +; CHECK-SAME: ptr [[S:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp ne i32 [[IV]], 1234 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]] +; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; CHECK-NEXT: br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[U:%.*]] = icmp ult i32 [[IV]], 1235 +; CHECK-NEXT: ret i1 [[U]] +; +entry: + br label %loop.header + +loop.header: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ] + %exitcond.not = icmp ne i32 %iv, 1234 + br i1 %exitcond.not, label %exit, label %loop.latch + +loop.latch: + %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv + %0 = load i8, ptr %arrayidx, align 1 + %latch.c = icmp ult i8 %0, 10 + %iv.next = add nuw nsw i32 %iv, 1 + br i1 %latch.c, label %loop.header, label %exit + +exit: + %u = icmp ult i32 %iv, 1235 + ret i1 %u +} -- GitLab From d075b7bbace8c0ef983ea6f9aad175bec3ede729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?W=C3=81NG=20Xu=C4=9Bru=C3=AC?= Date: Thu, 6 Jun 2024 20:49:54 +0800 Subject: [PATCH 086/462] [LoongArch] Allow f16 codegen with expansion to libcalls (#94456) The test case is adapted from llvm/test/CodeGen/RISCV/fp16-promote.ll, because it covers some more IR patterns that ought to be common. Fixes #93894 --- .../LoongArch/LoongArchISelLowering.cpp | 8 + llvm/test/CodeGen/LoongArch/fp16-promote.ll | 326 ++++++++++++++++++ 2 files changed, 334 insertions(+) create mode 100644 llvm/test/CodeGen/LoongArch/fp16-promote.ll diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index 32e02e3ee8f5..9d7e4636abac 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -171,6 +171,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM, // Set operations for 'F' feature. if (Subtarget.hasBasicF()) { + setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); + setTruncStoreAction(MVT::f32, MVT::f16, Expand); setCondCodeAction(FPCCToExpand, MVT::f32, Expand); setOperationAction(ISD::SELECT_CC, MVT::f32, Expand); @@ -186,6 +188,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM, setOperationAction(ISD::FSINCOS, MVT::f32, Expand); setOperationAction(ISD::FPOW, MVT::f32, Expand); setOperationAction(ISD::FREM, MVT::f32, Expand); + setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand); + setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand); if (Subtarget.is64Bit()) setOperationAction(ISD::FRINT, MVT::f32, Legal); @@ -202,7 +206,9 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM, // Set operations for 'D' feature. if (Subtarget.hasBasicD()) { + setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); + setTruncStoreAction(MVT::f64, MVT::f16, Expand); setTruncStoreAction(MVT::f64, MVT::f32, Expand); setCondCodeAction(FPCCToExpand, MVT::f64, Expand); @@ -219,6 +225,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM, setOperationAction(ISD::FSINCOS, MVT::f64, Expand); setOperationAction(ISD::FPOW, MVT::f64, Expand); setOperationAction(ISD::FREM, MVT::f64, Expand); + setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); + setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand); if (Subtarget.is64Bit()) setOperationAction(ISD::FRINT, MVT::f64, Legal); diff --git a/llvm/test/CodeGen/LoongArch/fp16-promote.ll b/llvm/test/CodeGen/LoongArch/fp16-promote.ll new file mode 100644 index 000000000000..75f920b43a06 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/fp16-promote.ll @@ -0,0 +1,326 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc --mtriple=loongarch32 --mattr=+d < %s | FileCheck %s --check-prefix=LA32 +; RUN: llc --mtriple=loongarch64 --mattr=+d < %s | FileCheck %s --check-prefix=LA64 + +define void @test_load_store(ptr %p, ptr %q) nounwind { +; LA32-LABEL: test_load_store: +; LA32: # %bb.0: +; LA32-NEXT: ld.h $a0, $a0, 0 +; LA32-NEXT: st.h $a0, $a1, 0 +; LA32-NEXT: ret +; +; LA64-LABEL: test_load_store: +; LA64: # %bb.0: +; LA64-NEXT: ld.h $a0, $a0, 0 +; LA64-NEXT: st.h $a0, $a1, 0 +; LA64-NEXT: ret + %a = load half, ptr %p + store half %a, ptr %q + ret void +} + +define float @test_fpextend_float(ptr %p) nounwind { +; LA32-LABEL: test_fpextend_float: +; LA32: # %bb.0: +; LA32-NEXT: ld.hu $a0, $a0, 0 +; LA32-NEXT: b %plt(__gnu_h2f_ieee) +; +; LA64-LABEL: test_fpextend_float: +; LA64: # %bb.0: +; LA64-NEXT: ld.hu $a0, $a0, 0 +; LA64-NEXT: b %plt(__gnu_h2f_ieee) + %a = load half, ptr %p + %r = fpext half %a to float + ret float %r +} + +define double @test_fpextend_double(ptr %p) nounwind { +; LA32-LABEL: test_fpextend_double: +; LA32: # %bb.0: +; LA32-NEXT: addi.w $sp, $sp, -16 +; LA32-NEXT: st.w $ra, $sp, 12 # 4-byte Folded Spill +; LA32-NEXT: ld.hu $a0, $a0, 0 +; LA32-NEXT: bl %plt(__gnu_h2f_ieee) +; LA32-NEXT: fcvt.d.s $fa0, $fa0 +; LA32-NEXT: ld.w $ra, $sp, 12 # 4-byte Folded Reload +; LA32-NEXT: addi.w $sp, $sp, 16 +; LA32-NEXT: ret +; +; LA64-LABEL: test_fpextend_double: +; LA64: # %bb.0: +; LA64-NEXT: addi.d $sp, $sp, -16 +; LA64-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill +; LA64-NEXT: ld.hu $a0, $a0, 0 +; LA64-NEXT: bl %plt(__gnu_h2f_ieee) +; LA64-NEXT: fcvt.d.s $fa0, $fa0 +; LA64-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload +; LA64-NEXT: addi.d $sp, $sp, 16 +; LA64-NEXT: ret + %a = load half, ptr %p + %r = fpext half %a to double + ret double %r +} + +define void @test_fptrunc_float(float %f, ptr %p) nounwind { +; LA32-LABEL: test_fptrunc_float: +; LA32: # %bb.0: +; LA32-NEXT: addi.w $sp, $sp, -16 +; LA32-NEXT: st.w $ra, $sp, 12 # 4-byte Folded Spill +; LA32-NEXT: st.w $fp, $sp, 8 # 4-byte Folded Spill +; LA32-NEXT: move $fp, $a0 +; LA32-NEXT: bl %plt(__gnu_f2h_ieee) +; LA32-NEXT: st.h $a0, $fp, 0 +; LA32-NEXT: ld.w $fp, $sp, 8 # 4-byte Folded Reload +; LA32-NEXT: ld.w $ra, $sp, 12 # 4-byte Folded Reload +; LA32-NEXT: addi.w $sp, $sp, 16 +; LA32-NEXT: ret +; +; LA64-LABEL: test_fptrunc_float: +; LA64: # %bb.0: +; LA64-NEXT: addi.d $sp, $sp, -16 +; LA64-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill +; LA64-NEXT: st.d $fp, $sp, 0 # 8-byte Folded Spill +; LA64-NEXT: move $fp, $a0 +; LA64-NEXT: bl %plt(__gnu_f2h_ieee) +; LA64-NEXT: st.h $a0, $fp, 0 +; LA64-NEXT: ld.d $fp, $sp, 0 # 8-byte Folded Reload +; LA64-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload +; LA64-NEXT: addi.d $sp, $sp, 16 +; LA64-NEXT: ret + %a = fptrunc float %f to half + store half %a, ptr %p + ret void +} + +define void @test_fptrunc_double(double %d, ptr %p) nounwind { +; LA32-LABEL: test_fptrunc_double: +; LA32: # %bb.0: +; LA32-NEXT: addi.w $sp, $sp, -16 +; LA32-NEXT: st.w $ra, $sp, 12 # 4-byte Folded Spill +; LA32-NEXT: st.w $fp, $sp, 8 # 4-byte Folded Spill +; LA32-NEXT: move $fp, $a0 +; LA32-NEXT: bl %plt(__truncdfhf2) +; LA32-NEXT: st.h $a0, $fp, 0 +; LA32-NEXT: ld.w $fp, $sp, 8 # 4-byte Folded Reload +; LA32-NEXT: ld.w $ra, $sp, 12 # 4-byte Folded Reload +; LA32-NEXT: addi.w $sp, $sp, 16 +; LA32-NEXT: ret +; +; LA64-LABEL: test_fptrunc_double: +; LA64: # %bb.0: +; LA64-NEXT: addi.d $sp, $sp, -16 +; LA64-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill +; LA64-NEXT: st.d $fp, $sp, 0 # 8-byte Folded Spill +; LA64-NEXT: move $fp, $a0 +; LA64-NEXT: bl %plt(__truncdfhf2) +; LA64-NEXT: st.h $a0, $fp, 0 +; LA64-NEXT: ld.d $fp, $sp, 0 # 8-byte Folded Reload +; LA64-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload +; LA64-NEXT: addi.d $sp, $sp, 16 +; LA64-NEXT: ret + %a = fptrunc double %d to half + store half %a, ptr %p + ret void +} + +define half @test_fadd_reg(half %a, half %b) nounwind { +; LA32-LABEL: test_fadd_reg: +; LA32: # %bb.0: +; LA32-NEXT: addi.w $sp, $sp, -32 +; LA32-NEXT: st.w $ra, $sp, 28 # 4-byte Folded Spill +; LA32-NEXT: fst.d $fs0, $sp, 16 # 8-byte Folded Spill +; LA32-NEXT: fst.d $fs1, $sp, 8 # 8-byte Folded Spill +; LA32-NEXT: fmov.s $fs0, $fa0 +; LA32-NEXT: fmov.s $fa0, $fa1 +; LA32-NEXT: bl %plt(__gnu_f2h_ieee) +; LA32-NEXT: bl %plt(__gnu_h2f_ieee) +; LA32-NEXT: fmov.s $fs1, $fa0 +; LA32-NEXT: fmov.s $fa0, $fs0 +; LA32-NEXT: bl %plt(__gnu_f2h_ieee) +; LA32-NEXT: bl %plt(__gnu_h2f_ieee) +; LA32-NEXT: fadd.s $fa0, $fa0, $fs1 +; LA32-NEXT: fld.d $fs1, $sp, 8 # 8-byte Folded Reload +; LA32-NEXT: fld.d $fs0, $sp, 16 # 8-byte Folded Reload +; LA32-NEXT: ld.w $ra, $sp, 28 # 4-byte Folded Reload +; LA32-NEXT: addi.w $sp, $sp, 32 +; LA32-NEXT: ret +; +; LA64-LABEL: test_fadd_reg: +; LA64: # %bb.0: +; LA64-NEXT: addi.d $sp, $sp, -32 +; LA64-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill +; LA64-NEXT: fst.d $fs0, $sp, 16 # 8-byte Folded Spill +; LA64-NEXT: fst.d $fs1, $sp, 8 # 8-byte Folded Spill +; LA64-NEXT: fmov.s $fs0, $fa0 +; LA64-NEXT: fmov.s $fa0, $fa1 +; LA64-NEXT: bl %plt(__gnu_f2h_ieee) +; LA64-NEXT: bl %plt(__gnu_h2f_ieee) +; LA64-NEXT: fmov.s $fs1, $fa0 +; LA64-NEXT: fmov.s $fa0, $fs0 +; LA64-NEXT: bl %plt(__gnu_f2h_ieee) +; LA64-NEXT: bl %plt(__gnu_h2f_ieee) +; LA64-NEXT: fadd.s $fa0, $fa0, $fs1 +; LA64-NEXT: fld.d $fs1, $sp, 8 # 8-byte Folded Reload +; LA64-NEXT: fld.d $fs0, $sp, 16 # 8-byte Folded Reload +; LA64-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload +; LA64-NEXT: addi.d $sp, $sp, 32 +; LA64-NEXT: ret + %r = fadd half %a, %b + ret half %r +} + +define void @test_fadd_mem(ptr %p, ptr %q) nounwind { +; LA32-LABEL: test_fadd_mem: +; LA32: # %bb.0: +; LA32-NEXT: addi.w $sp, $sp, -32 +; LA32-NEXT: st.w $ra, $sp, 28 # 4-byte Folded Spill +; LA32-NEXT: st.w $fp, $sp, 24 # 4-byte Folded Spill +; LA32-NEXT: st.w $s0, $sp, 20 # 4-byte Folded Spill +; LA32-NEXT: fst.d $fs0, $sp, 8 # 8-byte Folded Spill +; LA32-NEXT: move $fp, $a1 +; LA32-NEXT: move $s0, $a0 +; LA32-NEXT: ld.hu $a0, $a0, 0 +; LA32-NEXT: bl %plt(__gnu_h2f_ieee) +; LA32-NEXT: fmov.s $fs0, $fa0 +; LA32-NEXT: ld.hu $a0, $fp, 0 +; LA32-NEXT: bl %plt(__gnu_h2f_ieee) +; LA32-NEXT: fadd.s $fa0, $fs0, $fa0 +; LA32-NEXT: bl %plt(__gnu_f2h_ieee) +; LA32-NEXT: st.h $a0, $s0, 0 +; LA32-NEXT: fld.d $fs0, $sp, 8 # 8-byte Folded Reload +; LA32-NEXT: ld.w $s0, $sp, 20 # 4-byte Folded Reload +; LA32-NEXT: ld.w $fp, $sp, 24 # 4-byte Folded Reload +; LA32-NEXT: ld.w $ra, $sp, 28 # 4-byte Folded Reload +; LA32-NEXT: addi.w $sp, $sp, 32 +; LA32-NEXT: ret +; +; LA64-LABEL: test_fadd_mem: +; LA64: # %bb.0: +; LA64-NEXT: addi.d $sp, $sp, -32 +; LA64-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill +; LA64-NEXT: st.d $fp, $sp, 16 # 8-byte Folded Spill +; LA64-NEXT: st.d $s0, $sp, 8 # 8-byte Folded Spill +; LA64-NEXT: fst.d $fs0, $sp, 0 # 8-byte Folded Spill +; LA64-NEXT: move $fp, $a1 +; LA64-NEXT: move $s0, $a0 +; LA64-NEXT: ld.hu $a0, $a0, 0 +; LA64-NEXT: bl %plt(__gnu_h2f_ieee) +; LA64-NEXT: fmov.s $fs0, $fa0 +; LA64-NEXT: ld.hu $a0, $fp, 0 +; LA64-NEXT: bl %plt(__gnu_h2f_ieee) +; LA64-NEXT: fadd.s $fa0, $fs0, $fa0 +; LA64-NEXT: bl %plt(__gnu_f2h_ieee) +; LA64-NEXT: st.h $a0, $s0, 0 +; LA64-NEXT: fld.d $fs0, $sp, 0 # 8-byte Folded Reload +; LA64-NEXT: ld.d $s0, $sp, 8 # 8-byte Folded Reload +; LA64-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload +; LA64-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload +; LA64-NEXT: addi.d $sp, $sp, 32 +; LA64-NEXT: ret + %a = load half, ptr %p + %b = load half, ptr %q + %r = fadd half %a, %b + store half %r, ptr %p + ret void +} + +define half @test_fmul_reg(half %a, half %b) nounwind { +; LA32-LABEL: test_fmul_reg: +; LA32: # %bb.0: +; LA32-NEXT: addi.w $sp, $sp, -32 +; LA32-NEXT: st.w $ra, $sp, 28 # 4-byte Folded Spill +; LA32-NEXT: fst.d $fs0, $sp, 16 # 8-byte Folded Spill +; LA32-NEXT: fst.d $fs1, $sp, 8 # 8-byte Folded Spill +; LA32-NEXT: fmov.s $fs0, $fa0 +; LA32-NEXT: fmov.s $fa0, $fa1 +; LA32-NEXT: bl %plt(__gnu_f2h_ieee) +; LA32-NEXT: bl %plt(__gnu_h2f_ieee) +; LA32-NEXT: fmov.s $fs1, $fa0 +; LA32-NEXT: fmov.s $fa0, $fs0 +; LA32-NEXT: bl %plt(__gnu_f2h_ieee) +; LA32-NEXT: bl %plt(__gnu_h2f_ieee) +; LA32-NEXT: fmul.s $fa0, $fa0, $fs1 +; LA32-NEXT: fld.d $fs1, $sp, 8 # 8-byte Folded Reload +; LA32-NEXT: fld.d $fs0, $sp, 16 # 8-byte Folded Reload +; LA32-NEXT: ld.w $ra, $sp, 28 # 4-byte Folded Reload +; LA32-NEXT: addi.w $sp, $sp, 32 +; LA32-NEXT: ret +; +; LA64-LABEL: test_fmul_reg: +; LA64: # %bb.0: +; LA64-NEXT: addi.d $sp, $sp, -32 +; LA64-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill +; LA64-NEXT: fst.d $fs0, $sp, 16 # 8-byte Folded Spill +; LA64-NEXT: fst.d $fs1, $sp, 8 # 8-byte Folded Spill +; LA64-NEXT: fmov.s $fs0, $fa0 +; LA64-NEXT: fmov.s $fa0, $fa1 +; LA64-NEXT: bl %plt(__gnu_f2h_ieee) +; LA64-NEXT: bl %plt(__gnu_h2f_ieee) +; LA64-NEXT: fmov.s $fs1, $fa0 +; LA64-NEXT: fmov.s $fa0, $fs0 +; LA64-NEXT: bl %plt(__gnu_f2h_ieee) +; LA64-NEXT: bl %plt(__gnu_h2f_ieee) +; LA64-NEXT: fmul.s $fa0, $fa0, $fs1 +; LA64-NEXT: fld.d $fs1, $sp, 8 # 8-byte Folded Reload +; LA64-NEXT: fld.d $fs0, $sp, 16 # 8-byte Folded Reload +; LA64-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload +; LA64-NEXT: addi.d $sp, $sp, 32 +; LA64-NEXT: ret + %r = fmul half %a, %b + ret half %r +} + +define void @test_fmul_mem(ptr %p, ptr %q) nounwind { +; LA32-LABEL: test_fmul_mem: +; LA32: # %bb.0: +; LA32-NEXT: addi.w $sp, $sp, -32 +; LA32-NEXT: st.w $ra, $sp, 28 # 4-byte Folded Spill +; LA32-NEXT: st.w $fp, $sp, 24 # 4-byte Folded Spill +; LA32-NEXT: st.w $s0, $sp, 20 # 4-byte Folded Spill +; LA32-NEXT: fst.d $fs0, $sp, 8 # 8-byte Folded Spill +; LA32-NEXT: move $fp, $a1 +; LA32-NEXT: move $s0, $a0 +; LA32-NEXT: ld.hu $a0, $a0, 0 +; LA32-NEXT: bl %plt(__gnu_h2f_ieee) +; LA32-NEXT: fmov.s $fs0, $fa0 +; LA32-NEXT: ld.hu $a0, $fp, 0 +; LA32-NEXT: bl %plt(__gnu_h2f_ieee) +; LA32-NEXT: fmul.s $fa0, $fs0, $fa0 +; LA32-NEXT: bl %plt(__gnu_f2h_ieee) +; LA32-NEXT: st.h $a0, $s0, 0 +; LA32-NEXT: fld.d $fs0, $sp, 8 # 8-byte Folded Reload +; LA32-NEXT: ld.w $s0, $sp, 20 # 4-byte Folded Reload +; LA32-NEXT: ld.w $fp, $sp, 24 # 4-byte Folded Reload +; LA32-NEXT: ld.w $ra, $sp, 28 # 4-byte Folded Reload +; LA32-NEXT: addi.w $sp, $sp, 32 +; LA32-NEXT: ret +; +; LA64-LABEL: test_fmul_mem: +; LA64: # %bb.0: +; LA64-NEXT: addi.d $sp, $sp, -32 +; LA64-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill +; LA64-NEXT: st.d $fp, $sp, 16 # 8-byte Folded Spill +; LA64-NEXT: st.d $s0, $sp, 8 # 8-byte Folded Spill +; LA64-NEXT: fst.d $fs0, $sp, 0 # 8-byte Folded Spill +; LA64-NEXT: move $fp, $a1 +; LA64-NEXT: move $s0, $a0 +; LA64-NEXT: ld.hu $a0, $a0, 0 +; LA64-NEXT: bl %plt(__gnu_h2f_ieee) +; LA64-NEXT: fmov.s $fs0, $fa0 +; LA64-NEXT: ld.hu $a0, $fp, 0 +; LA64-NEXT: bl %plt(__gnu_h2f_ieee) +; LA64-NEXT: fmul.s $fa0, $fs0, $fa0 +; LA64-NEXT: bl %plt(__gnu_f2h_ieee) +; LA64-NEXT: st.h $a0, $s0, 0 +; LA64-NEXT: fld.d $fs0, $sp, 0 # 8-byte Folded Reload +; LA64-NEXT: ld.d $s0, $sp, 8 # 8-byte Folded Reload +; LA64-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload +; LA64-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload +; LA64-NEXT: addi.d $sp, $sp, 32 +; LA64-NEXT: ret + %a = load half, ptr %p + %b = load half, ptr %q + %r = fmul half %a, %b + store half %r, ptr %p + ret void +} -- GitLab From e49f902358ca03b1f11bc43b2eb5306078191622 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Thu, 6 Jun 2024 06:05:29 -0700 Subject: [PATCH 087/462] [workflows] Add scan-build to ci-ubuntu-22.04 container (#94543) This will be used for a new CI job that runs the static analyzer. --- .github/workflows/containers/github-action-ci/stage1.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/containers/github-action-ci/stage1.Dockerfile b/.github/workflows/containers/github-action-ci/stage1.Dockerfile index fbc4548e6636..8c6bcf463841 100644 --- a/.github/workflows/containers/github-action-ci/stage1.Dockerfile +++ b/.github/workflows/containers/github-action-ci/stage1.Dockerfile @@ -37,7 +37,7 @@ RUN cmake -B ./build -G Ninja ./llvm \ -DLLVM_ENABLE_RUNTIMES="compiler-rt" \ -DCMAKE_INSTALL_PREFIX="$LLVM_SYSROOT" \ -DLLVM_ENABLE_PROJECTS="bolt;clang;lld;clang-tools-extra" \ - -DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format" \ + -DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format;scan-build" \ -DCLANG_DEFAULT_LINKER="lld" \ -DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=/llvm-project-llvmorg-$LLVM_VERSION/llvm -- GitLab From e5c93ed3482d483bb5d2876a296cbe603d23d0e8 Mon Sep 17 00:00:00 2001 From: Phoebe Wang Date: Thu, 6 Jun 2024 21:10:14 +0800 Subject: [PATCH 088/462] [X86][AMX] Checking AMXProgModel in X86LowerTileCopy (#94358) This fixes compile time regression after #93692. --- llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 9 ++++++++ llvm/lib/Target/X86/X86ISelLowering.cpp | 13 ++++++++--- llvm/lib/Target/X86/X86InstrAMX.td | 2 +- llvm/lib/Target/X86/X86LowerTileCopy.cpp | 23 +++++--------------- llvm/lib/Target/X86/X86MachineFunctionInfo.h | 12 ++++++++++ 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp index 0bf3294af92a..3933e82b718f 100644 --- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp +++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp @@ -5120,6 +5120,9 @@ void X86DAGToDAGISel::Select(SDNode *Node) { case Intrinsic::x86_tileloaddt164_internal: { if (!Subtarget->hasAMXTILE()) break; + auto *MFI = + CurDAG->getMachineFunction().getInfo(); + MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA); unsigned Opc = IntNo == Intrinsic::x86_tileloadd64_internal ? X86::PTILELOADDV : X86::PTILELOADDT1V; @@ -5201,6 +5204,9 @@ void X86DAGToDAGISel::Select(SDNode *Node) { break; } case Intrinsic::x86_tilestored64_internal: { + auto *MFI = + CurDAG->getMachineFunction().getInfo(); + MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA); unsigned Opc = X86::PTILESTOREDV; // _tile_stored_internal(row, col, buf, STRIDE, c) SDValue Base = Node->getOperand(4); @@ -5228,6 +5234,9 @@ void X86DAGToDAGISel::Select(SDNode *Node) { case Intrinsic::x86_tilestored64: { if (!Subtarget->hasAMXTILE()) break; + auto *MFI = + CurDAG->getMachineFunction().getInfo(); + MFI->setAMXProgModel(AMXProgModelEnum::DirectReg); unsigned Opc; switch (IntNo) { default: llvm_unreachable("Unexpected intrinsic!"); diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index af1e45d25aac..3fbab3af32bb 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -26790,7 +26790,7 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget, case Intrinsic::swift_async_context_addr: { SDLoc dl(Op); auto &MF = DAG.getMachineFunction(); - auto X86FI = MF.getInfo(); + auto *X86FI = MF.getInfo(); if (X86::isExtendedSwiftAsyncFrameSupported(Subtarget, MF)) { MF.getFrameInfo().setFrameAddressIsTaken(true); X86FI->setHasSwiftAsyncContext(true); @@ -36795,7 +36795,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, } case TargetOpcode::PREALLOCATED_SETUP: { assert(Subtarget.is32Bit() && "preallocated only used in 32-bit"); - auto MFI = MF->getInfo(); + auto *MFI = MF->getInfo(); MFI->setHasPreallocatedCall(true); int64_t PreallocatedId = MI.getOperand(0).getImm(); size_t StackAdjustment = MFI->getPreallocatedStackSize(PreallocatedId); @@ -36812,7 +36812,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, assert(Subtarget.is32Bit() && "preallocated calls only used in 32-bit"); int64_t PreallocatedId = MI.getOperand(1).getImm(); int64_t ArgIdx = MI.getOperand(2).getImm(); - auto MFI = MF->getInfo(); + auto *MFI = MF->getInfo(); size_t ArgOffset = MFI->getPreallocatedArgOffsets(PreallocatedId)[ArgIdx]; LLVM_DEBUG(dbgs() << "PREALLOCATED_ARG arg index " << ArgIdx << ", arg offset " << ArgOffset << "\n"); @@ -36855,6 +36855,13 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, unsigned Imm = MI.getOperand(0).getImm(); BuildMI(*BB, MI, MIMD, TII->get(X86::TILEZERO), TMMImmToTMMReg(Imm)); MI.eraseFromParent(); // The pseudo is gone now. + auto *MFI = MF->getInfo(); + MFI->setAMXProgModel(AMXProgModelEnum::DirectReg); + return BB; + } + case X86::PTILEZEROV: { + auto *MFI = MF->getInfo(); + MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA); return BB; } case X86::PTILELOADD: diff --git a/llvm/lib/Target/X86/X86InstrAMX.td b/llvm/lib/Target/X86/X86InstrAMX.td index c47bee070e04..99deacc811a1 100644 --- a/llvm/lib/Target/X86/X86InstrAMX.td +++ b/llvm/lib/Target/X86/X86InstrAMX.td @@ -74,7 +74,7 @@ let SchedRW = [WriteSystem] in { GR16:$src2, opaquemem:$src3, TILE:$src4), []>; let isPseudo = true, isReMaterializable = 1, isAsCheapAsAMove = 1, - canFoldAsLoad = 1 in + canFoldAsLoad = 1, usesCustomInserter = 1 in def PTILEZEROV : PseudoI<(outs TILE:$dst), (ins GR16:$src1, GR16:$src2), [(set TILE:$dst, (int_x86_tilezero_internal GR16:$src1, GR16:$src2))]>; diff --git a/llvm/lib/Target/X86/X86LowerTileCopy.cpp b/llvm/lib/Target/X86/X86LowerTileCopy.cpp index f27676a27e86..613722b398f4 100644 --- a/llvm/lib/Target/X86/X86LowerTileCopy.cpp +++ b/llvm/lib/Target/X86/X86LowerTileCopy.cpp @@ -19,6 +19,7 @@ #include "X86.h" #include "X86InstrBuilder.h" #include "X86InstrInfo.h" +#include "X86MachineFunctionInfo.h" #include "X86Subtarget.h" #include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineBasicBlock.h" @@ -71,6 +72,10 @@ FunctionPass *llvm::createX86LowerTileCopyPass() { } bool X86LowerTileCopy::runOnMachineFunction(MachineFunction &MF) { + X86MachineFunctionInfo *FuncInfo = MF.getInfo(); + if (FuncInfo->getAMXProgModel() != AMXProgModelEnum::ManagedRA) + return false; + const X86Subtarget &ST = MF.getSubtarget(); const X86InstrInfo *TII = ST.getInstrInfo(); const TargetRegisterInfo *TRI = ST.getRegisterInfo(); @@ -81,26 +86,8 @@ bool X86LowerTileCopy::runOnMachineFunction(MachineFunction &MF) { bool Changed = false; for (MachineBasicBlock &MBB : MF) { - // There won't be a tile copy if neither tile register live in nor live out. - bool HasTileCopy = false; - for (const auto &LI : MBB.liveins()) { - if (TILERegs.test(LI.PhysReg)) { - HasTileCopy = true; - break; - } - } LiveRegUnits UsedRegs(*TRI); UsedRegs.addLiveOuts(MBB); - if (!HasTileCopy) { - for (auto RegT : TILERegs.set_bits()) { - if (UsedRegs.available(RegT)) { - HasTileCopy = true; - break; - } - } - } - if (!HasTileCopy) - continue; for (MachineInstr &MI : llvm::make_early_inc_range(reverse(MBB))) { UsedRegs.stepBackward(MI); if (!MI.isCopy()) diff --git a/llvm/lib/Target/X86/X86MachineFunctionInfo.h b/llvm/lib/Target/X86/X86MachineFunctionInfo.h index f6e853270e07..8aaa49945f9d 100644 --- a/llvm/lib/Target/X86/X86MachineFunctionInfo.h +++ b/llvm/lib/Target/X86/X86MachineFunctionInfo.h @@ -21,6 +21,8 @@ namespace llvm { +enum AMXProgModelEnum { None = 0, DirectReg = 1, ManagedRA = 2 }; + /// X86MachineFunctionInfo - This class is derived from MachineFunction and /// contains private X86 target-specific information for each MachineFunction. class X86MachineFunctionInfo : public MachineFunctionInfo { @@ -96,6 +98,9 @@ class X86MachineFunctionInfo : public MachineFunctionInfo { /// used to address arguments in a function using a base pointer. int SEHFramePtrSaveIndex = 0; + /// The AMX programing model used in the function. + AMXProgModelEnum AMXProgModel = AMXProgModelEnum::None; + /// True if this function has a subset of CSRs that is handled explicitly via /// copies. bool IsSplitCSR = false; @@ -219,6 +224,13 @@ public: int getSEHFramePtrSaveIndex() const { return SEHFramePtrSaveIndex; } void setSEHFramePtrSaveIndex(int Index) { SEHFramePtrSaveIndex = Index; } + AMXProgModelEnum getAMXProgModel() const { return AMXProgModel; } + void setAMXProgModel(AMXProgModelEnum Model) { + assert((AMXProgModel == AMXProgModelEnum::None || AMXProgModel == Model) && + "mixed model is not supported"); + AMXProgModel = Model; + } + SmallVectorImpl &getForwardedMustTailRegParms() { return ForwardedMustTailRegParms; } -- GitLab From 435aa7663d56e7216ad148ede3a422675b5f2be1 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Thu, 6 Jun 2024 08:10:56 -0500 Subject: [PATCH 089/462] [Libomptarget] Rework device initialization and image registration (#93844) Summary: Currently, we register images into a linear table according to the logical OpenMP device identifier. We then initialize all of these images as one block. This logic requires that images are compatible with *all* devices instead of just the one that it can run on. This prevents us from running on systems with heterogeneous devices (i.e. image 1 runs on device 0 image 0 runs on device 1). This patch reworks the logic by instead making the compatibility check a per-device query. We then scan every device to see if it's compatible and do it as they come. --- offload/include/PluginManager.h | 27 +- offload/plugins-nextgen/amdgpu/src/rtl.cpp | 21 +- .../common/include/PluginInterface.h | 34 +-- .../common/src/PluginInterface.cpp | 72 ++++-- offload/plugins-nextgen/cuda/src/rtl.cpp | 55 ++-- offload/plugins-nextgen/host/src/rtl.cpp | 4 +- offload/src/PluginManager.cpp | 235 ++++++++---------- offload/src/omptarget.cpp | 2 +- 8 files changed, 232 insertions(+), 218 deletions(-) diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h index 1d6804da75d9..fce2adca49c1 100644 --- a/offload/include/PluginManager.h +++ b/offload/include/PluginManager.h @@ -64,10 +64,6 @@ struct PluginManager { std::make_unique(TgtBinDesc, TgtDeviceImage)); } - /// Initialize as many devices as possible for this plugin. Devices that fail - /// to initialize are ignored. - void initDevices(GenericPluginTy &RTL); - /// Return the device presented to the user as device \p DeviceNo if it is /// initialized and ready. Otherwise return an error explaining the problem. llvm::Expected getDevice(uint32_t DeviceNo); @@ -117,20 +113,31 @@ struct PluginManager { return Devices.getExclusiveAccessor(); } - int getNumUsedPlugins() const { return DeviceOffsets.size(); } - // Initialize all plugins. void initAllPlugins(); /// Iterator range for all plugins (in use or not, but always valid). auto plugins() { return llvm::make_pointee_range(Plugins); } + /// Iterator range for all plugins (in use or not, but always valid). + auto plugins() const { return llvm::make_pointee_range(Plugins); } + /// Return the user provided requirements. int64_t getRequirements() const { return Requirements.getRequirements(); } /// Add \p Flags to the user provided requirements. void addRequirements(int64_t Flags) { Requirements.addRequirements(Flags); } + /// Returns the number of plugins that are active. + int getNumActivePlugins() const { + int count = 0; + for (auto &R : plugins()) + if (R.is_initialized()) + ++count; + + return count; + } + private: bool RTLsLoaded = false; llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc; @@ -138,11 +145,9 @@ private: // List of all plugins, in use or not. llvm::SmallVector> Plugins; - // Mapping of plugins to offsets in the device table. - llvm::DenseMap DeviceOffsets; - - // Mapping of plugins to the number of used devices. - llvm::DenseMap DeviceUsed; + // Mapping of plugins to the OpenMP device identifier. + llvm::DenseMap, int32_t> + DeviceIds; // Set of all device images currently in use. llvm::DenseSet UsedImages; diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp index c6dd954746e4..f088d5d1685d 100644 --- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp +++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp @@ -3163,25 +3163,24 @@ struct AMDGPUPluginTy final : public GenericPluginTy { uint16_t getMagicElfBits() const override { return ELF::EM_AMDGPU; } /// Check whether the image is compatible with an AMDGPU device. - Expected isELFCompatible(StringRef Image) const override { + Expected isELFCompatible(uint32_t DeviceId, + StringRef Image) const override { // Get the associated architecture and flags from the ELF. auto ElfOrErr = ELF64LEObjectFile::create( MemoryBufferRef(Image, /*Identifier=*/""), /*InitContent=*/false); if (!ElfOrErr) return ElfOrErr.takeError(); std::optional Processor = ElfOrErr->tryGetCPUName(); + if (!Processor) + return false; - for (hsa_agent_t Agent : KernelAgents) { - auto TargeTripleAndFeaturesOrError = - utils::getTargetTripleAndFeatures(Agent); - if (!TargeTripleAndFeaturesOrError) - return TargeTripleAndFeaturesOrError.takeError(); - if (!utils::isImageCompatibleWithEnv(Processor ? *Processor : "", + auto TargeTripleAndFeaturesOrError = + utils::getTargetTripleAndFeatures(getKernelAgent(DeviceId)); + if (!TargeTripleAndFeaturesOrError) + return TargeTripleAndFeaturesOrError.takeError(); + return utils::isImageCompatibleWithEnv(Processor ? *Processor : "", ElfOrErr->getPlatformFlags(), - *TargeTripleAndFeaturesOrError)) - return false; - } - return true; + *TargeTripleAndFeaturesOrError); } bool isDataExchangable(int32_t SrcDeviceId, int32_t DstDeviceId) override { diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h index eda6a4fd541e..88423be039af 100644 --- a/offload/plugins-nextgen/common/include/PluginInterface.h +++ b/offload/plugins-nextgen/common/include/PluginInterface.h @@ -993,11 +993,11 @@ struct GenericPluginTy { /// Get the number of active devices. int32_t getNumDevices() const { return NumDevices; } - /// Get the plugin-specific device identifier offset. - int32_t getDeviceIdStartIndex() const { return DeviceIdStartIndex; } - - /// Set the plugin-specific device identifier offset. - void setDeviceIdStartIndex(int32_t Offset) { DeviceIdStartIndex = Offset; } + /// Get the plugin-specific device identifier. + int32_t getUserId(int32_t DeviceId) const { + assert(UserDeviceIds.contains(DeviceId) && "No user-id registered"); + return UserDeviceIds.at(DeviceId); + } /// Get the ELF code to recognize the binary image of this plugin. virtual uint16_t getMagicElfBits() const = 0; @@ -1059,7 +1059,8 @@ struct GenericPluginTy { /// Indicate if an image is compatible with the plugin devices. Notice that /// this function may be called before actually initializing the devices. So /// we could not move this function into GenericDeviceTy. - virtual Expected isELFCompatible(StringRef Image) const = 0; + virtual Expected isELFCompatible(uint32_t DeviceID, + StringRef Image) const = 0; protected: /// Indicate whether a device id is valid. @@ -1070,11 +1071,18 @@ protected: public: // TODO: This plugin interface needs to be cleaned up. - /// Returns true if the plugin has been initialized. + /// Returns non-zero if the plugin runtime has been initialized. int32_t is_initialized() const; - /// Returns non-zero if the provided \p Image can be executed by the runtime. - int32_t is_valid_binary(__tgt_device_image *Image, bool Initialized = true); + /// Returns non-zero if the \p Image is compatible with the plugin. This + /// function does not require the plugin to be initialized before use. + int32_t is_plugin_compatible(__tgt_device_image *Image); + + /// Returns non-zero if the \p Image is compatible with the device. + int32_t is_device_compatible(int32_t DeviceId, __tgt_device_image *Image); + + /// Returns non-zero if the plugin device has been initialized. + int32_t is_device_initialized(int32_t DeviceId) const; /// Initialize the device inside of the plugin. int32_t init_device(int32_t DeviceId); @@ -1180,7 +1188,7 @@ public: const char **ErrStr); /// Sets the offset into the devices for use by OMPT. - int32_t set_device_offset(int32_t DeviceIdOffset); + int32_t set_device_identifier(int32_t UserId, int32_t DeviceId); /// Returns if the plugin can support auotmatic copy. int32_t use_auto_zero_copy(int32_t DeviceId); @@ -1200,10 +1208,8 @@ private: /// Number of devices available for the plugin. int32_t NumDevices = 0; - /// Index offset, which when added to a DeviceId, will yield a unique - /// user-observable device identifier. This is especially important when - /// DeviceIds of multiple plugins / RTLs need to be distinguishable. - int32_t DeviceIdStartIndex = 0; + /// Map of plugin device identifiers to the user device identifier. + llvm::DenseMap UserDeviceIds; /// Array of pointers to the devices. Initially, they are all set to nullptr. /// Once a device is initialized, the pointer is stored in the position given diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp index 913721a15d71..5a53c479e33d 100644 --- a/offload/plugins-nextgen/common/src/PluginInterface.cpp +++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp @@ -748,8 +748,7 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) { if (ompt::Initialized) { bool ExpectedStatus = false; if (OmptInitialized.compare_exchange_strong(ExpectedStatus, true)) - performOmptCallback(device_initialize, /*device_num=*/DeviceId + - Plugin.getDeviceIdStartIndex(), + performOmptCallback(device_initialize, Plugin.getUserId(DeviceId), /*type=*/getComputeUnitKind().c_str(), /*device=*/reinterpret_cast(this), /*lookup=*/ompt::lookupCallbackByName, @@ -847,9 +846,7 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) { if (ompt::Initialized) { bool ExpectedStatus = true; if (OmptInitialized.compare_exchange_strong(ExpectedStatus, false)) - performOmptCallback(device_finalize, - /*device_num=*/DeviceId + - Plugin.getDeviceIdStartIndex()); + performOmptCallback(device_finalize, Plugin.getUserId(DeviceId)); } #endif @@ -908,7 +905,7 @@ GenericDeviceTy::loadBinary(GenericPluginTy &Plugin, size_t Bytes = getPtrDiff(InputTgtImage->ImageEnd, InputTgtImage->ImageStart); performOmptCallback( - device_load, /*device_num=*/DeviceId + Plugin.getDeviceIdStartIndex(), + device_load, Plugin.getUserId(DeviceId), /*FileName=*/nullptr, /*FileOffset=*/0, /*VmaInFile=*/nullptr, /*ImgSize=*/Bytes, /*HostAddr=*/InputTgtImage->ImageStart, /*DeviceAddr=*/nullptr, /* FIXME: ModuleId */ 0); @@ -1492,11 +1489,14 @@ Error GenericDeviceTy::syncEvent(void *EventPtr) { bool GenericDeviceTy::useAutoZeroCopy() { return useAutoZeroCopyImpl(); } Error GenericPluginTy::init() { + if (Initialized) + return Plugin::success(); + auto NumDevicesOrErr = initImpl(); if (!NumDevicesOrErr) return NumDevicesOrErr.takeError(); - Initialized = true; + NumDevices = *NumDevicesOrErr; if (NumDevices == 0) return Plugin::success(); @@ -1517,6 +1517,8 @@ Error GenericPluginTy::init() { } Error GenericPluginTy::deinit() { + assert(Initialized && "Plugin was not initialized!"); + // Deinitialize all active devices. for (int32_t DeviceId = 0; DeviceId < NumDevices; ++DeviceId) { if (Devices[DeviceId]) { @@ -1537,7 +1539,11 @@ Error GenericPluginTy::deinit() { delete RecordReplay; // Perform last deinitializations on the plugin. - return deinitImpl(); + if (Error Err = deinitImpl()) + return Err; + Initialized = false; + + return Plugin::success(); } Error GenericPluginTy::initDevice(int32_t DeviceId) { @@ -1599,8 +1605,7 @@ Expected GenericPluginTy::checkBitcodeImage(StringRef Image) const { int32_t GenericPluginTy::is_initialized() const { return Initialized; } -int32_t GenericPluginTy::is_valid_binary(__tgt_device_image *Image, - bool Initialized) { +int32_t GenericPluginTy::is_plugin_compatible(__tgt_device_image *Image) { StringRef Buffer(reinterpret_cast(Image->ImageStart), target::getPtrDiff(Image->ImageEnd, Image->ImageStart)); @@ -1618,11 +1623,43 @@ int32_t GenericPluginTy::is_valid_binary(__tgt_device_image *Image, auto MatchOrErr = checkELFImage(Buffer); if (Error Err = MatchOrErr.takeError()) return HandleError(std::move(Err)); - if (!Initialized || !*MatchOrErr) - return *MatchOrErr; + return *MatchOrErr; + } + case file_magic::bitcode: { + auto MatchOrErr = checkBitcodeImage(Buffer); + if (Error Err = MatchOrErr.takeError()) + return HandleError(std::move(Err)); + return *MatchOrErr; + } + default: + return false; + } +} + +int32_t GenericPluginTy::is_device_compatible(int32_t DeviceId, + __tgt_device_image *Image) { + StringRef Buffer(reinterpret_cast(Image->ImageStart), + target::getPtrDiff(Image->ImageEnd, Image->ImageStart)); + + auto HandleError = [&](Error Err) -> bool { + [[maybe_unused]] std::string ErrStr = toString(std::move(Err)); + DP("Failure to check validity of image %p: %s", Image, ErrStr.c_str()); + return false; + }; + switch (identify_magic(Buffer)) { + case file_magic::elf: + case file_magic::elf_relocatable: + case file_magic::elf_executable: + case file_magic::elf_shared_object: + case file_magic::elf_core: { + auto MatchOrErr = checkELFImage(Buffer); + if (Error Err = MatchOrErr.takeError()) + return HandleError(std::move(Err)); + if (!*MatchOrErr) + return false; // Perform plugin-dependent checks for the specific architecture if needed. - auto CompatibleOrErr = isELFCompatible(Buffer); + auto CompatibleOrErr = isELFCompatible(DeviceId, Buffer); if (Error Err = CompatibleOrErr.takeError()) return HandleError(std::move(Err)); return *CompatibleOrErr; @@ -1638,6 +1675,10 @@ int32_t GenericPluginTy::is_valid_binary(__tgt_device_image *Image, } } +int32_t GenericPluginTy::is_device_initialized(int32_t DeviceId) const { + return isValidDeviceId(DeviceId) && Devices[DeviceId] != nullptr; +} + int32_t GenericPluginTy::init_device(int32_t DeviceId) { auto Err = initDevice(DeviceId); if (Err) { @@ -1985,8 +2026,9 @@ int32_t GenericPluginTy::init_device_info(int32_t DeviceId, return OFFLOAD_SUCCESS; } -int32_t GenericPluginTy::set_device_offset(int32_t DeviceIdOffset) { - setDeviceIdStartIndex(DeviceIdOffset); +int32_t GenericPluginTy::set_device_identifier(int32_t UserId, + int32_t DeviceId) { + UserDeviceIds[DeviceId] = UserId; return OFFLOAD_SUCCESS; } diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp index b260334baa18..62460c07415b 100644 --- a/offload/plugins-nextgen/cuda/src/rtl.cpp +++ b/offload/plugins-nextgen/cuda/src/rtl.cpp @@ -1388,8 +1388,9 @@ struct CUDAPluginTy final : public GenericPluginTy { const char *getName() const override { return GETNAME(TARGET_NAME); } - /// Check whether the image is compatible with the available CUDA devices. - Expected isELFCompatible(StringRef Image) const override { + /// Check whether the image is compatible with a CUDA device. + Expected isELFCompatible(uint32_t DeviceId, + StringRef Image) const override { auto ElfOrErr = ELF64LEObjectFile::create(MemoryBufferRef(Image, /*Identifier=*/""), /*InitContent=*/false); @@ -1399,33 +1400,29 @@ struct CUDAPluginTy final : public GenericPluginTy { // Get the numeric value for the image's `sm_` value. auto SM = ElfOrErr->getPlatformFlags() & ELF::EF_CUDA_SM; - for (int32_t DevId = 0; DevId < getNumDevices(); ++DevId) { - CUdevice Device; - CUresult Res = cuDeviceGet(&Device, DevId); - if (auto Err = Plugin::check(Res, "Error in cuDeviceGet: %s")) - return std::move(Err); - - int32_t Major, Minor; - Res = cuDeviceGetAttribute( - &Major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, Device); - if (auto Err = Plugin::check(Res, "Error in cuDeviceGetAttribute: %s")) - return std::move(Err); - - Res = cuDeviceGetAttribute( - &Minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, Device); - if (auto Err = Plugin::check(Res, "Error in cuDeviceGetAttribute: %s")) - return std::move(Err); - - int32_t ImageMajor = SM / 10; - int32_t ImageMinor = SM % 10; - - // A cubin generated for a certain compute capability is supported to - // run on any GPU with the same major revision and same or higher minor - // revision. - if (Major != ImageMajor || Minor < ImageMinor) - return false; - } - return true; + CUdevice Device; + CUresult Res = cuDeviceGet(&Device, DeviceId); + if (auto Err = Plugin::check(Res, "Error in cuDeviceGet: %s")) + return std::move(Err); + + int32_t Major, Minor; + Res = cuDeviceGetAttribute( + &Major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, Device); + if (auto Err = Plugin::check(Res, "Error in cuDeviceGetAttribute: %s")) + return std::move(Err); + + Res = cuDeviceGetAttribute( + &Minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, Device); + if (auto Err = Plugin::check(Res, "Error in cuDeviceGetAttribute: %s")) + return std::move(Err); + + int32_t ImageMajor = SM / 10; + int32_t ImageMinor = SM % 10; + + // A cubin generated for a certain compute capability is supported to + // run on any GPU with the same major revision and same or higher minor + // revision. + return Major == ImageMajor && Minor >= ImageMinor; } }; diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp index ef84cbaf5458..aa59ea618e39 100644 --- a/offload/plugins-nextgen/host/src/rtl.cpp +++ b/offload/plugins-nextgen/host/src/rtl.cpp @@ -418,7 +418,9 @@ struct GenELF64PluginTy final : public GenericPluginTy { } /// All images (ELF-compatible) should be compatible with this plugin. - Expected isELFCompatible(StringRef) const override { return true; } + Expected isELFCompatible(uint32_t, StringRef) const override { + return true; + } Triple::ArchType getTripleArch() const override { #if defined(__x86_64__) diff --git a/offload/src/PluginManager.cpp b/offload/src/PluginManager.cpp index 13f08b142b87..5e8f91792a55 100644 --- a/offload/src/PluginManager.cpp +++ b/offload/src/PluginManager.cpp @@ -47,6 +47,9 @@ void PluginManager::deinit() { DP("Unloading RTLs...\n"); for (auto &Plugin : Plugins) { + if (!Plugin->is_initialized()) + continue; + if (auto Err = Plugin->deinit()) { [[maybe_unused]] std::string InfoMsg = toString(std::move(Err)); DP("Failed to deinit plugin: %s\n", InfoMsg.c_str()); @@ -57,90 +60,15 @@ void PluginManager::deinit() { DP("RTLs unloaded!\n"); } -void PluginManager::initDevices(GenericPluginTy &RTL) { - // If this RTL has already been initialized. - if (PM->DeviceOffsets.contains(&RTL)) - return; - TIMESCOPE(); - - // If this RTL is not already in use, initialize it. - assert(RTL.number_of_devices() > 0 && "Tried to initialize useless plugin!"); - - // Initialize the device information for the RTL we are about to use. - auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor(); - - // Initialize the index of this RTL and save it in the used RTLs. - int32_t DeviceOffset = ExclusiveDevicesAccessor->size(); - - // Set the device identifier offset in the plugin. - RTL.set_device_offset(DeviceOffset); - - int32_t NumberOfUserDevices = 0; - int32_t NumPD = RTL.number_of_devices(); - ExclusiveDevicesAccessor->reserve(DeviceOffset + NumPD); - // Auto zero-copy is a per-device property. We need to ensure - // that all devices are suggesting to use it. - bool UseAutoZeroCopy = !(NumPD == 0); - for (int32_t PDevI = 0, UserDevId = DeviceOffset; PDevI < NumPD; PDevI++) { - auto Device = std::make_unique(&RTL, UserDevId, PDevI); - if (auto Err = Device->init()) { - DP("Skip plugin known device %d: %s\n", PDevI, - toString(std::move(Err)).c_str()); - continue; - } - UseAutoZeroCopy = UseAutoZeroCopy && Device->useAutoZeroCopy(); - - ExclusiveDevicesAccessor->push_back(std::move(Device)); - ++NumberOfUserDevices; - ++UserDevId; - } - - // Auto Zero-Copy can only be currently triggered when the system is an - // homogeneous APU architecture without attached discrete GPUs. - // If all devices suggest to use it, change requirment flags to trigger - // zero-copy behavior when mapping memory. - if (UseAutoZeroCopy) - addRequirements(OMPX_REQ_AUTO_ZERO_COPY); - - DeviceOffsets[&RTL] = DeviceOffset; - DeviceUsed[&RTL] = NumberOfUserDevices; - DP("Plugin has index %d, exposes %d out of %d devices!\n", DeviceOffset, - NumberOfUserDevices, RTL.number_of_devices()); -} - void PluginManager::initAllPlugins() { - for (auto &R : Plugins) - initDevices(*R); -} - -static void registerImageIntoTranslationTable(TranslationTable &TT, - int32_t DeviceOffset, - int32_t NumberOfUserDevices, - __tgt_device_image *Image) { - - // same size, as when we increase one, we also increase the other. - assert(TT.TargetsTable.size() == TT.TargetsImages.size() && - "We should have as many images as we have tables!"); - - // Resize the Targets Table and Images to accommodate the new targets if - // required - unsigned TargetsTableMinimumSize = DeviceOffset + NumberOfUserDevices; - - if (TT.TargetsTable.size() < TargetsTableMinimumSize) { - TT.DeviceTables.resize(TargetsTableMinimumSize, {}); - TT.TargetsImages.resize(TargetsTableMinimumSize, 0); - TT.TargetsEntries.resize(TargetsTableMinimumSize, {}); - TT.TargetsTable.resize(TargetsTableMinimumSize, 0); - } - - // Register the image in all devices for this target type. - for (int32_t I = 0; I < NumberOfUserDevices; ++I) { - // If we are changing the image we are also invalidating the target table. - if (TT.TargetsImages[DeviceOffset + I] != Image) { - TT.TargetsImages[DeviceOffset + I] = Image; - TT.TargetsTable[DeviceOffset + I] = - 0; // lazy initialization of target table. + for (auto &R : plugins()) { + if (auto Err = R.init()) { + [[maybe_unused]] std::string InfoMsg = toString(std::move(Err)); + DP("Failed to init plugin: %s\n", InfoMsg.c_str()); + continue; } + DP("Registered plugin %s with %d visible device(s)\n", R.getName(), + R.number_of_devices()); } } @@ -153,27 +81,6 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) { if (Entry.flags == OMP_REGISTER_REQUIRES) PM->addRequirements(Entry.data); - // Initialize all the plugins that have associated images. - for (auto &Plugin : Plugins) { - // Extract the exectuable image and extra information if availible. - for (int32_t i = 0; i < Desc->NumDeviceImages; ++i) { - if (Plugin->is_initialized()) - continue; - - if (!Plugin->is_valid_binary(&Desc->DeviceImages[i], - /*Initialized=*/false)) - continue; - - if (auto Err = Plugin->init()) { - [[maybe_unused]] std::string InfoMsg = toString(std::move(Err)); - DP("Failed to init plugin: %s\n", InfoMsg.c_str()); - } else { - DP("Registered plugin %s with %d visible device(s)\n", - Plugin->getName(), Plugin->number_of_devices()); - } - } - } - // Extract the exectuable image and extra information if availible. for (int32_t i = 0; i < Desc->NumDeviceImages; ++i) PM->addDeviceImage(*Desc, Desc->DeviceImages[i]); @@ -188,54 +95,110 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) { // Scan the RTLs that have associated images until we find one that supports // the current image. for (auto &R : PM->plugins()) { - if (!R.number_of_devices()) + if (!R.is_plugin_compatible(Img)) continue; - if (!R.is_valid_binary(Img, /*Initialized=*/true)) { - DP("Image " DPxMOD " is NOT compatible with RTL %s!\n", - DPxPTR(Img->ImageStart), R.getName()); - continue; + if (!R.is_initialized()) { + if (auto Err = R.init()) { + [[maybe_unused]] std::string InfoMsg = toString(std::move(Err)); + DP("Failed to init plugin: %s\n", InfoMsg.c_str()); + continue; + } + DP("Registered plugin %s with %d visible device(s)\n", R.getName(), + R.number_of_devices()); } - DP("Image " DPxMOD " is compatible with RTL %s!\n", - DPxPTR(Img->ImageStart), R.getName()); - - PM->initDevices(R); + if (!R.number_of_devices()) { + DP("Skipping plugin %s with no visible devices\n", R.getName()); + continue; + } - // Initialize (if necessary) translation table for this library. - PM->TrlTblMtx.lock(); - if (!PM->HostEntriesBeginToTransTable.count(Desc->HostEntriesBegin)) { - PM->HostEntriesBeginRegistrationOrder.push_back(Desc->HostEntriesBegin); - TranslationTable &TransTable = + for (int32_t DeviceId = 0; DeviceId < R.number_of_devices(); ++DeviceId) { + if (!R.is_device_compatible(DeviceId, Img)) + continue; + + DP("Image " DPxMOD " is compatible with RTL %s device %d!\n", + DPxPTR(Img->ImageStart), R.getName(), DeviceId); + + if (!R.is_device_initialized(DeviceId)) { + // Initialize the device information for the RTL we are about to use. + auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor(); + + int32_t UserId = ExclusiveDevicesAccessor->size(); + + // Set the device identifier offset in the plugin. +#ifdef OMPT_SUPPORT + R.set_device_identifier(UserId, DeviceId); +#endif + + auto Device = std::make_unique(&R, UserId, DeviceId); + if (auto Err = Device->init()) { + [[maybe_unused]] std::string InfoMsg = toString(std::move(Err)); + DP("Failed to init device %d: %s\n", DeviceId, InfoMsg.c_str()); + continue; + } + + ExclusiveDevicesAccessor->push_back(std::move(Device)); + + // We need to map between the plugin's device identifier and the one + // that OpenMP will use. + PM->DeviceIds[std::make_pair(&R, DeviceId)] = UserId; + } + + // Initialize (if necessary) translation table for this library. + PM->TrlTblMtx.lock(); + if (!PM->HostEntriesBeginToTransTable.count(Desc->HostEntriesBegin)) { + PM->HostEntriesBeginRegistrationOrder.push_back( + Desc->HostEntriesBegin); + TranslationTable &TT = + (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin]; + TT.HostTable.EntriesBegin = Desc->HostEntriesBegin; + TT.HostTable.EntriesEnd = Desc->HostEntriesEnd; + } + + // Retrieve translation table for this library. + TranslationTable &TT = (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin]; - TransTable.HostTable.EntriesBegin = Desc->HostEntriesBegin; - TransTable.HostTable.EntriesEnd = Desc->HostEntriesEnd; - } - // Retrieve translation table for this library. - TranslationTable &TransTable = - (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin]; + DP("Registering image " DPxMOD " with RTL %s!\n", + DPxPTR(Img->ImageStart), R.getName()); - DP("Registering image " DPxMOD " with RTL %s!\n", DPxPTR(Img->ImageStart), - R.getName()); + auto UserId = PM->DeviceIds[std::make_pair(&R, DeviceId)]; + if (TT.TargetsTable.size() < static_cast(UserId + 1)) { + TT.DeviceTables.resize(UserId + 1, {}); + TT.TargetsImages.resize(UserId + 1, nullptr); + TT.TargetsEntries.resize(UserId + 1, {}); + TT.TargetsTable.resize(UserId + 1, nullptr); + } - registerImageIntoTranslationTable(TransTable, PM->DeviceOffsets[&R], - PM->DeviceUsed[&R], Img); - PM->UsedImages.insert(Img); + // Register the image for this target type and invalidate the table. + TT.TargetsImages[UserId] = Img; + TT.TargetsTable[UserId] = nullptr; - PM->TrlTblMtx.unlock(); - FoundRTL = &R; + PM->UsedImages.insert(Img); + FoundRTL = &R; - // if an RTL was found we are done - proceed to register the next image - break; + PM->TrlTblMtx.unlock(); + } } - - if (!FoundRTL) { + if (!FoundRTL) DP("No RTL found for image " DPxMOD "!\n", DPxPTR(Img->ImageStart)); - } } PM->RTLsMtx.unlock(); + bool UseAutoZeroCopy = Plugins.size() > 0; + + auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor(); + for (const auto &Device : *ExclusiveDevicesAccessor) + UseAutoZeroCopy &= Device->useAutoZeroCopy(); + + // Auto Zero-Copy can only be currently triggered when the system is an + // homogeneous APU architecture without attached discrete GPUs. + // If all devices suggest to use it, change requirment flags to trigger + // zero-copy behavior when mapping memory. + if (UseAutoZeroCopy) + addRequirements(OMPX_REQ_AUTO_ZERO_COPY); + DP("Done registering entries!\n"); } @@ -257,7 +220,7 @@ void PluginManager::unregisterLib(__tgt_bin_desc *Desc) { // Scan the RTLs that have associated images until we find one that supports // the current image. We only need to scan RTLs that are already being used. for (auto &R : PM->plugins()) { - if (!DeviceOffsets.contains(&R)) + if (R.is_initialized()) continue; // Ensure that we do not use any unused images associated with this RTL. diff --git a/offload/src/omptarget.cpp b/offload/src/omptarget.cpp index 91e1213f175e..9bca8529c5ee 100644 --- a/offload/src/omptarget.cpp +++ b/offload/src/omptarget.cpp @@ -315,7 +315,7 @@ void handleTargetOutcome(bool Success, ident_t *Loc) { FAILURE_MESSAGE("Consult https://openmp.llvm.org/design/Runtimes.html " "for debugging options.\n"); - if (!PM->getNumUsedPlugins()) { + if (!PM->getNumActivePlugins()) { FAILURE_MESSAGE( "No images found compatible with the installed hardware. "); -- GitLab From 24a39f364dbef7e18d36be3919eacde32125df5e Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 6 Jun 2024 14:09:16 +0100 Subject: [PATCH 090/462] [X86] Fix pipe resources for HADD/SUB instructions IceLakeServer was copying these from SkylakeServer, but integer HADD/SUB can now run on an extra port --- llvm/lib/Target/X86/X86SchedIceLake.td | 14 +++--- .../X86/IceLakeServer/resources-avx1.s | 50 +++++++++---------- .../X86/IceLakeServer/resources-avx2.s | 50 +++++++++---------- .../X86/IceLakeServer/resources-ssse3.s | 50 +++++++++---------- 4 files changed, 82 insertions(+), 82 deletions(-) diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td index 2d296771b1c0..5c3f5f61c896 100644 --- a/llvm/lib/Target/X86/X86SchedIceLake.td +++ b/llvm/lib/Target/X86/X86SchedIceLake.td @@ -623,8 +623,8 @@ def : WriteRes; defm : ICXWriteResPair; defm : ICXWriteResPair; defm : ICXWriteResPair; -defm : ICXWriteResPair; -defm : ICXWriteResPair; +defm : ICXWriteResPair; +defm : ICXWriteResPair; // Remaining instrs. @@ -886,7 +886,7 @@ def ICXWriteResGroup37 : SchedWriteRes<[ICXPort0,ICXPort5]> { } def: InstRW<[ICXWriteResGroup37], (instregex "MMX_PH(ADD|SUB)SWrr")>; -def ICXWriteResGroup38 : SchedWriteRes<[ICXPort5,ICXPort01]> { +def ICXWriteResGroup38 : SchedWriteRes<[ICXPort15,ICXPort01]> { let Latency = 3; let NumMicroOps = 3; let ReleaseAtCycles = [2,1]; @@ -1739,13 +1739,13 @@ def ICXWriteResGroup137 : SchedWriteRes<[ICXPort23,ICXPort01]> { def: InstRW<[ICXWriteResGroup137], (instregex "MMX_CVT(T?)PS2PIrm", "(V?)CVTPS2PDrm")>; -def ICXWriteResGroup143 : SchedWriteRes<[ICXPort5,ICXPort01,ICXPort23]> { +def ICXWriteResGroup143 : SchedWriteRes<[ICXPort15,ICXPort01,ICXPort23]> { let Latency = 9; let NumMicroOps = 4; let ReleaseAtCycles = [2,1,1]; } -def: InstRW<[ICXWriteResGroup143], (instregex "(V?)PHADDSWrm", - "(V?)PHSUBSWrm")>; +def: InstRW<[ICXWriteResGroup143], (instrs PHADDSWrm, VPHADDSWrm, + PHSUBSWrm, VPHSUBSWrm)>; def ICXWriteResGroup146 : SchedWriteRes<[ICXPort1,ICXPort6,ICXPort23,ICXPort0156]> { let Latency = 9; @@ -1842,7 +1842,7 @@ def: InstRW<[ICXWriteResGroup151], (instregex "VEXPANDPDZ128rm(b?)", "VPEXPANDDZ128rm(b?)", "VPEXPANDQZ128rm(b?)")>; -def ICXWriteResGroup154 : SchedWriteRes<[ICXPort5,ICXPort01,ICXPort23]> { +def ICXWriteResGroup154 : SchedWriteRes<[ICXPort15,ICXPort01,ICXPort23]> { let Latency = 10; let NumMicroOps = 4; let ReleaseAtCycles = [2,1,1]; diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s index bd7a4894b45d..76bc4ecc9326 100644 --- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s +++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s @@ -1455,20 +1455,20 @@ vzeroupper # CHECK-NEXT: 3 2 1.00 * vpextrq $1, %xmm0, (%rax) # CHECK-NEXT: 2 3 1.00 vpextrw $1, %xmm0, %ecx # CHECK-NEXT: 3 2 1.00 * vpextrw $1, %xmm0, (%rax) -# CHECK-NEXT: 3 3 2.00 vphaddd %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: 4 9 2.00 * vphaddd (%rax), %xmm1, %xmm2 -# CHECK-NEXT: 3 3 2.00 vphaddsw %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: 4 9 2.00 * vphaddsw (%rax), %xmm1, %xmm2 -# CHECK-NEXT: 3 3 2.00 vphaddw %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: 4 9 2.00 * vphaddw (%rax), %xmm1, %xmm2 +# CHECK-NEXT: 3 3 1.00 vphaddd %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: 4 9 1.00 * vphaddd (%rax), %xmm1, %xmm2 +# CHECK-NEXT: 3 3 1.00 vphaddsw %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: 4 9 1.00 * vphaddsw (%rax), %xmm1, %xmm2 +# CHECK-NEXT: 3 3 1.00 vphaddw %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: 4 9 1.00 * vphaddw (%rax), %xmm1, %xmm2 # CHECK-NEXT: 1 4 1.00 vphminposuw %xmm0, %xmm2 # CHECK-NEXT: 2 10 1.00 * vphminposuw (%rax), %xmm2 -# CHECK-NEXT: 3 3 2.00 vphsubd %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: 4 9 2.00 * vphsubd (%rax), %xmm1, %xmm2 -# CHECK-NEXT: 3 3 2.00 vphsubsw %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: 4 9 2.00 * vphsubsw (%rax), %xmm1, %xmm2 -# CHECK-NEXT: 3 3 2.00 vphsubw %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: 4 9 2.00 * vphsubw (%rax), %xmm1, %xmm2 +# CHECK-NEXT: 3 3 1.00 vphsubd %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: 4 9 1.00 * vphsubd (%rax), %xmm1, %xmm2 +# CHECK-NEXT: 3 3 1.00 vphsubsw %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: 4 9 1.00 * vphsubsw (%rax), %xmm1, %xmm2 +# CHECK-NEXT: 3 3 1.00 vphsubw %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: 4 9 1.00 * vphsubw (%rax), %xmm1, %xmm2 # CHECK-NEXT: 2 2 2.00 vpinsrb $1, %eax, %xmm1, %xmm2 # CHECK-NEXT: 2 6 1.00 * vpinsrb $1, (%rax), %xmm1, %xmm2 # CHECK-NEXT: 2 2 2.00 vpinsrd $1, %eax, %xmm1, %xmm2 @@ -1738,7 +1738,7 @@ vzeroupper # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] -# CHECK-NEXT: - 126.00 322.92 237.92 160.50 160.50 19.00 291.92 6.25 19.00 19.00 19.00 +# CHECK-NEXT: - 126.00 322.92 249.92 160.50 160.50 19.00 279.92 6.25 19.00 19.00 19.00 # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] Instructions: @@ -2167,20 +2167,20 @@ vzeroupper # CHECK-NEXT: - - - - - - 0.50 1.00 - 0.50 0.50 0.50 vpextrq $1, %xmm0, (%rax) # CHECK-NEXT: - - 1.00 - - - - 1.00 - - - - vpextrw $1, %xmm0, %ecx # CHECK-NEXT: - - - - - - 0.50 1.00 - 0.50 0.50 0.50 vpextrw $1, %xmm0, (%rax) -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vphaddd %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vphaddd (%rax), %xmm1, %xmm2 -# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - vphaddsw %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - vphaddsw (%rax), %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vphaddw %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vphaddw (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.33 1.33 - - - 1.33 - - - - vphaddd %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.33 1.33 0.50 0.50 - 1.33 - - - - vphaddd (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 1.50 - - - 1.00 - - - - vphaddsw %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 1.50 0.50 0.50 - 1.00 - - - - vphaddsw (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.33 1.33 - - - 1.33 - - - - vphaddw %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.33 1.33 0.50 0.50 - 1.33 - - - - vphaddw (%rax), %xmm1, %xmm2 # CHECK-NEXT: - - 1.00 - - - - - - - - - vphminposuw %xmm0, %xmm2 # CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vphminposuw (%rax), %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vphsubd %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vphsubd (%rax), %xmm1, %xmm2 -# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - vphsubsw %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - vphsubsw (%rax), %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vphsubw %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vphsubw (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.33 1.33 - - - 1.33 - - - - vphsubd %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.33 1.33 0.50 0.50 - 1.33 - - - - vphsubd (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 1.50 - - - 1.00 - - - - vphsubsw %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 1.50 0.50 0.50 - 1.00 - - - - vphsubsw (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.33 1.33 - - - 1.33 - - - - vphsubw %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.33 1.33 0.50 0.50 - 1.33 - - - - vphsubw (%rax), %xmm1, %xmm2 # CHECK-NEXT: - - - - - - - 2.00 - - - - vpinsrb $1, %eax, %xmm1, %xmm2 # CHECK-NEXT: - - - - 0.50 0.50 - 1.00 - - - - vpinsrb $1, (%rax), %xmm1, %xmm2 # CHECK-NEXT: - - - - - - - 2.00 - - - - vpinsrd $1, %eax, %xmm1, %xmm2 diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s index dcf883445ba4..c251dc349587 100644 --- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s +++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s @@ -576,18 +576,18 @@ vpxor (%rax), %ymm1, %ymm2 # CHECK-NEXT: 5 20 2.00 * vpgatherqd %xmm0, (%rax,%ymm1,2), %xmm2 # CHECK-NEXT: 5 18 1.00 * vpgatherqq %xmm0, (%rax,%xmm1,2), %xmm2 # CHECK-NEXT: 5 20 2.00 * vpgatherqq %ymm0, (%rax,%ymm1,2), %ymm2 -# CHECK-NEXT: 3 3 2.00 vphaddd %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: 4 10 2.00 * vphaddd (%rax), %ymm1, %ymm2 -# CHECK-NEXT: 3 3 2.00 vphaddsw %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: 4 10 2.00 * vphaddsw (%rax), %ymm1, %ymm2 -# CHECK-NEXT: 3 3 2.00 vphaddw %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: 4 10 2.00 * vphaddw (%rax), %ymm1, %ymm2 -# CHECK-NEXT: 3 3 2.00 vphsubd %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: 4 10 2.00 * vphsubd (%rax), %ymm1, %ymm2 -# CHECK-NEXT: 3 3 2.00 vphsubsw %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: 4 10 2.00 * vphsubsw (%rax), %ymm1, %ymm2 -# CHECK-NEXT: 3 3 2.00 vphsubw %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: 4 10 2.00 * vphsubw (%rax), %ymm1, %ymm2 +# CHECK-NEXT: 3 3 1.00 vphaddd %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: 4 10 1.00 * vphaddd (%rax), %ymm1, %ymm2 +# CHECK-NEXT: 3 3 1.00 vphaddsw %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: 4 10 1.00 * vphaddsw (%rax), %ymm1, %ymm2 +# CHECK-NEXT: 3 3 1.00 vphaddw %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: 4 10 1.00 * vphaddw (%rax), %ymm1, %ymm2 +# CHECK-NEXT: 3 3 1.00 vphsubd %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: 4 10 1.00 * vphsubd (%rax), %ymm1, %ymm2 +# CHECK-NEXT: 3 3 1.00 vphsubsw %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: 4 10 1.00 * vphsubsw (%rax), %ymm1, %ymm2 +# CHECK-NEXT: 3 3 1.00 vphsubw %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: 4 10 1.00 * vphsubw (%rax), %ymm1, %ymm2 # CHECK-NEXT: 1 5 0.50 vpmaddubsw %ymm0, %ymm1, %ymm2 # CHECK-NEXT: 2 12 0.50 * vpmaddubsw (%rax), %ymm1, %ymm2 # CHECK-NEXT: 1 5 0.50 vpmaddwd %ymm0, %ymm1, %ymm2 @@ -778,7 +778,7 @@ vpxor (%rax), %ymm1, %ymm2 # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] -# CHECK-NEXT: - - 110.33 104.33 98.00 98.00 2.50 149.33 - 2.50 2.50 2.50 +# CHECK-NEXT: - - 110.33 116.33 98.00 98.00 2.50 137.33 - 2.50 2.50 2.50 # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] Instructions: @@ -898,18 +898,18 @@ vpxor (%rax), %ymm1, %ymm2 # CHECK-NEXT: - - 1.33 0.33 2.00 2.00 - 1.33 - - - - vpgatherqd %xmm0, (%rax,%ymm1,2), %xmm2 # CHECK-NEXT: - - 1.33 0.33 1.00 1.00 - 1.33 - - - - vpgatherqq %xmm0, (%rax,%xmm1,2), %xmm2 # CHECK-NEXT: - - 1.33 0.33 2.00 2.00 - 1.33 - - - - vpgatherqq %ymm0, (%rax,%ymm1,2), %ymm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vphaddd %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vphaddd (%rax), %ymm1, %ymm2 -# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - vphaddsw %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - vphaddsw (%rax), %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vphaddw %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vphaddw (%rax), %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vphsubd %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vphsubd (%rax), %ymm1, %ymm2 -# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - vphsubsw %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - vphsubsw (%rax), %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vphsubw %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vphsubw (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.33 1.33 - - - 1.33 - - - - vphaddd %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.33 1.33 0.50 0.50 - 1.33 - - - - vphaddd (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 1.50 - - - 1.00 - - - - vphaddsw %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 1.50 0.50 0.50 - 1.00 - - - - vphaddsw (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.33 1.33 - - - 1.33 - - - - vphaddw %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.33 1.33 0.50 0.50 - 1.33 - - - - vphaddw (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.33 1.33 - - - 1.33 - - - - vphsubd %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.33 1.33 0.50 0.50 - 1.33 - - - - vphsubd (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 1.50 - - - 1.00 - - - - vphsubsw %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 1.50 0.50 0.50 - 1.00 - - - - vphsubsw (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.33 1.33 - - - 1.33 - - - - vphsubw %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.33 1.33 0.50 0.50 - 1.33 - - - - vphsubw (%rax), %ymm1, %ymm2 # CHECK-NEXT: - - 0.50 0.50 - - - - - - - - vpmaddubsw %ymm0, %ymm1, %ymm2 # CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - - - - - - vpmaddubsw (%rax), %ymm1, %ymm2 # CHECK-NEXT: - - 0.50 0.50 - - - - - - - - vpmaddwd %ymm0, %ymm1, %ymm2 diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-ssse3.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-ssse3.s index 3a6668cedb60..d034cbd0f639 100644 --- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-ssse3.s +++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-ssse3.s @@ -124,28 +124,28 @@ psignw (%rax), %xmm2 # CHECK-NEXT: 2 7 1.00 * palignr $1, (%rax), %xmm2 # CHECK-NEXT: 3 3 2.00 phaddd %mm0, %mm2 # CHECK-NEXT: 4 8 2.00 * phaddd (%rax), %mm2 -# CHECK-NEXT: 3 3 2.00 phaddd %xmm0, %xmm2 -# CHECK-NEXT: 4 9 2.00 * phaddd (%rax), %xmm2 +# CHECK-NEXT: 3 3 1.00 phaddd %xmm0, %xmm2 +# CHECK-NEXT: 4 9 1.00 * phaddd (%rax), %xmm2 # CHECK-NEXT: 3 3 2.00 phaddsw %mm0, %mm2 # CHECK-NEXT: 4 8 2.00 * phaddsw (%rax), %mm2 -# CHECK-NEXT: 3 3 2.00 phaddsw %xmm0, %xmm2 -# CHECK-NEXT: 4 9 2.00 * phaddsw (%rax), %xmm2 +# CHECK-NEXT: 3 3 1.00 phaddsw %xmm0, %xmm2 +# CHECK-NEXT: 4 9 1.00 * phaddsw (%rax), %xmm2 # CHECK-NEXT: 3 3 2.00 phaddw %mm0, %mm2 # CHECK-NEXT: 4 8 2.00 * phaddw (%rax), %mm2 -# CHECK-NEXT: 3 3 2.00 phaddw %xmm0, %xmm2 -# CHECK-NEXT: 4 9 2.00 * phaddw (%rax), %xmm2 +# CHECK-NEXT: 3 3 1.00 phaddw %xmm0, %xmm2 +# CHECK-NEXT: 4 9 1.00 * phaddw (%rax), %xmm2 # CHECK-NEXT: 3 3 2.00 phsubd %mm0, %mm2 # CHECK-NEXT: 4 8 2.00 * phsubd (%rax), %mm2 -# CHECK-NEXT: 3 3 2.00 phsubd %xmm0, %xmm2 -# CHECK-NEXT: 4 9 2.00 * phsubd (%rax), %xmm2 +# CHECK-NEXT: 3 3 1.00 phsubd %xmm0, %xmm2 +# CHECK-NEXT: 4 9 1.00 * phsubd (%rax), %xmm2 # CHECK-NEXT: 3 3 2.00 phsubsw %mm0, %mm2 # CHECK-NEXT: 4 8 2.00 * phsubsw (%rax), %mm2 -# CHECK-NEXT: 3 3 2.00 phsubsw %xmm0, %xmm2 -# CHECK-NEXT: 4 9 2.00 * phsubsw (%rax), %xmm2 +# CHECK-NEXT: 3 3 1.00 phsubsw %xmm0, %xmm2 +# CHECK-NEXT: 4 9 1.00 * phsubsw (%rax), %xmm2 # CHECK-NEXT: 3 3 2.00 phsubw %mm0, %mm2 # CHECK-NEXT: 4 8 2.00 * phsubw (%rax), %mm2 -# CHECK-NEXT: 3 3 2.00 phsubw %xmm0, %xmm2 -# CHECK-NEXT: 4 9 2.00 * phsubw (%rax), %xmm2 +# CHECK-NEXT: 3 3 1.00 phsubw %xmm0, %xmm2 +# CHECK-NEXT: 4 9 1.00 * phsubw (%rax), %xmm2 # CHECK-NEXT: 1 5 1.00 pmaddubsw %mm0, %mm2 # CHECK-NEXT: 2 10 1.00 * pmaddubsw (%rax), %mm2 # CHECK-NEXT: 1 5 0.50 pmaddubsw %xmm0, %xmm2 @@ -187,7 +187,7 @@ psignw (%rax), %xmm2 # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] -# CHECK-NEXT: - - 30.67 13.67 16.00 16.00 - 67.67 - - - - +# CHECK-NEXT: - - 30.67 25.67 16.00 16.00 - 55.67 - - - - # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] Instructions: @@ -209,28 +209,28 @@ psignw (%rax), %xmm2 # CHECK-NEXT: - - - - 0.50 0.50 - 1.00 - - - - palignr $1, (%rax), %xmm2 # CHECK-NEXT: - - 0.50 - - - - 2.50 - - - - phaddd %mm0, %mm2 # CHECK-NEXT: - - 0.50 - 0.50 0.50 - 2.50 - - - - phaddd (%rax), %mm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - phaddd %xmm0, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - phaddd (%rax), %xmm2 +# CHECK-NEXT: - - 0.33 1.33 - - - 1.33 - - - - phaddd %xmm0, %xmm2 +# CHECK-NEXT: - - 0.33 1.33 0.50 0.50 - 1.33 - - - - phaddd (%rax), %xmm2 # CHECK-NEXT: - - 1.00 - - - - 2.00 - - - - phaddsw %mm0, %mm2 # CHECK-NEXT: - - 1.00 - 0.50 0.50 - 2.00 - - - - phaddsw (%rax), %mm2 -# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - phaddsw %xmm0, %xmm2 -# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - phaddsw (%rax), %xmm2 +# CHECK-NEXT: - - 0.50 1.50 - - - 1.00 - - - - phaddsw %xmm0, %xmm2 +# CHECK-NEXT: - - 0.50 1.50 0.50 0.50 - 1.00 - - - - phaddsw (%rax), %xmm2 # CHECK-NEXT: - - 0.50 - - - - 2.50 - - - - phaddw %mm0, %mm2 # CHECK-NEXT: - - 0.50 - 0.50 0.50 - 2.50 - - - - phaddw (%rax), %mm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - phaddw %xmm0, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - phaddw (%rax), %xmm2 +# CHECK-NEXT: - - 0.33 1.33 - - - 1.33 - - - - phaddw %xmm0, %xmm2 +# CHECK-NEXT: - - 0.33 1.33 0.50 0.50 - 1.33 - - - - phaddw (%rax), %xmm2 # CHECK-NEXT: - - 0.50 - - - - 2.50 - - - - phsubd %mm0, %mm2 # CHECK-NEXT: - - 0.50 - 0.50 0.50 - 2.50 - - - - phsubd (%rax), %mm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - phsubd %xmm0, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - phsubd (%rax), %xmm2 +# CHECK-NEXT: - - 0.33 1.33 - - - 1.33 - - - - phsubd %xmm0, %xmm2 +# CHECK-NEXT: - - 0.33 1.33 0.50 0.50 - 1.33 - - - - phsubd (%rax), %xmm2 # CHECK-NEXT: - - 1.00 - - - - 2.00 - - - - phsubsw %mm0, %mm2 # CHECK-NEXT: - - 1.00 - 0.50 0.50 - 2.00 - - - - phsubsw (%rax), %mm2 -# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - phsubsw %xmm0, %xmm2 -# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - phsubsw (%rax), %xmm2 +# CHECK-NEXT: - - 0.50 1.50 - - - 1.00 - - - - phsubsw %xmm0, %xmm2 +# CHECK-NEXT: - - 0.50 1.50 0.50 0.50 - 1.00 - - - - phsubsw (%rax), %xmm2 # CHECK-NEXT: - - 0.50 - - - - 2.50 - - - - phsubw %mm0, %mm2 # CHECK-NEXT: - - 0.50 - 0.50 0.50 - 2.50 - - - - phsubw (%rax), %mm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - phsubw %xmm0, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - phsubw (%rax), %xmm2 +# CHECK-NEXT: - - 0.33 1.33 - - - 1.33 - - - - phsubw %xmm0, %xmm2 +# CHECK-NEXT: - - 0.33 1.33 0.50 0.50 - 1.33 - - - - phsubw (%rax), %xmm2 # CHECK-NEXT: - - 1.00 - - - - - - - - - pmaddubsw %mm0, %mm2 # CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - pmaddubsw (%rax), %mm2 # CHECK-NEXT: - - 0.50 0.50 - - - - - - - - pmaddubsw %xmm0, %xmm2 -- GitLab From 62e2eb2154cac79db20012287823abbf35de8cb4 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 6 Jun 2024 14:16:57 +0100 Subject: [PATCH 091/462] [X86] Fix pipe resources for FP HADD/SUB instructions IceLakeServer/SkylakeServer can only use Port01 for the FADD/FSUB stage Confirmed with uops.info + Agner --- llvm/lib/Target/X86/X86SchedIceLake.td | 4 +-- llvm/lib/Target/X86/X86SchedSkylakeServer.td | 4 +-- .../X86/IceLakeServer/resources-avx1.s | 34 +++++++++---------- .../X86/IceLakeServer/resources-sse3.s | 18 +++++----- .../X86/SkylakeServer/resources-avx1.s | 34 +++++++++---------- .../X86/SkylakeServer/resources-sse3.s | 18 +++++----- 6 files changed, 56 insertions(+), 56 deletions(-) diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td index 5c3f5f61c896..186d4d84c251 100644 --- a/llvm/lib/Target/X86/X86SchedIceLake.td +++ b/llvm/lib/Target/X86/X86SchedIceLake.td @@ -620,8 +620,8 @@ def : WriteRes; // Horizontal add/sub instructions. //////////////////////////////////////////////////////////////////////////////// -defm : ICXWriteResPair; -defm : ICXWriteResPair; +defm : ICXWriteResPair; +defm : ICXWriteResPair; defm : ICXWriteResPair; defm : ICXWriteResPair; defm : ICXWriteResPair; diff --git a/llvm/lib/Target/X86/X86SchedSkylakeServer.td b/llvm/lib/Target/X86/X86SchedSkylakeServer.td index a7dff0ecbcd9..4fded44085e8 100644 --- a/llvm/lib/Target/X86/X86SchedSkylakeServer.td +++ b/llvm/lib/Target/X86/X86SchedSkylakeServer.td @@ -615,8 +615,8 @@ def : WriteRes; // Horizontal add/sub instructions. //////////////////////////////////////////////////////////////////////////////// -defm : SKXWriteResPair; -defm : SKXWriteResPair; +defm : SKXWriteResPair; +defm : SKXWriteResPair; defm : SKXWriteResPair; defm : SKXWriteResPair; defm : SKXWriteResPair; diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s index 76bc4ecc9326..7150a586fe6f 100644 --- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s +++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s @@ -1738,7 +1738,7 @@ vzeroupper # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] -# CHECK-NEXT: - 126.00 322.92 249.92 160.50 160.50 19.00 279.92 6.25 19.00 19.00 19.00 +# CHECK-NEXT: - 126.00 325.58 252.58 160.50 160.50 19.00 274.58 6.25 19.00 19.00 19.00 # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] Instructions: @@ -1908,22 +1908,22 @@ vzeroupper # CHECK-NEXT: - - - - - - 0.50 - - 0.50 0.50 0.50 vextractf128 $1, %ymm0, (%rax) # CHECK-NEXT: - - 1.00 - - - - 1.00 - - - - vextractps $1, %xmm0, %ecx # CHECK-NEXT: - - - - - - 0.50 1.00 - 0.50 0.50 0.50 vextractps $1, %xmm0, (%rax) -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vhaddpd %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vhaddpd (%rax), %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vhaddpd %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vhaddpd (%rax), %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vhaddps %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vhaddps (%rax), %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vhaddps %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vhaddps (%rax), %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vhsubpd %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vhsubpd (%rax), %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vhsubpd %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vhsubpd (%rax), %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vhsubps %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vhsubps (%rax), %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - vhsubps %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - vhsubps (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - vhaddpd %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - vhaddpd (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - vhaddpd %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - vhaddpd (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - vhaddps %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - vhaddps (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - vhaddps %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - vhaddps (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - vhsubpd %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - vhsubpd (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - vhsubpd %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - vhsubpd (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - vhsubps %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - vhsubps (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - vhsubps %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - vhsubps (%rax), %ymm1, %ymm2 # CHECK-NEXT: - - - - - - - 1.00 - - - - vinsertf128 $1, %xmm0, %ymm1, %ymm2 # CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 0.33 - - - - vinsertf128 $1, (%rax), %ymm1, %ymm2 # CHECK-NEXT: - - - - - - - 1.00 - - - - vinsertps $1, %xmm0, %xmm1, %xmm2 diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse3.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse3.s index 4d1942450ec6..0d075a9bc3bf 100644 --- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse3.s +++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse3.s @@ -81,7 +81,7 @@ mwait # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] -# CHECK-NEXT: - - 6.67 7.67 5.00 5.00 - 23.67 4.00 - - - +# CHECK-NEXT: - - 8.00 9.00 5.00 5.00 - 21.00 4.00 - - - # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] Instructions: @@ -89,14 +89,14 @@ mwait # CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - - - - - - addsubpd (%rax), %xmm2 # CHECK-NEXT: - - 0.50 0.50 - - - - - - - - addsubps %xmm0, %xmm2 # CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - - - - - - addsubps (%rax), %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - haddpd %xmm0, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - haddpd (%rax), %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - haddps %xmm0, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - haddps (%rax), %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - hsubpd %xmm0, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - hsubpd (%rax), %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - - - hsubps %xmm0, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - - - hsubps (%rax), %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - haddpd %xmm0, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - haddpd (%rax), %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - haddps %xmm0, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - haddps (%rax), %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - hsubpd %xmm0, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - hsubpd (%rax), %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - - - hsubps %xmm0, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - - - hsubps (%rax), %xmm2 # CHECK-NEXT: - - - - 0.50 0.50 - - - - - - lddqu (%rax), %xmm2 # CHECK-NEXT: - - 0.25 0.25 - - - 0.25 0.25 - - - monitor # CHECK-NEXT: - - - - - - - 1.00 - - - - movddup %xmm0, %xmm2 diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx1.s index cabb002b8241..f4904e4467ff 100644 --- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx1.s +++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx1.s @@ -1736,7 +1736,7 @@ vzeroupper # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] -# CHECK-NEXT: - 126.00 325.25 202.25 173.83 173.83 38.00 326.25 7.25 11.33 +# CHECK-NEXT: - 126.00 327.92 204.92 173.83 173.83 38.00 320.92 7.25 11.33 # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] Instructions: @@ -1906,22 +1906,22 @@ vzeroupper # CHECK-NEXT: - - - - 0.33 0.33 1.00 - - 0.33 vextractf128 $1, %ymm0, (%rax) # CHECK-NEXT: - - 1.00 - - - - 1.00 - - vextractps $1, %xmm0, %ecx # CHECK-NEXT: - - - - 0.33 0.33 1.00 1.00 - 0.33 vextractps $1, %xmm0, (%rax) -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - vhaddpd %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - vhaddpd (%rax), %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - vhaddpd %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - vhaddpd (%rax), %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - vhaddps %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - vhaddps (%rax), %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - vhaddps %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - vhaddps (%rax), %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - vhsubpd %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - vhsubpd (%rax), %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - vhsubpd %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - vhsubpd (%rax), %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - vhsubps %xmm0, %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - vhsubps (%rax), %xmm1, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - vhsubps %ymm0, %ymm1, %ymm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - vhsubps (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - vhaddpd %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - vhaddpd (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - vhaddpd %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - vhaddpd (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - vhaddps %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - vhaddps (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - vhaddps %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - vhaddps (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - vhsubpd %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - vhsubpd (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - vhsubpd %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - vhsubpd (%rax), %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - vhsubps %xmm0, %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - vhsubps (%rax), %xmm1, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - vhsubps %ymm0, %ymm1, %ymm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - vhsubps (%rax), %ymm1, %ymm2 # CHECK-NEXT: - - - - - - - 1.00 - - vinsertf128 $1, %xmm0, %ymm1, %ymm2 # CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 0.33 - - vinsertf128 $1, (%rax), %ymm1, %ymm2 # CHECK-NEXT: - - - - - - - 1.00 - - vinsertps $1, %xmm0, %xmm1, %xmm2 diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse3.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse3.s index e6bec1953fb8..0b6b03508389 100644 --- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse3.s +++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse3.s @@ -79,7 +79,7 @@ mwait # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] -# CHECK-NEXT: - - 6.67 6.67 5.00 5.00 - 24.67 4.00 - +# CHECK-NEXT: - - 8.00 8.00 5.00 5.00 - 22.00 4.00 - # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] Instructions: @@ -87,14 +87,14 @@ mwait # CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - - - - addsubpd (%rax), %xmm2 # CHECK-NEXT: - - 0.50 0.50 - - - - - - addsubps %xmm0, %xmm2 # CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - - - - addsubps (%rax), %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - haddpd %xmm0, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - haddpd (%rax), %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - haddps %xmm0, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - haddps (%rax), %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - hsubpd %xmm0, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - hsubpd (%rax), %xmm2 -# CHECK-NEXT: - - 0.33 0.33 - - - 2.33 - - hsubps %xmm0, %xmm2 -# CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 2.33 - - hsubps (%rax), %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - haddpd %xmm0, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - haddpd (%rax), %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - haddps %xmm0, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - haddps (%rax), %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - hsubpd %xmm0, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - hsubpd (%rax), %xmm2 +# CHECK-NEXT: - - 0.50 0.50 - - - 2.00 - - hsubps %xmm0, %xmm2 +# CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - 2.00 - - hsubps (%rax), %xmm2 # CHECK-NEXT: - - - - 0.50 0.50 - - - - lddqu (%rax), %xmm2 # CHECK-NEXT: - - 0.25 0.25 - - - 0.25 0.25 - monitor # CHECK-NEXT: - - - - - - - 1.00 - - movddup %xmm0, %xmm2 -- GitLab From 170d45c0eb018744ab94e3baa96af6ab74c92bfc Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Thu, 6 Jun 2024 09:29:59 -0400 Subject: [PATCH 092/462] [Clang][AMDGPU] Use `I` to decorate imm argument for `__builtin_amdgcn_global_load_lds` (#94376) --- clang/include/clang/Basic/BuiltinsAMDGPU.def | 2 +- clang/lib/Sema/SemaAMDGPU.cpp | 3 +-- clang/test/SemaOpenCL/builtins-amdgcn-gfx940-err.cl | 6 ++++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def index 433c7795325f..9e6800ea814a 100644 --- a/clang/include/clang/Basic/BuiltinsAMDGPU.def +++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def @@ -240,7 +240,7 @@ TARGET_BUILTIN(__builtin_amdgcn_flat_atomic_fadd_v2bf16, "V2sV2s*0V2s", "t", "at TARGET_BUILTIN(__builtin_amdgcn_global_atomic_fadd_v2bf16, "V2sV2s*1V2s", "t", "atomic-global-pk-add-bf16-inst") TARGET_BUILTIN(__builtin_amdgcn_ds_atomic_fadd_v2bf16, "V2sV2s*3V2s", "t", "atomic-ds-pk-add-16-insts") TARGET_BUILTIN(__builtin_amdgcn_ds_atomic_fadd_v2f16, "V2hV2h*3V2h", "t", "atomic-ds-pk-add-16-insts") -TARGET_BUILTIN(__builtin_amdgcn_global_load_lds, "vv*1v*3UiiUi", "t", "gfx940-insts") +TARGET_BUILTIN(__builtin_amdgcn_global_load_lds, "vv*1v*3IUiIiIUi", "t", "gfx940-insts") //===----------------------------------------------------------------------===// // Deep learning builtins. diff --git a/clang/lib/Sema/SemaAMDGPU.cpp b/clang/lib/Sema/SemaAMDGPU.cpp index c446cc1d042a..51d4f0d3d964 100644 --- a/clang/lib/Sema/SemaAMDGPU.cpp +++ b/clang/lib/Sema/SemaAMDGPU.cpp @@ -32,8 +32,7 @@ bool SemaAMDGPU::CheckAMDGCNBuiltinFunctionCall(unsigned BuiltinID, llvm::APSInt Size; Expr *ArgExpr = TheCall->getArg(SizeIdx); ExprResult R = SemaRef.VerifyIntegerConstantExpression(ArgExpr, &Size); - if (R.isInvalid()) - return true; + assert(!R.isInvalid()); switch (Size.getSExtValue()) { case 1: case 2: diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-gfx940-err.cl b/clang/test/SemaOpenCL/builtins-amdgcn-gfx940-err.cl index 487cc53e8ad8..2a1ba4300864 100644 --- a/clang/test/SemaOpenCL/builtins-amdgcn-gfx940-err.cl +++ b/clang/test/SemaOpenCL/builtins-amdgcn-gfx940-err.cl @@ -3,8 +3,10 @@ typedef unsigned int u32; -void test_global_load_lds_unsupported_size(global u32* src, local u32 *dst, u32 size) { - __builtin_amdgcn_global_load_lds(src, dst, size, /*offset=*/0, /*aux=*/0); // expected-error{{expression is not an integer constant expression}} +void test_global_load_lds_unsupported_size(global u32* src, local u32 *dst, u32 size, u32 offset, u32 aux) { + __builtin_amdgcn_global_load_lds(src, dst, size, /*offset=*/0, /*aux=*/0); // expected-error{{argument to '__builtin_amdgcn_global_load_lds' must be a constant integer}} + __builtin_amdgcn_global_load_lds(src, dst, /*size=*/4, offset, /*aux=*/0); // expected-error{{argument to '__builtin_amdgcn_global_load_lds' must be a constant integer}} + __builtin_amdgcn_global_load_lds(src, dst, /*size=*/4, /*offset=*/0, aux); // expected-error{{argument to '__builtin_amdgcn_global_load_lds' must be a constant integer}} __builtin_amdgcn_global_load_lds(src, dst, /*size=*/5, /*offset=*/0, /*aux=*/0); // expected-error{{invalid size value}} expected-note {{size must be 1, 2, or 4}} __builtin_amdgcn_global_load_lds(src, dst, /*size=*/0, /*offset=*/0, /*aux=*/0); // expected-error{{invalid size value}} expected-note {{size must be 1, 2, or 4}} __builtin_amdgcn_global_load_lds(src, dst, /*size=*/3, /*offset=*/0, /*aux=*/0); // expected-error{{invalid size value}} expected-note {{size must be 1, 2, or 4}} -- GitLab From 31442c9669ca234ab71294125f484882b1cef9d4 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Thu, 6 Jun 2024 08:27:19 -0500 Subject: [PATCH 093/462] [libc] Enable varargs tests for AMDGPU targets Summary: This reverts commit 574ab7e7b759a400ecf54c1141b1433bb2415e93. --- libc/config/gpu/entrypoints.txt | 11 +++++++++++ libc/test/src/__support/CMakeLists.txt | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt index b678350e9fcb..2217a696fc5d 100644 --- a/libc/config/gpu/entrypoints.txt +++ b/libc/config/gpu/entrypoints.txt @@ -1,3 +1,13 @@ +if(LIBC_TARGET_ARCHITECTURE_IS_AMDGPU) + set(extra_entrypoints + # stdio.h entrypoints + libc.src.stdio.sprintf + libc.src.stdio.snprintf + libc.src.stdio.vsprintf + libc.src.stdio.vsnprintf + ) +endif() + set(TARGET_LIBC_ENTRYPOINTS # assert.h entrypoints libc.src.assert.__assert_fail @@ -175,6 +185,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.errno.errno # stdio.h entrypoints + ${extra_entrypoints} libc.src.stdio.feof libc.src.stdio.ferror libc.src.stdio.fseek diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt index 663aa2bb82ca..5afc4173f61a 100644 --- a/libc/test/src/__support/CMakeLists.txt +++ b/libc/test/src/__support/CMakeLists.txt @@ -86,8 +86,8 @@ add_libc_test( libc.src.__support.uint128 ) -# The GPU does not support varargs currently. -if(NOT LIBC_TARGET_OS_IS_GPU) +# NVPTX does not support varargs currently. +if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX) add_libc_test( arg_list_test SUITE -- GitLab From 435addbf5088be14b1b7e0398ecd16501dbf4498 Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Thu, 6 Jun 2024 06:42:46 -0700 Subject: [PATCH 094/462] [NVPTX] Revamp NVVMIntrRange pass (#94422) Revamp the NVVMIntrRange pass making the following updates: - Use range attributes over range metadata. This is what instcombine has move to for ranges on intrinsics in https://github.com/llvm/llvm-project/pull/88776 and it seems a bit cleaner. - Consider the `!"maxntid{x,y,z}"` and `!"reqntid{x,y,z}"` function metadata when adding ranges for `tid` srge instrinsics. This can allow for smaller ranges and more optimization. - When range attributes are already present, use the intersection of the old and new range. This complements the metadata change by allowing ranges to be shrunk when an intrinsic is in a function which is inlined into a kernel with metadata. While we don't call this more then once yet, we should consider adding a second call after inlining, once this has had a chance to soak for a while and no issues have arisen. I've also re-enabled this pass in the TM, it was disabled years ago due to "numerical discrepancies" https://reviews.llvm.org/D96166. In our testing we haven't seen any issues with adding ranges to intrinsics, and I cannot find any further info about what issues were encountered. --- clang/test/CodeGenCUDA/cuda-builtin-vars.cu | 24 +-- llvm/lib/Target/NVPTX/NVPTX.h | 7 +- llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp | 32 ++- llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp | 6 +- llvm/lib/Target/NVPTX/NVPTXUtilities.cpp | 57 ++++-- llvm/lib/Target/NVPTX/NVPTXUtilities.h | 16 +- llvm/lib/Target/NVPTX/NVVMIntrRange.cpp | 197 +++++++++---------- llvm/test/CodeGen/NVPTX/intr-range.ll | 88 +++++++++ llvm/test/CodeGen/NVPTX/intrinsic-old.ll | 51 ++--- 9 files changed, 280 insertions(+), 198 deletions(-) create mode 100644 llvm/test/CodeGen/NVPTX/intr-range.ll diff --git a/clang/test/CodeGenCUDA/cuda-builtin-vars.cu b/clang/test/CodeGenCUDA/cuda-builtin-vars.cu index ba5e5f13ebe7..dba0a76af21d 100644 --- a/clang/test/CodeGenCUDA/cuda-builtin-vars.cu +++ b/clang/test/CodeGenCUDA/cuda-builtin-vars.cu @@ -6,21 +6,21 @@ __attribute__((global)) void kernel(int *out) { int i = 0; - out[i++] = threadIdx.x; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.tid.x() - out[i++] = threadIdx.y; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.tid.y() - out[i++] = threadIdx.z; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.tid.z() + out[i++] = threadIdx.x; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.tid.x() + out[i++] = threadIdx.y; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.tid.y() + out[i++] = threadIdx.z; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.tid.z() - out[i++] = blockIdx.x; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() - out[i++] = blockIdx.y; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.ctaid.y() - out[i++] = blockIdx.z; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.ctaid.z() + out[i++] = blockIdx.x; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() + out[i++] = blockIdx.y; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.ctaid.y() + out[i++] = blockIdx.z; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.ctaid.z() - out[i++] = blockDim.x; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.ntid.x() - out[i++] = blockDim.y; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.ntid.y() - out[i++] = blockDim.z; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.ntid.z() + out[i++] = blockDim.x; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.ntid.x() + out[i++] = blockDim.y; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.ntid.y() + out[i++] = blockDim.z; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.ntid.z() - out[i++] = gridDim.x; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.nctaid.x() - out[i++] = gridDim.y; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.nctaid.y() - out[i++] = gridDim.z; // CHECK: call noundef i32 @llvm.nvvm.read.ptx.sreg.nctaid.z() + out[i++] = gridDim.x; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.nctaid.x() + out[i++] = gridDim.y; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.nctaid.y() + out[i++] = gridDim.z; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.nctaid.z() out[i++] = warpSize; // CHECK: store i32 32, diff --git a/llvm/lib/Target/NVPTX/NVPTX.h b/llvm/lib/Target/NVPTX/NVPTX.h index 5eefab59a6ab..b0cb24c63c3c 100644 --- a/llvm/lib/Target/NVPTX/NVPTX.h +++ b/llvm/lib/Target/NVPTX/NVPTX.h @@ -40,7 +40,7 @@ FunctionPass *createNVPTXISelDag(NVPTXTargetMachine &TM, ModulePass *createNVPTXAssignValidGlobalNamesPass(); ModulePass *createGenericToNVVMLegacyPass(); ModulePass *createNVPTXCtorDtorLoweringLegacyPass(); -FunctionPass *createNVVMIntrRangePass(unsigned int SmVersion); +FunctionPass *createNVVMIntrRangePass(); FunctionPass *createNVVMReflectPass(unsigned int SmVersion); MachineFunctionPass *createNVPTXPrologEpilogPass(); MachineFunctionPass *createNVPTXReplaceImageHandlesPass(); @@ -53,12 +53,7 @@ MachineFunctionPass *createNVPTXPeephole(); MachineFunctionPass *createNVPTXProxyRegErasurePass(); struct NVVMIntrRangePass : PassInfoMixin { - NVVMIntrRangePass(); - NVVMIntrRangePass(unsigned SmVersion) : SmVersion(SmVersion) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); - -private: - unsigned SmVersion; }; struct NVVMReflectPass : PassInfoMixin { diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp index f63697916d90..82770f866085 100644 --- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp @@ -542,30 +542,24 @@ void NVPTXAsmPrinter::emitKernelFunctionDirectives(const Function &F, // If the NVVM IR has some of reqntid* specified, then output // the reqntid directive, and set the unspecified ones to 1. // If none of Reqntid* is specified, don't output reqntid directive. - unsigned Reqntidx, Reqntidy, Reqntidz; - Reqntidx = Reqntidy = Reqntidz = 1; - bool ReqSpecified = false; - ReqSpecified |= getReqNTIDx(F, Reqntidx); - ReqSpecified |= getReqNTIDy(F, Reqntidy); - ReqSpecified |= getReqNTIDz(F, Reqntidz); + std::optional Reqntidx = getReqNTIDx(F); + std::optional Reqntidy = getReqNTIDy(F); + std::optional Reqntidz = getReqNTIDz(F); - if (ReqSpecified) - O << ".reqntid " << Reqntidx << ", " << Reqntidy << ", " << Reqntidz - << "\n"; + if (Reqntidx || Reqntidy || Reqntidz) + O << ".reqntid " << Reqntidx.value_or(1) << ", " << Reqntidy.value_or(1) + << ", " << Reqntidz.value_or(1) << "\n"; // If the NVVM IR has some of maxntid* specified, then output // the maxntid directive, and set the unspecified ones to 1. // If none of maxntid* is specified, don't output maxntid directive. - unsigned Maxntidx, Maxntidy, Maxntidz; - Maxntidx = Maxntidy = Maxntidz = 1; - bool MaxSpecified = false; - MaxSpecified |= getMaxNTIDx(F, Maxntidx); - MaxSpecified |= getMaxNTIDy(F, Maxntidy); - MaxSpecified |= getMaxNTIDz(F, Maxntidz); - - if (MaxSpecified) - O << ".maxntid " << Maxntidx << ", " << Maxntidy << ", " << Maxntidz - << "\n"; + std::optional Maxntidx = getMaxNTIDx(F); + std::optional Maxntidy = getMaxNTIDy(F); + std::optional Maxntidz = getMaxNTIDz(F); + + if (Maxntidx || Maxntidy || Maxntidz) + O << ".maxntid " << Maxntidx.value_or(1) << ", " << Maxntidy.value_or(1) + << ", " << Maxntidz.value_or(1) << "\n"; unsigned Mincta = 0; if (getMinCTASm(F, Mincta)) diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp index 4dc3cea4bd8e..b60a1d747af7 100644 --- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp @@ -233,9 +233,9 @@ void NVPTXTargetMachine::registerPassBuilderCallbacks( [this](ModulePassManager &PM, OptimizationLevel Level) { FunctionPassManager FPM; FPM.addPass(NVVMReflectPass(Subtarget.getSmVersion())); - // FIXME: NVVMIntrRangePass is causing numerical discrepancies, - // investigate and re-enable. - // FPM.addPass(NVVMIntrRangePass(Subtarget.getSmVersion())); + // Note: NVVMIntrRangePass was causing numerical discrepancies at one + // point, if issues crop up, consider disabling. + FPM.addPass(NVVMIntrRangePass()); PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); }); } diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp index 013afe916e86..3a536db1c972 100644 --- a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp @@ -128,6 +128,14 @@ bool findOneNVVMAnnotation(const GlobalValue *gv, const std::string &prop, return true; } +static std::optional +findOneNVVMAnnotation(const GlobalValue &GV, const std::string &PropName) { + unsigned RetVal; + if (findOneNVVMAnnotation(&GV, PropName, RetVal)) + return RetVal; + return std::nullopt; +} + bool findAllNVVMAnnotation(const GlobalValue *gv, const std::string &prop, std::vector &retval) { auto &AC = getAnnotationCache(); @@ -252,32 +260,57 @@ std::string getSamplerName(const Value &val) { return std::string(val.getName()); } -bool getMaxNTIDx(const Function &F, unsigned &x) { - return findOneNVVMAnnotation(&F, "maxntidx", x); +std::optional getMaxNTIDx(const Function &F) { + return findOneNVVMAnnotation(F, "maxntidx"); } -bool getMaxNTIDy(const Function &F, unsigned &y) { - return findOneNVVMAnnotation(&F, "maxntidy", y); +std::optional getMaxNTIDy(const Function &F) { + return findOneNVVMAnnotation(F, "maxntidy"); } -bool getMaxNTIDz(const Function &F, unsigned &z) { - return findOneNVVMAnnotation(&F, "maxntidz", z); +std::optional getMaxNTIDz(const Function &F) { + return findOneNVVMAnnotation(F, "maxntidz"); +} + +std::optional getMaxNTID(const Function &F) { + // Note: The semantics here are a bit strange. The PTX ISA states the + // following (11.4.2. Performance-Tuning Directives: .maxntid): + // + // Note that this directive guarantees that the total number of threads does + // not exceed the maximum, but does not guarantee that the limit in any + // particular dimension is not exceeded. + std::optional MaxNTIDx = getMaxNTIDx(F); + std::optional MaxNTIDy = getMaxNTIDy(F); + std::optional MaxNTIDz = getMaxNTIDz(F); + if (MaxNTIDx || MaxNTIDy || MaxNTIDz) + return MaxNTIDx.value_or(1) * MaxNTIDy.value_or(1) * MaxNTIDz.value_or(1); + return std::nullopt; } bool getMaxClusterRank(const Function &F, unsigned &x) { return findOneNVVMAnnotation(&F, "maxclusterrank", x); } -bool getReqNTIDx(const Function &F, unsigned &x) { - return findOneNVVMAnnotation(&F, "reqntidx", x); +std::optional getReqNTIDx(const Function &F) { + return findOneNVVMAnnotation(F, "reqntidx"); +} + +std::optional getReqNTIDy(const Function &F) { + return findOneNVVMAnnotation(F, "reqntidy"); } -bool getReqNTIDy(const Function &F, unsigned &y) { - return findOneNVVMAnnotation(&F, "reqntidy", y); +std::optional getReqNTIDz(const Function &F) { + return findOneNVVMAnnotation(F, "reqntidz"); } -bool getReqNTIDz(const Function &F, unsigned &z) { - return findOneNVVMAnnotation(&F, "reqntidz", z); +std::optional getReqNTID(const Function &F) { + // Note: The semantics here are a bit strange. See getMaxNTID. + std::optional ReqNTIDx = getReqNTIDx(F); + std::optional ReqNTIDy = getReqNTIDy(F); + std::optional ReqNTIDz = getReqNTIDz(F); + if (ReqNTIDx || ReqNTIDy || ReqNTIDz) + return ReqNTIDx.value_or(1) * ReqNTIDy.value_or(1) * ReqNTIDz.value_or(1); + return std::nullopt; } bool getMinCTASm(const Function &F, unsigned &x) { diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.h b/llvm/lib/Target/NVPTX/NVPTXUtilities.h index 2872db9fa213..e020bc0f02e9 100644 --- a/llvm/lib/Target/NVPTX/NVPTXUtilities.h +++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.h @@ -48,13 +48,15 @@ std::string getTextureName(const Value &); std::string getSurfaceName(const Value &); std::string getSamplerName(const Value &); -bool getMaxNTIDx(const Function &, unsigned &); -bool getMaxNTIDy(const Function &, unsigned &); -bool getMaxNTIDz(const Function &, unsigned &); - -bool getReqNTIDx(const Function &, unsigned &); -bool getReqNTIDy(const Function &, unsigned &); -bool getReqNTIDz(const Function &, unsigned &); +std::optional getMaxNTIDx(const Function &); +std::optional getMaxNTIDy(const Function &); +std::optional getMaxNTIDz(const Function &); +std::optional getMaxNTID(const Function &F); + +std::optional getReqNTIDx(const Function &); +std::optional getReqNTIDy(const Function &); +std::optional getReqNTIDz(const Function &); +std::optional getReqNTID(const Function &); bool getMaxClusterRank(const Function &, unsigned &); bool getMinCTASm(const Function &, unsigned &); diff --git a/llvm/lib/Target/NVPTX/NVVMIntrRange.cpp b/llvm/lib/Target/NVPTX/NVVMIntrRange.cpp index 5381646434eb..9afce99ce79f 100644 --- a/llvm/lib/Target/NVPTX/NVVMIntrRange.cpp +++ b/llvm/lib/Target/NVPTX/NVVMIntrRange.cpp @@ -1,4 +1,4 @@ -//===- NVVMIntrRange.cpp - Set !range metadata for NVVM intrinsics --------===// +//===- NVVMIntrRange.cpp - Set range attributes for NVVM intrinsics -------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,19 +6,21 @@ // //===----------------------------------------------------------------------===// // -// This pass adds appropriate !range metadata for calls to NVVM +// This pass adds appropriate range attributes for calls to NVVM // intrinsics that return a limited range of values. // //===----------------------------------------------------------------------===// #include "NVPTX.h" -#include "llvm/IR/Constants.h" +#include "NVPTXUtilities.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsNVPTX.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/CommandLine.h" +#include using namespace llvm; @@ -26,31 +28,22 @@ using namespace llvm; namespace llvm { void initializeNVVMIntrRangePass(PassRegistry &); } -// Add !range metadata based on limits of given SM variant. -static cl::opt NVVMIntrRangeSM("nvvm-intr-range-sm", cl::init(20), - cl::Hidden, cl::desc("SM variant")); - namespace { class NVVMIntrRange : public FunctionPass { - private: - unsigned SmVersion; + unsigned SmVersion; - public: - static char ID; - NVVMIntrRange() : NVVMIntrRange(NVVMIntrRangeSM) {} - NVVMIntrRange(unsigned int SmVersion) - : FunctionPass(ID), SmVersion(SmVersion) { +public: + static char ID; + NVVMIntrRange() : FunctionPass(ID) { - initializeNVVMIntrRangePass(*PassRegistry::getPassRegistry()); - } + initializeNVVMIntrRangePass(*PassRegistry::getPassRegistry()); + } - bool runOnFunction(Function &) override; + bool runOnFunction(Function &) override; }; -} +} // namespace -FunctionPass *llvm::createNVVMIntrRangePass(unsigned int SmVersion) { - return new NVVMIntrRange(SmVersion); -} +FunctionPass *llvm::createNVVMIntrRangePass() { return new NVVMIntrRange(); } char NVVMIntrRange::ID = 0; INITIALIZE_PASS(NVVMIntrRange, "nvvm-intr-range", @@ -58,112 +51,110 @@ INITIALIZE_PASS(NVVMIntrRange, "nvvm-intr-range", // Adds the passed-in [Low,High) range information as metadata to the // passed-in call instruction. -static bool addRangeMetadata(uint64_t Low, uint64_t High, CallInst *C) { - // This call already has range metadata, nothing to do. - if (C->getMetadata(LLVMContext::MD_range)) +static bool addRangeAttr(uint64_t Low, uint64_t High, IntrinsicInst *II) { + if (II->getMetadata(LLVMContext::MD_range)) return false; - LLVMContext &Context = C->getParent()->getContext(); - IntegerType *Int32Ty = Type::getInt32Ty(Context); - Metadata *LowAndHigh[] = { - ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Low)), - ConstantAsMetadata::get(ConstantInt::get(Int32Ty, High))}; - C->setMetadata(LLVMContext::MD_range, MDNode::get(Context, LowAndHigh)); + const uint64_t BitWidth = II->getType()->getIntegerBitWidth(); + ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High)); + + if (auto CurrentRange = II->getRange()) + Range = Range.intersectWith(CurrentRange.value()); + + II->addRangeRetAttr(Range); return true; } -static bool runNVVMIntrRange(Function &F, unsigned SmVersion) { +static bool runNVVMIntrRange(Function &F) { struct { unsigned x, y, z; } MaxBlockSize, MaxGridSize; - MaxBlockSize.x = 1024; - MaxBlockSize.y = 1024; - MaxBlockSize.z = 64; - MaxGridSize.x = SmVersion >= 30 ? 0x7fffffff : 0xffff; + const unsigned MetadataNTID = getReqNTID(F).value_or( + getMaxNTID(F).value_or(std::numeric_limits::max())); + + MaxBlockSize.x = std::min(1024u, MetadataNTID); + MaxBlockSize.y = std::min(1024u, MetadataNTID); + MaxBlockSize.z = std::min(64u, MetadataNTID); + + MaxGridSize.x = 0x7fffffff; MaxGridSize.y = 0xffff; MaxGridSize.z = 0xffff; // Go through the calls in this function. bool Changed = false; for (Instruction &I : instructions(F)) { - CallInst *Call = dyn_cast(&I); - if (!Call) + IntrinsicInst *II = dyn_cast(&I); + if (!II) continue; - if (Function *Callee = Call->getCalledFunction()) { - switch (Callee->getIntrinsicID()) { - // Index within block - case Intrinsic::nvvm_read_ptx_sreg_tid_x: - Changed |= addRangeMetadata(0, MaxBlockSize.x, Call); - break; - case Intrinsic::nvvm_read_ptx_sreg_tid_y: - Changed |= addRangeMetadata(0, MaxBlockSize.y, Call); - break; - case Intrinsic::nvvm_read_ptx_sreg_tid_z: - Changed |= addRangeMetadata(0, MaxBlockSize.z, Call); - break; - - // Block size - case Intrinsic::nvvm_read_ptx_sreg_ntid_x: - Changed |= addRangeMetadata(1, MaxBlockSize.x+1, Call); - break; - case Intrinsic::nvvm_read_ptx_sreg_ntid_y: - Changed |= addRangeMetadata(1, MaxBlockSize.y+1, Call); - break; - case Intrinsic::nvvm_read_ptx_sreg_ntid_z: - Changed |= addRangeMetadata(1, MaxBlockSize.z+1, Call); - break; - - // Index within grid - case Intrinsic::nvvm_read_ptx_sreg_ctaid_x: - Changed |= addRangeMetadata(0, MaxGridSize.x, Call); - break; - case Intrinsic::nvvm_read_ptx_sreg_ctaid_y: - Changed |= addRangeMetadata(0, MaxGridSize.y, Call); - break; - case Intrinsic::nvvm_read_ptx_sreg_ctaid_z: - Changed |= addRangeMetadata(0, MaxGridSize.z, Call); - break; - - // Grid size - case Intrinsic::nvvm_read_ptx_sreg_nctaid_x: - Changed |= addRangeMetadata(1, MaxGridSize.x+1, Call); - break; - case Intrinsic::nvvm_read_ptx_sreg_nctaid_y: - Changed |= addRangeMetadata(1, MaxGridSize.y+1, Call); - break; - case Intrinsic::nvvm_read_ptx_sreg_nctaid_z: - Changed |= addRangeMetadata(1, MaxGridSize.z+1, Call); - break; - - // warp size is constant 32. - case Intrinsic::nvvm_read_ptx_sreg_warpsize: - Changed |= addRangeMetadata(32, 32+1, Call); - break; - - // Lane ID is [0..warpsize) - case Intrinsic::nvvm_read_ptx_sreg_laneid: - Changed |= addRangeMetadata(0, 32, Call); - break; - - default: - break; - } + switch (II->getIntrinsicID()) { + // Index within block + case Intrinsic::nvvm_read_ptx_sreg_tid_x: + Changed |= addRangeAttr(0, MaxBlockSize.x, II); + break; + case Intrinsic::nvvm_read_ptx_sreg_tid_y: + Changed |= addRangeAttr(0, MaxBlockSize.y, II); + break; + case Intrinsic::nvvm_read_ptx_sreg_tid_z: + Changed |= addRangeAttr(0, MaxBlockSize.z, II); + break; + + // Block size + case Intrinsic::nvvm_read_ptx_sreg_ntid_x: + Changed |= addRangeAttr(1, MaxBlockSize.x + 1, II); + break; + case Intrinsic::nvvm_read_ptx_sreg_ntid_y: + Changed |= addRangeAttr(1, MaxBlockSize.y + 1, II); + break; + case Intrinsic::nvvm_read_ptx_sreg_ntid_z: + Changed |= addRangeAttr(1, MaxBlockSize.z + 1, II); + break; + + // Index within grid + case Intrinsic::nvvm_read_ptx_sreg_ctaid_x: + Changed |= addRangeAttr(0, MaxGridSize.x, II); + break; + case Intrinsic::nvvm_read_ptx_sreg_ctaid_y: + Changed |= addRangeAttr(0, MaxGridSize.y, II); + break; + case Intrinsic::nvvm_read_ptx_sreg_ctaid_z: + Changed |= addRangeAttr(0, MaxGridSize.z, II); + break; + + // Grid size + case Intrinsic::nvvm_read_ptx_sreg_nctaid_x: + Changed |= addRangeAttr(1, MaxGridSize.x + 1, II); + break; + case Intrinsic::nvvm_read_ptx_sreg_nctaid_y: + Changed |= addRangeAttr(1, MaxGridSize.y + 1, II); + break; + case Intrinsic::nvvm_read_ptx_sreg_nctaid_z: + Changed |= addRangeAttr(1, MaxGridSize.z + 1, II); + break; + + // warp size is constant 32. + case Intrinsic::nvvm_read_ptx_sreg_warpsize: + Changed |= addRangeAttr(32, 32 + 1, II); + break; + + // Lane ID is [0..warpsize) + case Intrinsic::nvvm_read_ptx_sreg_laneid: + Changed |= addRangeAttr(0, 32, II); + break; + + default: + break; } } return Changed; } -bool NVVMIntrRange::runOnFunction(Function &F) { - return runNVVMIntrRange(F, SmVersion); -} - -NVVMIntrRangePass::NVVMIntrRangePass() : NVVMIntrRangePass(NVVMIntrRangeSM) {} +bool NVVMIntrRange::runOnFunction(Function &F) { return runNVVMIntrRange(F); } PreservedAnalyses NVVMIntrRangePass::run(Function &F, FunctionAnalysisManager &AM) { - return runNVVMIntrRange(F, SmVersion) ? PreservedAnalyses::none() - : PreservedAnalyses::all(); + return runNVVMIntrRange(F) ? PreservedAnalyses::none() + : PreservedAnalyses::all(); } diff --git a/llvm/test/CodeGen/NVPTX/intr-range.ll b/llvm/test/CodeGen/NVPTX/intr-range.ll new file mode 100644 index 000000000000..2f3e08a039f5 --- /dev/null +++ b/llvm/test/CodeGen/NVPTX/intr-range.ll @@ -0,0 +1,88 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --version 5 +; RUN: opt < %s -S -mtriple=nvptx-nvidia-cuda -mcpu=sm_20 -passes=nvvm-intr-range | FileCheck %s + +define i32 @test_maxntid() { +; CHECK-LABEL: define i32 @test_maxntid( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 96) i32 @llvm.nvvm.read.ptx.sreg.tid.x() +; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 96) i32 @llvm.nvvm.read.ptx.sreg.tid.y() +; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 64) i32 @llvm.nvvm.read.ptx.sreg.tid.z() +; CHECK-NEXT: [[TMP11:%.*]] = call range(i32 1, 97) i32 @llvm.nvvm.read.ptx.sreg.ntid.x() +; CHECK-NEXT: [[TMP4:%.*]] = call range(i32 1, 97) i32 @llvm.nvvm.read.ptx.sreg.ntid.y() +; CHECK-NEXT: [[TMP6:%.*]] = call range(i32 1, 65) i32 @llvm.nvvm.read.ptx.sreg.ntid.z() +; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[TMP1]], [[TMP3]] +; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], [[TMP2]] +; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[TMP8]], [[TMP11]] +; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP9]], [[TMP4]] +; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[TMP10]], [[TMP6]] +; CHECK-NEXT: ret i32 [[TMP5]] +; + %1 = call i32 @llvm.nvvm.read.ptx.sreg.tid.x() + %2 = call i32 @llvm.nvvm.read.ptx.sreg.tid.y() + %3 = call i32 @llvm.nvvm.read.ptx.sreg.tid.z() + %4 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.x() + %5 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.y() + %6 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.z() + %7 = add i32 %1, %2 + %8 = add i32 %7, %3 + %9 = add i32 %8, %4 + %10 = add i32 %9, %5 + %11 = add i32 %10, %6 + ret i32 %11 +} + +define i32 @test_reqntid() { +; CHECK-LABEL: define i32 @test_reqntid( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 20) i32 @llvm.nvvm.read.ptx.sreg.tid.x() +; CHECK-NEXT: [[TMP5:%.*]] = call range(i32 0, 20) i32 @llvm.nvvm.read.ptx.sreg.tid.y() +; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 20) i32 @llvm.nvvm.read.ptx.sreg.tid.z() +; CHECK-NEXT: [[TMP4:%.*]] = call range(i32 1, 21) i32 @llvm.nvvm.read.ptx.sreg.ntid.x() +; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 1, 21) i32 @llvm.nvvm.read.ptx.sreg.ntid.y() +; CHECK-NEXT: [[TMP6:%.*]] = call range(i32 1, 21) i32 @llvm.nvvm.read.ptx.sreg.ntid.z() +; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[TMP1]], [[TMP5]] +; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], [[TMP2]] +; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[TMP8]], [[TMP4]] +; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP9]], [[TMP3]] +; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP10]], [[TMP6]] +; CHECK-NEXT: ret i32 [[TMP3]] +; + %1 = call i32 @llvm.nvvm.read.ptx.sreg.tid.x() + %2 = call i32 @llvm.nvvm.read.ptx.sreg.tid.y() + %3 = call i32 @llvm.nvvm.read.ptx.sreg.tid.z() + %4 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.x() + %5 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.y() + %6 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.z() + %7 = add i32 %1, %2 + %8 = add i32 %7, %3 + %9 = add i32 %8, %4 + %10 = add i32 %9, %5 + %11 = add i32 %10, %6 + ret i32 %5 +} + +;; A case like this could occur if a function with the sreg intrinsic was +;; inlined into a kernel where the tid metadata is present, ensure the range is +;; updated. +define i32 @test_inlined() { +; CHECK-LABEL: define i32 @test_inlined( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 4) i32 @llvm.nvvm.read.ptx.sreg.tid.x() +; CHECK-NEXT: ret i32 [[TMP1]] +; + %1 = call range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.x() + ret i32 %1 +} + +declare i32 @llvm.nvvm.read.ptx.sreg.tid.x() +declare i32 @llvm.nvvm.read.ptx.sreg.tid.y() +declare i32 @llvm.nvvm.read.ptx.sreg.tid.z() + +declare i32 @llvm.nvvm.read.ptx.sreg.ntid.x() +declare i32 @llvm.nvvm.read.ptx.sreg.ntid.y() +declare i32 @llvm.nvvm.read.ptx.sreg.ntid.z() + +!nvvm.annotations = !{!0, !1, !2} +!0 = !{ptr @test_maxntid, !"kernel", i32 1, !"maxntidx", i32 32, !"maxntidz", i32 3} +!1 = !{ptr @test_reqntid, !"kernel", i32 1, !"reqntidx", i32 20} +!2 = !{ptr @test_inlined, !"kernel", i32 1, !"maxntidx", i32 4} diff --git a/llvm/test/CodeGen/NVPTX/intrinsic-old.ll b/llvm/test/CodeGen/NVPTX/intrinsic-old.ll index 3930e6d77418..85f7817f08dc 100644 --- a/llvm/test/CodeGen/NVPTX/intrinsic-old.ll +++ b/llvm/test/CodeGen/NVPTX/intrinsic-old.ll @@ -1,21 +1,13 @@ ; RUN: llc < %s -march=nvptx -mcpu=sm_20 | FileCheck -allow-deprecated-dag-overlap %s ; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 | FileCheck -allow-deprecated-dag-overlap %s ; RUN: opt < %s -S -mtriple=nvptx-nvidia-cuda -passes=nvvm-intr-range \ -; RUN: | FileCheck -allow-deprecated-dag-overlap --check-prefix=RANGE --check-prefix=RANGE_20 %s -; RUN: opt < %s -S -mtriple=nvptx-nvidia-cuda -passes=nvvm-intr-range \ -; RUN: | FileCheck -allow-deprecated-dag-overlap --check-prefix=RANGE --check-prefix=RANGE_20 %s -; RUN: opt < %s -S -mtriple=nvptx-nvidia-cuda \ -; RUN: -passes=nvvm-intr-range -nvvm-intr-range-sm=30 \ -; RUN: | FileCheck -allow-deprecated-dag-overlap --check-prefix=RANGE --check-prefix=RANGE_30 %s -; RUN: opt < %s -S -mtriple=nvptx-nvidia-cuda \ -; RUN: -passes=nvvm-intr-range -nvvm-intr-range-sm=30 \ -; RUN: | FileCheck -allow-deprecated-dag-overlap --check-prefix=RANGE --check-prefix=RANGE_30 %s +; RUN: | FileCheck -allow-deprecated-dag-overlap --check-prefix=RANGE %s ; RUN: %if ptxas && !ptxas-12.0 %{ llc < %s -march=nvptx -mcpu=sm_20 | %ptxas-verify %} ; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_20 | %ptxas-verify %} define ptx_device i32 @test_tid_x() { ; CHECK: mov.u32 %r{{[0-9]+}}, %tid.x; -; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range ![[BLK_IDX_XY:[0-9]+]] +; RANGE: call range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.x() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.tid.x() ret i32 %x @@ -23,7 +15,7 @@ define ptx_device i32 @test_tid_x() { define ptx_device i32 @test_tid_y() { ; CHECK: mov.u32 %r{{[0-9]+}}, %tid.y; -; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.tid.y(), !range ![[BLK_IDX_XY]] +; RANGE: call range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.y() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.tid.y() ret i32 %x @@ -31,7 +23,7 @@ define ptx_device i32 @test_tid_y() { define ptx_device i32 @test_tid_z() { ; CHECK: mov.u32 %r{{[0-9]+}}, %tid.z; -; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.tid.z(), !range ![[BLK_IDX_Z:[0-9]+]] +; RANGE: call range(i32 0, 64) i32 @llvm.nvvm.read.ptx.sreg.tid.z() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.tid.z() ret i32 %x @@ -46,7 +38,7 @@ define ptx_device i32 @test_tid_w() { define ptx_device i32 @test_ntid_x() { ; CHECK: mov.u32 %r{{[0-9]+}}, %ntid.x; -; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.ntid.x(), !range ![[BLK_SIZE_XY:[0-9]+]] +; RANGE: call range(i32 1, 1025) i32 @llvm.nvvm.read.ptx.sreg.ntid.x() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.ntid.x() ret i32 %x @@ -54,7 +46,7 @@ define ptx_device i32 @test_ntid_x() { define ptx_device i32 @test_ntid_y() { ; CHECK: mov.u32 %r{{[0-9]+}}, %ntid.y; -; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.ntid.y(), !range ![[BLK_SIZE_XY]] +; RANGE: call range(i32 1, 1025) i32 @llvm.nvvm.read.ptx.sreg.ntid.y() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.ntid.y() ret i32 %x @@ -62,7 +54,7 @@ define ptx_device i32 @test_ntid_y() { define ptx_device i32 @test_ntid_z() { ; CHECK: mov.u32 %r{{[0-9]+}}, %ntid.z; -; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.ntid.z(), !range ![[BLK_SIZE_Z:[0-9]+]] +; RANGE: call range(i32 1, 65) i32 @llvm.nvvm.read.ptx.sreg.ntid.z() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.ntid.z() ret i32 %x @@ -77,7 +69,7 @@ define ptx_device i32 @test_ntid_w() { define ptx_device i32 @test_laneid() { ; CHECK: mov.u32 %r{{[0-9]+}}, %laneid; -; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.laneid(), !range ![[LANEID:[0-9]+]] +; RANGE: call range(i32 0, 32) i32 @llvm.nvvm.read.ptx.sreg.laneid() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.laneid() ret i32 %x @@ -85,7 +77,7 @@ define ptx_device i32 @test_laneid() { define ptx_device i32 @test_warpsize() { ; CHECK: mov.u32 %r{{[0-9]+}}, WARP_SZ; -; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.warpsize(), !range ![[WARPSIZE:[0-9]+]] +; RANGE: call range(i32 32, 33) i32 @llvm.nvvm.read.ptx.sreg.warpsize() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.warpsize() ret i32 %x @@ -107,7 +99,7 @@ define ptx_device i32 @test_nwarpid() { define ptx_device i32 @test_ctaid_y() { ; CHECK: mov.u32 %r{{[0-9]+}}, %ctaid.y; -; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.ctaid.y(), !range ![[GRID_IDX_YZ:[0-9]+]] +; RANGE: call range(i32 0, 65535) i32 @llvm.nvvm.read.ptx.sreg.ctaid.y() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.y() ret i32 %x @@ -115,7 +107,7 @@ define ptx_device i32 @test_ctaid_y() { define ptx_device i32 @test_ctaid_z() { ; CHECK: mov.u32 %r{{[0-9]+}}, %ctaid.z; -; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.ctaid.z(), !range ![[GRID_IDX_YZ]] +; RANGE: call range(i32 0, 65535) i32 @llvm.nvvm.read.ptx.sreg.ctaid.z() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.z() ret i32 %x @@ -123,8 +115,7 @@ define ptx_device i32 @test_ctaid_z() { define ptx_device i32 @test_ctaid_x() { ; CHECK: mov.u32 %r{{[0-9]+}}, %ctaid.x; -; RANGE_30: call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range ![[GRID_IDX_X:[0-9]+]] -; RANGE_20: call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range ![[GRID_IDX_YZ]] +; RANGE: call range(i32 0, 2147483647) i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() ret i32 %x @@ -139,7 +130,7 @@ define ptx_device i32 @test_ctaid_w() { define ptx_device i32 @test_nctaid_y() { ; CHECK: mov.u32 %r{{[0-9]+}}, %nctaid.y; -; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.nctaid.y(), !range ![[GRID_SIZE_YZ:[0-9]+]] +; RANGE: call range(i32 1, 65536) i32 @llvm.nvvm.read.ptx.sreg.nctaid.y() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.y() ret i32 %x @@ -147,7 +138,7 @@ define ptx_device i32 @test_nctaid_y() { define ptx_device i32 @test_nctaid_z() { ; CHECK: mov.u32 %r{{[0-9]+}}, %nctaid.z; -; RANGE: call i32 @llvm.nvvm.read.ptx.sreg.nctaid.z(), !range ![[GRID_SIZE_YZ]] +; RANGE: call range(i32 1, 65536) i32 @llvm.nvvm.read.ptx.sreg.nctaid.z() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.z() ret i32 %x @@ -155,8 +146,7 @@ define ptx_device i32 @test_nctaid_z() { define ptx_device i32 @test_nctaid_x() { ; CHECK: mov.u32 %r{{[0-9]+}}, %nctaid.x; -; RANGE_30: call i32 @llvm.nvvm.read.ptx.sreg.nctaid.x(), !range ![[GRID_SIZE_X:[0-9]+]] -; RANGE_20: call i32 @llvm.nvvm.read.ptx.sreg.nctaid.x(), !range ![[GRID_SIZE_YZ]] +; RANGE: call range(i32 1, -2147483648) i32 @llvm.nvvm.read.ptx.sreg.nctaid.x() ; CHECK: ret; %x = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.x() ret i32 %x @@ -327,14 +317,3 @@ declare void @llvm.nvvm.bar.sync(i32 %i) !0 = !{i32 0, i32 19} ; RANGE-DAG: ![[ALREADY]] = !{i32 0, i32 19} -; RANGE-DAG: ![[BLK_IDX_XY]] = !{i32 0, i32 1024} -; RANGE-DAG: ![[BLK_IDX_XY]] = !{i32 0, i32 1024} -; RANGE-DAG: ![[BLK_IDX_Z]] = !{i32 0, i32 64} -; RANGE-DAG: ![[BLK_SIZE_XY]] = !{i32 1, i32 1025} -; RANGE-DAG: ![[BLK_SIZE_Z]] = !{i32 1, i32 65} -; RANGE-DAG: ![[LANEID]] = !{i32 0, i32 32} -; RANGE-DAG: ![[WARPSIZE]] = !{i32 32, i32 33} -; RANGE_30-DAG: ![[GRID_IDX_X]] = !{i32 0, i32 2147483647} -; RANGE-DAG: ![[GRID_IDX_YZ]] = !{i32 0, i32 65535} -; RANGE_30-DAG: ![[GRID_SIZE_X]] = !{i32 1, i32 -2147483648} -; RANGE-DAG: ![[GRID_SIZE_YZ]] = !{i32 1, i32 65536} -- GitLab From a97871e07dd62510dea1bb71b0f74261f52e4479 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Thu, 6 Jun 2024 15:43:05 +0200 Subject: [PATCH 095/462] [bazel] Fix layering check violation for nextafter_test_template target for 63cda2d --- utils/bazel/llvm-project-overlay/libc/test/src/math/BUILD.bazel | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/math/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/math/BUILD.bazel index 4f72a0a8e186..fac692addb9e 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/src/math/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/src/math/BUILD.bazel @@ -529,7 +529,9 @@ libc_support_library( "//libc:__support_cpp_bit", "//libc:__support_cpp_type_traits", "//libc:__support_fputil_basic_operations", + "//libc:__support_fputil_fenv_impl", "//libc:__support_fputil_fp_bits", + "//libc:hdr_fenv_macros", "//libc:hdr_math_macros", "//libc/test/UnitTest:LibcUnitTest", "//libc/test/UnitTest:fp_test_helpers", -- GitLab From e16f2f5d2491fde19afb63d5cec83625d391be30 Mon Sep 17 00:00:00 2001 From: Graham Hunter Date: Thu, 6 Jun 2024 14:45:36 +0100 Subject: [PATCH 096/462] [AArch64] Override isLSRCostLess, take number of instructions into account (#84189) Adds an AArch64-specific version of isLSRCostLess, changing the relative importance of the various terms from the formulae being evaluated. This has been split out from my vscale-aware LSR work, see the RFC for reference: https://discourse.llvm.org/t/rfc-vscale-aware-loopstrengthreduce/77131 --- .../AArch64/AArch64TargetTransformInfo.cpp | 19 ++ .../AArch64/AArch64TargetTransformInfo.h | 3 + .../AArch64/arm64-2011-10-18-LdStOptBug.ll | 2 +- .../test/CodeGen/AArch64/arm64-ldp-cluster.ll | 12 +- ...rleaving-reductions-predicated-scalable.ll | 112 +++++----- ...plex-deinterleaving-reductions-scalable.ll | 126 +++++------ .../complex-deinterleaving-reductions.ll | 17 +- llvm/test/CodeGen/AArch64/zext-to-tbl.ll | 195 +++++++++--------- .../LoopStrengthReduce/AArch64/lsr-reuse.ll | 6 +- 9 files changed, 248 insertions(+), 244 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index f49c73dc7951..9f5756fc7e40 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -58,6 +58,9 @@ static cl::opt InlineCallPenaltyChangeSM( static cl::opt EnableOrLikeSelectOpt("enable-aarch64-or-like-select", cl::init(true), cl::Hidden); +static cl::opt EnableLSRCostOpt("enable-aarch64-lsr-cost-opt", + cl::init(true), cl::Hidden); + namespace { class TailFoldingOption { // These bitfields will only ever be set to something non-zero in operator=, @@ -4216,3 +4219,19 @@ bool AArch64TTIImpl::shouldTreatInstructionLikeSelect(const Instruction *I) { return true; return BaseT::shouldTreatInstructionLikeSelect(I); } + +bool AArch64TTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1, + const TargetTransformInfo::LSRCost &C2) { + // AArch64 specific here is adding the number of instructions to the + // comparison (though not as the first consideration, as some targets do) + // along with changing the priority of the base additions. + // TODO: Maybe a more nuanced tradeoff between instruction count + // and number of registers? To be investigated at a later date. + if (EnableLSRCostOpt) + return std::tie(C1.NumRegs, C1.Insns, C1.NumBaseAdds, C1.AddRecCost, + C1.NumIVMuls, C1.ScaleCost, C1.ImmCost, C1.SetupCost) < + std::tie(C2.NumRegs, C2.Insns, C2.NumBaseAdds, C2.AddRecCost, + C2.NumIVMuls, C2.ScaleCost, C2.ImmCost, C2.SetupCost); + + return TargetTransformInfoImplBase::isLSRCostLess(C1, C2); +} \ No newline at end of file diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index 2f44aaa3e26a..feec1a4289c3 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -425,6 +425,9 @@ public: } std::optional getMinPageSize() const { return 4096; } + + bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1, + const TargetTransformInfo::LSRCost &C2); }; } // end namespace llvm diff --git a/llvm/test/CodeGen/AArch64/arm64-2011-10-18-LdStOptBug.ll b/llvm/test/CodeGen/AArch64/arm64-2011-10-18-LdStOptBug.ll index 3b6c4fa875e6..dafdcf82f311 100644 --- a/llvm/test/CodeGen/AArch64/arm64-2011-10-18-LdStOptBug.ll +++ b/llvm/test/CodeGen/AArch64/arm64-2011-10-18-LdStOptBug.ll @@ -12,7 +12,7 @@ entry: for.body: ; CHECK: for.body -; CHECK: ldr w{{[0-9]+}}, [x{{[0-9]+}}, x{{[0-9]+}}] +; CHECK: ldr w{{[0-9]+}}, [x{{[0-9]+}}] ; CHECK: add x[[REG:[0-9]+]], ; CHECK: x[[REG]], #1, lsl #12 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] diff --git a/llvm/test/CodeGen/AArch64/arm64-ldp-cluster.ll b/llvm/test/CodeGen/AArch64/arm64-ldp-cluster.ll index 8c7b31fd34c4..114203e46f19 100644 --- a/llvm/test/CodeGen/AArch64/arm64-ldp-cluster.ll +++ b/llvm/test/CodeGen/AArch64/arm64-ldp-cluster.ll @@ -176,13 +176,13 @@ exit: ; CHECK: ********** MI Scheduling ********** ; CHECK: LDURDi_LDRDui:%bb.1 vector_body ; -; CHECK: Cluster ld/st SU(2) - SU(6) -; CHECK: Cluster ld/st SU(3) - SU(7) +; CHECK: Cluster ld/st SU(0) - SU(4) +; CHECK: Cluster ld/st SU(1) - SU(5) ; -; CHECK: SU(2): %{{[0-9]+}}:fpr64 = LDURDi -; CHECK: SU(3): %{{[0-9]+}}:fpr64 = LDURDi -; CHECK: SU(6): %{{[0-9]+}}:fpr64 = LDRDui -; CHECK: SU(7): %{{[0-9]+}}:fpr64 = LDRDui +; CHECK: SU(0): %{{[0-9]+}}:fpr64 = LDURDi +; CHECK: SU(1): %{{[0-9]+}}:fpr64 = LDURDi +; CHECK: SU(4): %{{[0-9]+}}:fpr64 = LDRDui +; CHECK: SU(5): %{{[0-9]+}}:fpr64 = LDRDui ; define void @LDURDi_LDRDui(ptr nocapture readonly %arg) { entry: diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll index ac2b21af29ab..2ef35283568c 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll @@ -15,36 +15,34 @@ define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_v2f64: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: mov w9, #100 // =0x64 -; CHECK-NEXT: cntd x10 -; CHECK-NEXT: whilelo p1.d, xzr, x9 -; CHECK-NEXT: mov x8, xzr -; CHECK-NEXT: rdvl x11, #2 +; CHECK-NEXT: mov w8, #100 // =0x64 +; CHECK-NEXT: cntd x9 +; CHECK-NEXT: whilelo p1.d, xzr, x8 +; CHECK-NEXT: rdvl x10, #2 +; CHECK-NEXT: mov x11, x9 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov x12, x10 ; CHECK-NEXT: zip2 z0.d, z1.d, z1.d ; CHECK-NEXT: zip1 z1.d, z1.d, z1.d ; CHECK-NEXT: .LBB0_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: zip2 p2.d, p1.d, p1.d -; CHECK-NEXT: add x13, x0, x8 -; CHECK-NEXT: add x14, x1, x8 -; CHECK-NEXT: zip1 p1.d, p1.d, p1.d ; CHECK-NEXT: mov z6.d, z1.d ; CHECK-NEXT: mov z7.d, z0.d -; CHECK-NEXT: ld1d { z2.d }, p2/z, [x13, #1, mul vl] -; CHECK-NEXT: ld1d { z4.d }, p2/z, [x14, #1, mul vl] -; CHECK-NEXT: add x8, x8, x11 -; CHECK-NEXT: ld1d { z3.d }, p1/z, [x13] -; CHECK-NEXT: ld1d { z5.d }, p1/z, [x14] +; CHECK-NEXT: zip1 p1.d, p1.d, p1.d +; CHECK-NEXT: ld1d { z2.d }, p2/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z4.d }, p2/z, [x1, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z5.d }, p1/z, [x1] +; CHECK-NEXT: add x1, x1, x10 +; CHECK-NEXT: add x0, x0, x10 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #0 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #90 ; CHECK-NEXT: mov z0.d, p2/m, z7.d ; CHECK-NEXT: mov z1.d, p1/m, z6.d -; CHECK-NEXT: whilelo p1.d, x12, x9 -; CHECK-NEXT: add x12, x12, x10 +; CHECK-NEXT: whilelo p1.d, x11, x8 +; CHECK-NEXT: add x11, x11, x9 ; CHECK-NEXT: b.mi .LBB0_1 ; CHECK-NEXT: // %bb.2: // %exit.block ; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d @@ -114,39 +112,37 @@ define %"class.std::complex" @complex_mul_predicated_v2f64(ptr %a, ptr %b, ptr % ; CHECK-LABEL: complex_mul_predicated_v2f64: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: cntd x10 -; CHECK-NEXT: mov w12, #100 // =0x64 -; CHECK-NEXT: neg x11, x10 +; CHECK-NEXT: cntd x9 +; CHECK-NEXT: mov w11, #100 // =0x64 +; CHECK-NEXT: neg x10, x9 ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr -; CHECK-NEXT: mov x9, xzr -; CHECK-NEXT: and x11, x11, x12 -; CHECK-NEXT: rdvl x12, #2 +; CHECK-NEXT: and x10, x10, x11 +; CHECK-NEXT: rdvl x11, #2 ; CHECK-NEXT: zip2 z0.d, z1.d, z1.d ; CHECK-NEXT: zip1 z1.d, z1.d, z1.d ; CHECK-NEXT: .LBB1_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ld1w { z2.d }, p0/z, [x2, x9, lsl #2] -; CHECK-NEXT: add x13, x0, x8 -; CHECK-NEXT: add x14, x1, x8 +; CHECK-NEXT: ld1w { z2.d }, p0/z, [x2, x8, lsl #2] ; CHECK-NEXT: mov z6.d, z1.d ; CHECK-NEXT: mov z7.d, z0.d -; CHECK-NEXT: add x9, x9, x10 -; CHECK-NEXT: add x8, x8, x12 -; CHECK-NEXT: cmpne p2.d, p0/z, z2.d, #0 -; CHECK-NEXT: cmp x11, x9 -; CHECK-NEXT: zip2 p1.d, p2.d, p2.d -; CHECK-NEXT: zip1 p2.d, p2.d, p2.d -; CHECK-NEXT: ld1d { z2.d }, p1/z, [x13, #1, mul vl] -; CHECK-NEXT: ld1d { z4.d }, p1/z, [x14, #1, mul vl] -; CHECK-NEXT: ld1d { z3.d }, p2/z, [x13] -; CHECK-NEXT: ld1d { z5.d }, p2/z, [x14] +; CHECK-NEXT: add x8, x8, x9 +; CHECK-NEXT: cmpne p1.d, p0/z, z2.d, #0 +; CHECK-NEXT: cmp x10, x8 +; CHECK-NEXT: zip2 p2.d, p1.d, p1.d +; CHECK-NEXT: zip1 p1.d, p1.d, p1.d +; CHECK-NEXT: ld1d { z2.d }, p2/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z4.d }, p2/z, [x1, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z5.d }, p1/z, [x1] +; CHECK-NEXT: add x1, x1, x11 +; CHECK-NEXT: add x0, x0, x11 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #0 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, p1/m, z7.d -; CHECK-NEXT: mov z1.d, p2/m, z6.d +; CHECK-NEXT: mov z0.d, p2/m, z7.d +; CHECK-NEXT: mov z1.d, p1/m, z6.d ; CHECK-NEXT: b.ne .LBB1_1 ; CHECK-NEXT: // %bb.2: // %exit.block ; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d @@ -218,38 +214,38 @@ define %"class.std::complex" @complex_mul_predicated_x2_v2f64(ptr %a, ptr %b, pt ; CHECK-LABEL: complex_mul_predicated_x2_v2f64: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: mov w10, #100 // =0x64 +; CHECK-NEXT: mov w8, #100 // =0x64 +; CHECK-NEXT: cntd x9 +; CHECK-NEXT: whilelo p1.d, xzr, x8 +; CHECK-NEXT: rdvl x10, #2 +; CHECK-NEXT: cnth x11 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: whilelo p1.d, xzr, x10 -; CHECK-NEXT: mov x8, xzr -; CHECK-NEXT: mov x9, xzr -; CHECK-NEXT: cntd x11 -; CHECK-NEXT: rdvl x12, #2 +; CHECK-NEXT: mov x12, x9 ; CHECK-NEXT: zip2 z0.d, z1.d, z1.d ; CHECK-NEXT: zip1 z1.d, z1.d, z1.d ; CHECK-NEXT: .LBB2_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ld1w { z2.d }, p1/z, [x2, x9, lsl #2] -; CHECK-NEXT: add x13, x0, x8 -; CHECK-NEXT: add x14, x1, x8 +; CHECK-NEXT: ld1w { z2.d }, p1/z, [x2] ; CHECK-NEXT: mov z6.d, z1.d ; CHECK-NEXT: mov z7.d, z0.d -; CHECK-NEXT: add x9, x9, x11 -; CHECK-NEXT: add x8, x8, x12 -; CHECK-NEXT: cmpne p2.d, p1/z, z2.d, #0 -; CHECK-NEXT: zip2 p1.d, p2.d, p2.d -; CHECK-NEXT: zip1 p2.d, p2.d, p2.d -; CHECK-NEXT: ld1d { z2.d }, p1/z, [x13, #1, mul vl] -; CHECK-NEXT: ld1d { z4.d }, p1/z, [x14, #1, mul vl] -; CHECK-NEXT: ld1d { z3.d }, p2/z, [x13] -; CHECK-NEXT: ld1d { z5.d }, p2/z, [x14] +; CHECK-NEXT: add x2, x2, x11 +; CHECK-NEXT: cmpne p1.d, p1/z, z2.d, #0 +; CHECK-NEXT: zip2 p2.d, p1.d, p1.d +; CHECK-NEXT: zip1 p1.d, p1.d, p1.d +; CHECK-NEXT: ld1d { z2.d }, p2/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z4.d }, p2/z, [x1, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z5.d }, p1/z, [x1] +; CHECK-NEXT: add x1, x1, x10 +; CHECK-NEXT: add x0, x0, x10 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #0 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, p1/m, z7.d -; CHECK-NEXT: whilelo p1.d, x9, x10 -; CHECK-NEXT: mov z1.d, p2/m, z6.d +; CHECK-NEXT: mov z0.d, p2/m, z7.d +; CHECK-NEXT: mov z1.d, p1/m, z6.d +; CHECK-NEXT: whilelo p1.d, x12, x8 +; CHECK-NEXT: add x12, x12, x9 ; CHECK-NEXT: b.mi .LBB2_1 ; CHECK-NEXT: // %bb.2: // %exit.block ; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll index af07519ad53d..8e26ef6b87ec 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll @@ -15,30 +15,27 @@ define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_v2f64: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: cntd x9 -; CHECK-NEXT: ptrue p1.b -; CHECK-NEXT: neg x10, x9 -; CHECK-NEXT: mov w11, #100 // =0x64 +; CHECK-NEXT: cntd x8 +; CHECK-NEXT: mov w10, #100 // =0x64 +; CHECK-NEXT: neg x9, x8 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov x8, xzr -; CHECK-NEXT: and x10, x10, x11 -; CHECK-NEXT: rdvl x11, #2 +; CHECK-NEXT: and x9, x9, x10 +; CHECK-NEXT: rdvl x10, #2 ; CHECK-NEXT: zip2 z0.d, z1.d, z1.d ; CHECK-NEXT: zip1 z1.d, z1.d, z1.d ; CHECK-NEXT: .LBB0_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: add x12, x0, x8 -; CHECK-NEXT: add x13, x1, x8 -; CHECK-NEXT: ld1b { z2.b }, p1/z, [x0, x8] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x12, #1, mul vl] -; CHECK-NEXT: ld1b { z4.b }, p1/z, [x1, x8] -; CHECK-NEXT: ld1d { z5.d }, p0/z, [x13, #1, mul vl] -; CHECK-NEXT: subs x10, x10, x9 -; CHECK-NEXT: add x8, x8, x11 -; CHECK-NEXT: fcmla z1.d, p0/m, z4.d, z2.d, #0 -; CHECK-NEXT: fcmla z0.d, p0/m, z5.d, z3.d, #0 -; CHECK-NEXT: fcmla z1.d, p0/m, z4.d, z2.d, #90 -; CHECK-NEXT: fcmla z0.d, p0/m, z5.d, z3.d, #90 +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0] +; CHECK-NEXT: subs x9, x9, x8 +; CHECK-NEXT: ld1d { z4.d }, p0/z, [x1, #1, mul vl] +; CHECK-NEXT: ld1d { z5.d }, p0/z, [x1] +; CHECK-NEXT: add x1, x1, x10 +; CHECK-NEXT: add x0, x0, x10 +; CHECK-NEXT: fcmla z1.d, p0/m, z5.d, z3.d, #0 +; CHECK-NEXT: fcmla z0.d, p0/m, z4.d, z2.d, #0 +; CHECK-NEXT: fcmla z1.d, p0/m, z5.d, z3.d, #90 +; CHECK-NEXT: fcmla z0.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: b.ne .LBB0_1 ; CHECK-NEXT: // %bb.2: // %exit.block ; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d @@ -103,34 +100,31 @@ define %"class.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) { ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: fmov d0, #1.00000000 ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: cntd x9 +; CHECK-NEXT: cntd x8 ; CHECK-NEXT: fmov d2, #2.00000000 ; CHECK-NEXT: ptrue p0.d, vl1 -; CHECK-NEXT: neg x10, x9 -; CHECK-NEXT: ptrue p1.b -; CHECK-NEXT: mov w11, #100 // =0x64 -; CHECK-NEXT: mov x8, xzr +; CHECK-NEXT: neg x9, x8 +; CHECK-NEXT: mov w10, #100 // =0x64 ; CHECK-NEXT: sel z3.d, p0, z0.d, z1.d -; CHECK-NEXT: and x10, x10, x11 -; CHECK-NEXT: rdvl x11, #2 +; CHECK-NEXT: and x9, x9, x10 +; CHECK-NEXT: rdvl x10, #2 ; CHECK-NEXT: mov z1.d, p0/m, z2.d ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: zip2 z0.d, z1.d, z3.d ; CHECK-NEXT: zip1 z1.d, z1.d, z3.d ; CHECK-NEXT: .LBB1_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: add x12, x0, x8 -; CHECK-NEXT: add x13, x1, x8 -; CHECK-NEXT: ld1b { z2.b }, p1/z, [x0, x8] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x12, #1, mul vl] -; CHECK-NEXT: ld1b { z4.b }, p1/z, [x1, x8] -; CHECK-NEXT: ld1d { z5.d }, p0/z, [x13, #1, mul vl] -; CHECK-NEXT: subs x10, x10, x9 -; CHECK-NEXT: add x8, x8, x11 -; CHECK-NEXT: fcmla z1.d, p0/m, z4.d, z2.d, #0 -; CHECK-NEXT: fcmla z0.d, p0/m, z5.d, z3.d, #0 -; CHECK-NEXT: fcmla z1.d, p0/m, z4.d, z2.d, #90 -; CHECK-NEXT: fcmla z0.d, p0/m, z5.d, z3.d, #90 +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0] +; CHECK-NEXT: subs x9, x9, x8 +; CHECK-NEXT: ld1d { z4.d }, p0/z, [x1, #1, mul vl] +; CHECK-NEXT: ld1d { z5.d }, p0/z, [x1] +; CHECK-NEXT: add x1, x1, x10 +; CHECK-NEXT: add x0, x0, x10 +; CHECK-NEXT: fcmla z1.d, p0/m, z5.d, z3.d, #0 +; CHECK-NEXT: fcmla z0.d, p0/m, z4.d, z2.d, #0 +; CHECK-NEXT: fcmla z1.d, p0/m, z5.d, z3.d, #90 +; CHECK-NEXT: fcmla z0.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: b.ne .LBB1_1 ; CHECK-NEXT: // %bb.2: // %exit.block ; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d @@ -190,45 +184,37 @@ define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_v2f64_unrolled: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: cntw x9 -; CHECK-NEXT: mov w11, #1000 // =0x3e8 -; CHECK-NEXT: neg x10, x9 -; CHECK-NEXT: rdvl x12, #2 -; CHECK-NEXT: ptrue p1.b +; CHECK-NEXT: cntw x8 +; CHECK-NEXT: mov w10, #1000 // =0x3e8 +; CHECK-NEXT: neg x9, x8 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov x8, xzr -; CHECK-NEXT: and x10, x10, x11 +; CHECK-NEXT: and x9, x9, x10 +; CHECK-NEXT: rdvl x10, #4 ; CHECK-NEXT: zip2 z0.d, z1.d, z1.d ; CHECK-NEXT: zip1 z1.d, z1.d, z1.d -; CHECK-NEXT: add x11, x1, x12 -; CHECK-NEXT: add x12, x0, x12 -; CHECK-NEXT: rdvl x13, #4 ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: .LBB2_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: add x14, x0, x8 -; CHECK-NEXT: add x15, x12, x8 -; CHECK-NEXT: add x16, x1, x8 -; CHECK-NEXT: add x17, x11, x8 -; CHECK-NEXT: ld1b { z4.b }, p1/z, [x0, x8] -; CHECK-NEXT: ld1d { z5.d }, p0/z, [x14, #1, mul vl] -; CHECK-NEXT: ld1b { z6.b }, p1/z, [x12, x8] -; CHECK-NEXT: ld1b { z7.b }, p1/z, [x1, x8] -; CHECK-NEXT: ld1d { z16.d }, p0/z, [x16, #1, mul vl] -; CHECK-NEXT: ld1d { z17.d }, p0/z, [x15, #1, mul vl] -; CHECK-NEXT: ld1b { z18.b }, p1/z, [x11, x8] -; CHECK-NEXT: ld1d { z19.d }, p0/z, [x17, #1, mul vl] -; CHECK-NEXT: subs x10, x10, x9 -; CHECK-NEXT: add x8, x8, x13 -; CHECK-NEXT: fcmla z1.d, p0/m, z7.d, z4.d, #0 -; CHECK-NEXT: fcmla z0.d, p0/m, z16.d, z5.d, #0 -; CHECK-NEXT: fcmla z2.d, p0/m, z18.d, z6.d, #0 -; CHECK-NEXT: fcmla z3.d, p0/m, z19.d, z17.d, #0 -; CHECK-NEXT: fcmla z1.d, p0/m, z7.d, z4.d, #90 -; CHECK-NEXT: fcmla z0.d, p0/m, z16.d, z5.d, #90 -; CHECK-NEXT: fcmla z2.d, p0/m, z18.d, z6.d, #90 -; CHECK-NEXT: fcmla z3.d, p0/m, z19.d, z17.d, #90 +; CHECK-NEXT: ld1d { z4.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z5.d }, p0/z, [x0] +; CHECK-NEXT: subs x9, x9, x8 +; CHECK-NEXT: ld1d { z6.d }, p0/z, [x0, #3, mul vl] +; CHECK-NEXT: ld1d { z7.d }, p0/z, [x1, #1, mul vl] +; CHECK-NEXT: ld1d { z16.d }, p0/z, [x1] +; CHECK-NEXT: ld1d { z17.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: add x0, x0, x10 +; CHECK-NEXT: ld1d { z18.d }, p0/z, [x1, #3, mul vl] +; CHECK-NEXT: ld1d { z19.d }, p0/z, [x1, #2, mul vl] +; CHECK-NEXT: add x1, x1, x10 +; CHECK-NEXT: fcmla z1.d, p0/m, z16.d, z5.d, #0 +; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #0 +; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z6.d, #0 +; CHECK-NEXT: fcmla z2.d, p0/m, z19.d, z17.d, #0 +; CHECK-NEXT: fcmla z1.d, p0/m, z16.d, z5.d, #90 +; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #90 +; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z6.d, #90 +; CHECK-NEXT: fcmla z2.d, p0/m, z19.d, z17.d, #90 ; CHECK-NEXT: b.ne .LBB2_1 ; CHECK-NEXT: // %bb.2: // %exit.block ; CHECK-NEXT: uzp1 z4.d, z2.d, z3.d diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll index 44d0a9392ba6..aed3072bb4af 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll @@ -148,17 +148,16 @@ define %"struct.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) { ; CHECK-NEXT: adrp x8, .LCPI2_0 ; CHECK-NEXT: movi v3.2d, #0000000000000000 ; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI2_0] -; CHECK-NEXT: mov x8, xzr +; CHECK-NEXT: add x8, x0, #32 +; CHECK-NEXT: add x9, x1, #32 +; CHECK-NEXT: mov x10, #-100 // =0xffffffffffffff9c ; CHECK-NEXT: .LBB2_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: add x9, x0, x8 -; CHECK-NEXT: add x10, x1, x8 -; CHECK-NEXT: add x8, x8, #64 -; CHECK-NEXT: ldp q5, q4, [x9] -; CHECK-NEXT: cmp x8, #1600 -; CHECK-NEXT: ldp q7, q6, [x10] -; CHECK-NEXT: ldp q17, q16, [x9, #32] -; CHECK-NEXT: ldp q19, q18, [x10, #32] +; CHECK-NEXT: ldp q5, q4, [x8, #-32] +; CHECK-NEXT: adds x10, x10, #4 +; CHECK-NEXT: ldp q7, q6, [x9, #-32] +; CHECK-NEXT: ldp q17, q16, [x8], #64 +; CHECK-NEXT: ldp q19, q18, [x9], #64 ; CHECK-NEXT: fcmla v2.2d, v7.2d, v5.2d, #0 ; CHECK-NEXT: fcmla v0.2d, v6.2d, v4.2d, #0 ; CHECK-NEXT: fcmla v1.2d, v19.2d, v17.2d, #0 diff --git a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll index 599bd811d7d5..66bb131ce724 100644 --- a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll +++ b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll @@ -1669,42 +1669,41 @@ define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst) ; CHECK-LABEL: zext_v8i8_to_v8i64_with_add_in_sequence_in_loop: ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: Lloh18: -; CHECK-NEXT: adrp x9, lCPI17_0@PAGE +; CHECK-NEXT: adrp x8, lCPI17_0@PAGE ; CHECK-NEXT: Lloh19: -; CHECK-NEXT: adrp x10, lCPI17_1@PAGE -; CHECK-NEXT: mov x8, xzr +; CHECK-NEXT: adrp x9, lCPI17_1@PAGE +; CHECK-NEXT: mov w10, #128 ; =0x80 ; CHECK-NEXT: Lloh20: -; CHECK-NEXT: ldr q0, [x9, lCPI17_0@PAGEOFF] +; CHECK-NEXT: ldr q0, [x8, lCPI17_0@PAGEOFF] ; CHECK-NEXT: Lloh21: -; CHECK-NEXT: ldr q1, [x10, lCPI17_1@PAGEOFF] +; CHECK-NEXT: ldr q1, [x9, lCPI17_1@PAGEOFF] +; CHECK-NEXT: add x8, x1, #64 ; CHECK-NEXT: add x9, x0, #8 ; CHECK-NEXT: LBB17_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldp d2, d3, [x9, #-8] -; CHECK-NEXT: add x10, x1, x8 -; CHECK-NEXT: ldp q6, q5, [x10, #32] -; CHECK-NEXT: add x8, x8, #128 -; CHECK-NEXT: ldp q17, q16, [x10] -; CHECK-NEXT: cmp x8, #1024 +; CHECK-NEXT: subs x10, x10, #16 +; CHECK-NEXT: ldp q6, q5, [x8, #-32] +; CHECK-NEXT: add x9, x9, #16 +; CHECK-NEXT: ldp q17, q16, [x8, #-64] ; CHECK-NEXT: tbl.16b v4, { v2 }, v1 ; CHECK-NEXT: tbl.16b v2, { v2 }, v0 ; CHECK-NEXT: tbl.16b v7, { v3 }, v1 ; CHECK-NEXT: tbl.16b v3, { v3 }, v0 -; CHECK-NEXT: add x9, x9, #16 ; CHECK-NEXT: uaddw2.2d v5, v5, v4 ; CHECK-NEXT: uaddw.2d v4, v6, v4 ; CHECK-NEXT: uaddw2.2d v6, v16, v2 -; CHECK-NEXT: ldp q18, q16, [x10, #96] +; CHECK-NEXT: ldp q18, q16, [x8, #32] ; CHECK-NEXT: uaddw.2d v2, v17, v2 -; CHECK-NEXT: stp q4, q5, [x10, #32] +; CHECK-NEXT: stp q4, q5, [x8, #-32] ; CHECK-NEXT: uaddw2.2d v5, v16, v7 -; CHECK-NEXT: ldp q16, q4, [x10, #64] +; CHECK-NEXT: ldp q16, q4, [x8] ; CHECK-NEXT: uaddw.2d v7, v18, v7 -; CHECK-NEXT: stp q2, q6, [x10] +; CHECK-NEXT: stp q2, q6, [x8, #-64] ; CHECK-NEXT: uaddw2.2d v4, v4, v3 ; CHECK-NEXT: uaddw.2d v2, v16, v3 -; CHECK-NEXT: stp q7, q5, [x10, #96] -; CHECK-NEXT: stp q2, q4, [x10, #64] +; CHECK-NEXT: stp q7, q5, [x8, #32] +; CHECK-NEXT: stp q2, q4, [x8], #128 ; CHECK-NEXT: b.ne LBB17_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret @@ -1715,67 +1714,67 @@ define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst) ; CHECK-BE: // %bb.0: // %entry ; CHECK-BE-NEXT: adrp x9, .LCPI17_0 ; CHECK-BE-NEXT: add x9, x9, :lo12:.LCPI17_0 -; CHECK-BE-NEXT: mov x8, xzr +; CHECK-BE-NEXT: mov w8, #128 // =0x80 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x9] ; CHECK-BE-NEXT: adrp x9, .LCPI17_1 ; CHECK-BE-NEXT: add x9, x9, :lo12:.LCPI17_1 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x9] -; CHECK-BE-NEXT: add x9, x0, #8 +; CHECK-BE-NEXT: add x9, x1, #64 +; CHECK-BE-NEXT: add x10, x0, #8 ; CHECK-BE-NEXT: .LBB17_1: // %loop ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-BE-NEXT: sub x10, x9, #8 -; CHECK-BE-NEXT: ld1 { v2.8b }, [x9] -; CHECK-BE-NEXT: add x9, x9, #16 -; CHECK-BE-NEXT: ld1 { v3.8b }, [x10] -; CHECK-BE-NEXT: add x10, x1, x8 -; CHECK-BE-NEXT: add x8, x8, #128 -; CHECK-BE-NEXT: add x11, x10, #32 -; CHECK-BE-NEXT: add x14, x10, #64 -; CHECK-BE-NEXT: add x15, x10, #96 +; CHECK-BE-NEXT: ld1 { v2.8b }, [x10] +; CHECK-BE-NEXT: sub x11, x10, #8 +; CHECK-BE-NEXT: add x15, x9, #32 +; CHECK-BE-NEXT: ld1 { v3.8b }, [x11] +; CHECK-BE-NEXT: ld1 { v16.2d }, [x15] +; CHECK-BE-NEXT: sub x11, x9, #64 +; CHECK-BE-NEXT: sub x12, x9, #32 +; CHECK-BE-NEXT: ld1 { v6.2d }, [x9] +; CHECK-BE-NEXT: ld1 { v21.2d }, [x11] ; CHECK-BE-NEXT: tbl v4.16b, { v2.16b }, v1.16b ; CHECK-BE-NEXT: tbl v2.16b, { v2.16b }, v0.16b -; CHECK-BE-NEXT: ld1 { v5.2d }, [x10] -; CHECK-BE-NEXT: tbl v6.16b, { v3.16b }, v1.16b +; CHECK-BE-NEXT: ld1 { v19.2d }, [x12] +; CHECK-BE-NEXT: tbl v5.16b, { v3.16b }, v1.16b ; CHECK-BE-NEXT: tbl v3.16b, { v3.16b }, v0.16b -; CHECK-BE-NEXT: ld1 { v16.2d }, [x15] -; CHECK-BE-NEXT: ld1 { v19.2d }, [x14] -; CHECK-BE-NEXT: ld1 { v21.2d }, [x11] -; CHECK-BE-NEXT: add x12, x10, #48 -; CHECK-BE-NEXT: add x13, x10, #16 -; CHECK-BE-NEXT: add x16, x10, #112 -; CHECK-BE-NEXT: add x17, x10, #80 +; CHECK-BE-NEXT: sub x13, x9, #16 +; CHECK-BE-NEXT: sub x14, x9, #48 +; CHECK-BE-NEXT: add x16, x9, #48 +; CHECK-BE-NEXT: add x17, x9, #16 +; CHECK-BE-NEXT: ld1 { v22.2d }, [x13] +; CHECK-BE-NEXT: subs x8, x8, #16 +; CHECK-BE-NEXT: add x10, x10, #16 ; CHECK-BE-NEXT: rev32 v7.8b, v4.8b ; CHECK-BE-NEXT: ext v4.16b, v4.16b, v4.16b, #8 ; CHECK-BE-NEXT: rev32 v17.8b, v2.8b -; CHECK-BE-NEXT: ext v18.16b, v6.16b, v6.16b, #8 +; CHECK-BE-NEXT: ext v18.16b, v5.16b, v5.16b, #8 ; CHECK-BE-NEXT: ext v20.16b, v3.16b, v3.16b, #8 ; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8 -; CHECK-BE-NEXT: rev32 v6.8b, v6.8b +; CHECK-BE-NEXT: rev32 v5.8b, v5.8b ; CHECK-BE-NEXT: rev32 v3.8b, v3.8b -; CHECK-BE-NEXT: ld1 { v22.2d }, [x12] -; CHECK-BE-NEXT: cmp x8, #1024 -; CHECK-BE-NEXT: rev32 v4.8b, v4.8b ; CHECK-BE-NEXT: uaddw v7.2d, v16.2d, v7.2s -; CHECK-BE-NEXT: ld1 { v16.2d }, [x16] -; CHECK-BE-NEXT: rev32 v18.8b, v18.8b +; CHECK-BE-NEXT: rev32 v4.8b, v4.8b +; CHECK-BE-NEXT: uaddw v6.2d, v6.2d, v17.2s +; CHECK-BE-NEXT: rev32 v17.8b, v18.8b ; CHECK-BE-NEXT: rev32 v20.8b, v20.8b ; CHECK-BE-NEXT: rev32 v2.8b, v2.8b -; CHECK-BE-NEXT: uaddw v17.2d, v19.2d, v17.2s -; CHECK-BE-NEXT: ld1 { v19.2d }, [x13] -; CHECK-BE-NEXT: uaddw v6.2d, v21.2d, v6.2s -; CHECK-BE-NEXT: uaddw v3.2d, v5.2d, v3.2s -; CHECK-BE-NEXT: ld1 { v5.2d }, [x17] +; CHECK-BE-NEXT: ld1 { v16.2d }, [x16] +; CHECK-BE-NEXT: ld1 { v18.2d }, [x14] +; CHECK-BE-NEXT: uaddw v5.2d, v19.2d, v5.2s +; CHECK-BE-NEXT: uaddw v3.2d, v21.2d, v3.2s ; CHECK-BE-NEXT: st1 { v7.2d }, [x15] +; CHECK-BE-NEXT: ld1 { v7.2d }, [x17] +; CHECK-BE-NEXT: st1 { v6.2d }, [x9] +; CHECK-BE-NEXT: add x9, x9, #128 ; CHECK-BE-NEXT: uaddw v4.2d, v16.2d, v4.2s -; CHECK-BE-NEXT: st1 { v6.2d }, [x11] -; CHECK-BE-NEXT: uaddw v6.2d, v22.2d, v18.2s -; CHECK-BE-NEXT: st1 { v3.2d }, [x10] -; CHECK-BE-NEXT: uaddw v3.2d, v19.2d, v20.2s -; CHECK-BE-NEXT: uaddw v2.2d, v5.2d, v2.2s -; CHECK-BE-NEXT: st1 { v17.2d }, [x14] +; CHECK-BE-NEXT: st1 { v5.2d }, [x12] +; CHECK-BE-NEXT: uaddw v5.2d, v22.2d, v17.2s +; CHECK-BE-NEXT: st1 { v3.2d }, [x11] +; CHECK-BE-NEXT: uaddw v3.2d, v18.2d, v20.2s +; CHECK-BE-NEXT: uaddw v2.2d, v7.2d, v2.2s ; CHECK-BE-NEXT: st1 { v4.2d }, [x16] -; CHECK-BE-NEXT: st1 { v6.2d }, [x12] -; CHECK-BE-NEXT: st1 { v3.2d }, [x13] +; CHECK-BE-NEXT: st1 { v5.2d }, [x13] +; CHECK-BE-NEXT: st1 { v3.2d }, [x14] ; CHECK-BE-NEXT: st1 { v2.2d }, [x17] ; CHECK-BE-NEXT: b.ne .LBB17_1 ; CHECK-BE-NEXT: // %bb.2: // %exit @@ -1813,14 +1812,14 @@ exit: define void @zext_v16i8_to_v16i64_in_sequence_in_loop(ptr %src, ptr %dst) { ; CHECK-LABEL: zext_v16i8_to_v16i64_in_sequence_in_loop: ; CHECK: ; %bb.0: ; %entry -; CHECK-NEXT: mov x8, xzr +; CHECK-NEXT: mov w8, #128 ; =0x80 ; CHECK-NEXT: add x9, x1, #128 +; CHECK-NEXT: add x10, x0, #16 ; CHECK-NEXT: LBB18_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: add x10, x0, x8 -; CHECK-NEXT: add x8, x8, #16 -; CHECK-NEXT: ldp q0, q1, [x10] -; CHECK-NEXT: cmp x8, #128 +; CHECK-NEXT: ldp q0, q1, [x10, #-16] +; CHECK-NEXT: subs x8, x8, #16 +; CHECK-NEXT: add x10, x10, #16 ; CHECK-NEXT: ushll2.8h v2, v0, #0 ; CHECK-NEXT: ushll.8h v0, v0, #0 ; CHECK-NEXT: ushll2.8h v6, v1, #0 @@ -1863,18 +1862,18 @@ define void @zext_v16i8_to_v16i64_in_sequence_in_loop(ptr %src, ptr %dst) { ; ; CHECK-BE-LABEL: zext_v16i8_to_v16i64_in_sequence_in_loop: ; CHECK-BE: // %bb.0: // %entry -; CHECK-BE-NEXT: mov x8, xzr +; CHECK-BE-NEXT: mov w8, #128 // =0x80 ; CHECK-BE-NEXT: add x9, x1, #128 +; CHECK-BE-NEXT: add x10, x0, #16 ; CHECK-BE-NEXT: .LBB18_1: // %loop ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-BE-NEXT: add x10, x0, x8 -; CHECK-BE-NEXT: sub x11, x9, #32 -; CHECK-BE-NEXT: add x8, x8, #16 -; CHECK-BE-NEXT: ld1 { v0.16b }, [x10] -; CHECK-BE-NEXT: add x10, x10, #16 -; CHECK-BE-NEXT: cmp x8, #128 +; CHECK-BE-NEXT: sub x11, x10, #16 ; CHECK-BE-NEXT: ld1 { v5.16b }, [x10] -; CHECK-BE-NEXT: sub x10, x9, #16 +; CHECK-BE-NEXT: sub x12, x9, #32 +; CHECK-BE-NEXT: ld1 { v0.16b }, [x11] +; CHECK-BE-NEXT: sub x11, x9, #16 +; CHECK-BE-NEXT: subs x8, x8, #16 +; CHECK-BE-NEXT: add x10, x10, #16 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0 @@ -1885,54 +1884,54 @@ define void @zext_v16i8_to_v16i64_in_sequence_in_loop(ptr %src, ptr %dst) { ; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #0 ; CHECK-BE-NEXT: ushll2 v6.2d, v1.4s, #0 ; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-BE-NEXT: st1 { v4.2d }, [x10] +; CHECK-BE-NEXT: st1 { v4.2d }, [x11] ; CHECK-BE-NEXT: ushll2 v4.2d, v3.4s, #0 ; CHECK-BE-NEXT: ushll v3.2d, v3.2s, #0 -; CHECK-BE-NEXT: st1 { v2.2d }, [x11] +; CHECK-BE-NEXT: st1 { v2.2d }, [x12] ; CHECK-BE-NEXT: ushll2 v2.8h, v5.16b, #0 -; CHECK-BE-NEXT: sub x11, x9, #80 -; CHECK-BE-NEXT: sub x10, x9, #48 -; CHECK-BE-NEXT: st1 { v4.2d }, [x11] +; CHECK-BE-NEXT: sub x12, x9, #80 +; CHECK-BE-NEXT: sub x11, x9, #48 +; CHECK-BE-NEXT: st1 { v4.2d }, [x12] ; CHECK-BE-NEXT: ushll v4.8h, v5.8b, #0 -; CHECK-BE-NEXT: sub x11, x9, #64 +; CHECK-BE-NEXT: sub x12, x9, #64 ; CHECK-BE-NEXT: ushll2 v5.4s, v2.8h, #0 -; CHECK-BE-NEXT: st1 { v1.2d }, [x11] -; CHECK-BE-NEXT: sub x11, x9, #96 +; CHECK-BE-NEXT: st1 { v1.2d }, [x12] +; CHECK-BE-NEXT: sub x12, x9, #96 ; CHECK-BE-NEXT: ushll2 v1.2d, v0.4s, #0 ; CHECK-BE-NEXT: ushll v2.4s, v2.4h, #0 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-BE-NEXT: st1 { v6.2d }, [x10] -; CHECK-BE-NEXT: sub x10, x9, #128 -; CHECK-BE-NEXT: st1 { v3.2d }, [x11] +; CHECK-BE-NEXT: st1 { v6.2d }, [x11] +; CHECK-BE-NEXT: sub x11, x9, #128 +; CHECK-BE-NEXT: st1 { v3.2d }, [x12] ; CHECK-BE-NEXT: ushll2 v3.4s, v4.8h, #0 ; CHECK-BE-NEXT: ushll2 v6.2d, v5.4s, #0 -; CHECK-BE-NEXT: sub x11, x9, #112 +; CHECK-BE-NEXT: sub x12, x9, #112 ; CHECK-BE-NEXT: ushll v5.2d, v5.2s, #0 -; CHECK-BE-NEXT: st1 { v0.2d }, [x10] -; CHECK-BE-NEXT: st1 { v1.2d }, [x11] +; CHECK-BE-NEXT: st1 { v0.2d }, [x11] +; CHECK-BE-NEXT: st1 { v1.2d }, [x12] ; CHECK-BE-NEXT: ushll2 v1.2d, v2.4s, #0 -; CHECK-BE-NEXT: add x10, x9, #112 +; CHECK-BE-NEXT: add x11, x9, #112 ; CHECK-BE-NEXT: ushll v4.4s, v4.4h, #0 ; CHECK-BE-NEXT: ushll2 v0.2d, v3.4s, #0 -; CHECK-BE-NEXT: st1 { v6.2d }, [x10] -; CHECK-BE-NEXT: add x10, x9, #96 +; CHECK-BE-NEXT: st1 { v6.2d }, [x11] +; CHECK-BE-NEXT: add x11, x9, #96 ; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #0 ; CHECK-BE-NEXT: ushll v3.2d, v3.2s, #0 -; CHECK-BE-NEXT: st1 { v5.2d }, [x10] -; CHECK-BE-NEXT: add x10, x9, #80 -; CHECK-BE-NEXT: st1 { v1.2d }, [x10] -; CHECK-BE-NEXT: add x10, x9, #48 +; CHECK-BE-NEXT: st1 { v5.2d }, [x11] +; CHECK-BE-NEXT: add x11, x9, #80 +; CHECK-BE-NEXT: st1 { v1.2d }, [x11] +; CHECK-BE-NEXT: add x11, x9, #48 ; CHECK-BE-NEXT: ushll2 v1.2d, v4.4s, #0 -; CHECK-BE-NEXT: st1 { v0.2d }, [x10] +; CHECK-BE-NEXT: st1 { v0.2d }, [x11] ; CHECK-BE-NEXT: ushll v0.2d, v4.2s, #0 -; CHECK-BE-NEXT: add x10, x9, #64 -; CHECK-BE-NEXT: st1 { v2.2d }, [x10] -; CHECK-BE-NEXT: add x10, x9, #32 -; CHECK-BE-NEXT: st1 { v3.2d }, [x10] -; CHECK-BE-NEXT: add x10, x9, #16 +; CHECK-BE-NEXT: add x11, x9, #64 +; CHECK-BE-NEXT: st1 { v2.2d }, [x11] +; CHECK-BE-NEXT: add x11, x9, #32 +; CHECK-BE-NEXT: st1 { v3.2d }, [x11] +; CHECK-BE-NEXT: add x11, x9, #16 ; CHECK-BE-NEXT: st1 { v0.2d }, [x9] ; CHECK-BE-NEXT: add x9, x9, #128 -; CHECK-BE-NEXT: st1 { v1.2d }, [x10] +; CHECK-BE-NEXT: st1 { v1.2d }, [x11] ; CHECK-BE-NEXT: b.ne .LBB18_1 ; CHECK-BE-NEXT: // %bb.2: // %exit ; CHECK-BE-NEXT: ret diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-reuse.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-reuse.ll index 323e242620e6..64e8a6be998e 100644 --- a/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-reuse.ll +++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-reuse.ll @@ -2,7 +2,7 @@ declare void @foo(i64) -; Verify that redundant adds aren't inserted by LSR. +; Verify that redundant adds or geps aren't inserted by LSR. ; CHECK-LABEL: @bar( define void @bar(ptr %A) { entry: @@ -10,9 +10,11 @@ entry: while.cond: ; CHECK-LABEL: while.cond: -; CHECK: add i64 %lsr.iv, 1 ; CHECK-NOT: add i64 %lsr.iv, 1 ; CHECK-LABEL: land.rhs: +; CHECK: getelementptr i8, ptr %lsr.iv, i64 -8 +; CHECK-NOT: getelementptr i8, ptr %lsr.iv, i64 -8 +; CHECK-NOT: add i64, %lsr.iv, 1 %indvars.iv28 = phi i64 [ %indvars.iv.next29, %land.rhs ], [ 50, %entry ] %cmp = icmp sgt i64 %indvars.iv28, 0 br i1 %cmp, label %land.rhs, label %while.end -- GitLab From 79393124ff74aaaf6a43f7c88e67fd76a6e44239 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Thu, 6 Jun 2024 15:47:38 +0200 Subject: [PATCH 097/462] Fix -Wunused-variable in SemaAMDGPU.cpp in release build, NFC --- clang/lib/Sema/SemaAMDGPU.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clang/lib/Sema/SemaAMDGPU.cpp b/clang/lib/Sema/SemaAMDGPU.cpp index 51d4f0d3d964..d11bc9eec330 100644 --- a/clang/lib/Sema/SemaAMDGPU.cpp +++ b/clang/lib/Sema/SemaAMDGPU.cpp @@ -31,7 +31,8 @@ bool SemaAMDGPU::CheckAMDGCNBuiltinFunctionCall(unsigned BuiltinID, constexpr const int SizeIdx = 2; llvm::APSInt Size; Expr *ArgExpr = TheCall->getArg(SizeIdx); - ExprResult R = SemaRef.VerifyIntegerConstantExpression(ArgExpr, &Size); + [[maybe_unused]] ExprResult R = + SemaRef.VerifyIntegerConstantExpression(ArgExpr, &Size); assert(!R.isInvalid()); switch (Size.getSExtValue()) { case 1: -- GitLab From 00089f963fa333932126dd289aa2fabbb1c19f78 Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Thu, 6 Jun 2024 21:52:53 +0800 Subject: [PATCH 098/462] [NVPTX] Remove unused private field in NVVMIntrRange.cpp (NFC) /llvm-project/llvm/lib/Target/NVPTX/NVVMIntrRange.cpp:33:12: error: private field 'SmVersion' is not used [-Werror,-Wunused-private-field] unsigned SmVersion; ^ 1 error generated. --- llvm/lib/Target/NVPTX/NVVMIntrRange.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/llvm/lib/Target/NVPTX/NVVMIntrRange.cpp b/llvm/lib/Target/NVPTX/NVVMIntrRange.cpp index 9afce99ce79f..f9d21b38a7ec 100644 --- a/llvm/lib/Target/NVPTX/NVVMIntrRange.cpp +++ b/llvm/lib/Target/NVPTX/NVVMIntrRange.cpp @@ -30,8 +30,6 @@ namespace llvm { void initializeNVVMIntrRangePass(PassRegistry &); } namespace { class NVVMIntrRange : public FunctionPass { - unsigned SmVersion; - public: static char ID; NVVMIntrRange() : FunctionPass(ID) { -- GitLab From d9e6a563206cea4e682d4c52eccf24178d7d8343 Mon Sep 17 00:00:00 2001 From: Marianne Mailhot-Sarrasin <32456002+mariannems@users.noreply.github.com> Date: Thu, 6 Jun 2024 10:05:55 -0400 Subject: [PATCH 099/462] [lldb] Fix ThreadPlanStepOverRange name in log message (#94611) Co-authored-by: Marianne Mailhot-Sarrasin --- lldb/source/Target/ThreadPlanStepOverRange.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/source/Target/ThreadPlanStepOverRange.cpp b/lldb/source/Target/ThreadPlanStepOverRange.cpp index 84f282f1de52..3fe02e0bf4fa 100644 --- a/lldb/source/Target/ThreadPlanStepOverRange.cpp +++ b/lldb/source/Target/ThreadPlanStepOverRange.cpp @@ -355,7 +355,7 @@ bool ThreadPlanStepOverRange::DoPlanExplainsStop(Event *event_ptr) { return_value = NextRangeBreakpointExplainsStop(stop_info_sp); } else { if (log) - log->PutCString("ThreadPlanStepInRange got asked if it explains the " + log->PutCString("ThreadPlanStepOverRange got asked if it explains the " "stop for some reason other than step."); return_value = false; } -- GitLab From ff77f67c47b7f06334110b5d7d4b9350288832ba Mon Sep 17 00:00:00 2001 From: Joachim Date: Thu, 6 Jun 2024 16:13:50 +0200 Subject: [PATCH 100/462] [OpenMP][NFC] Fix warning for OpenMP standalone build (#93463) PR #75125 introduced upward propagation of some OMPT-related CMake variables. For stand-alone builds this results in a warning that `SCOPE_PARENT` has no meaning in a top-level directory. --- openmp/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt index c228a392e4c7..e565354ccc7f 100644 --- a/openmp/CMakeLists.txt +++ b/openmp/CMakeLists.txt @@ -137,8 +137,10 @@ if (OPENMP_ENABLE_OMPT_TOOLS) endif() # Propagate OMPT support to offload -set(LIBOMP_HAVE_OMPT_SUPPORT ${LIBOMP_HAVE_OMPT_SUPPORT} PARENT_SCOPE) -set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${LIBOMP_OMP_TOOLS_INCLUDE_DIR} PARENT_SCOPE) +if(NOT ${OPENMP_STANDALONE_BUILD}) + set(LIBOMP_HAVE_OMPT_SUPPORT ${LIBOMP_HAVE_OMPT_SUPPORT} PARENT_SCOPE) + set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${LIBOMP_OMP_TOOLS_INCLUDE_DIR} PARENT_SCOPE) +endif() option(OPENMP_MSVC_NAME_SCHEME "Build dll with MSVC naming scheme." OFF) -- GitLab From 2ec47e5e85f9c0e68eaeb7041bb9aa783bfb97d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Spaits?= Date: Thu, 6 Jun 2024 16:15:39 +0200 Subject: [PATCH 101/462] [RISCV] Fix duplicate test cases for G_UNMERGE_VALUES (#94622) `unmerge_i64` and `unmerge_i32` were exactly the same test cases. This PR would fix that, so `unmerge_i32` would actually unmerge a 32 bit value into two 16 bit values. --- .../RISCV/GlobalISel/legalizer/merge-unmerge-rv32.mir | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/merge-unmerge-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/merge-unmerge-rv32.mir index 2e4a39c46811..46a7df449558 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/merge-unmerge-rv32.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/merge-unmerge-rv32.mir @@ -68,12 +68,14 @@ body: | ; RV32: liveins: $x10 ; RV32-NEXT: {{ $}} ; RV32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 - ; RV32-NEXT: $x10 = COPY [[COPY]](s32) + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535 + ; RV32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]] + ; RV32-NEXT: $x10 = COPY [[AND]](s32) ; RV32-NEXT: PseudoRET implicit $x10 %0:_(s32) = COPY $x10 - %1:_(s64) = G_ZEXT %0(s32) - %2:_(s32), %3:_(s32) = G_UNMERGE_VALUES %1(s64) - $x10 = COPY %2(s32) + %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0(s32) + %4:_(s32) = G_ZEXT %2(s16) + $x10 = COPY %4(s32) PseudoRET implicit $x10 ... --- -- GitLab From a9205c5c9d5aeadbb97ed7283a35515df4ba49da Mon Sep 17 00:00:00 2001 From: Spenser Bauman Date: Thu, 6 Jun 2024 10:22:16 -0400 Subject: [PATCH 102/462] [mlir][tensor] Implement constant folder for tensor.pad (#92691) Extend the folding ability of the RewriteAsConstant patterns to include tensor.pad operations on constants. The new pattern with constant fold tensor.pad operations which operate on tensor constants and have statically resolvable padding sizes/values. %init = arith.constant dense<[[6, 7], [8, 9]]> : tensor<2x2xi32> %pad_value = arith.constant 0 : i32 %0 = tensor.pad %init low[1, 1] high[1, 1] { ^bb0(%arg1: index, %arg2: index): tensor.yield %pad_value : i32 } : tensor<2x2xi32> to tensor<4x4xi32> becomes %cst = arith.constant dense<[[0, 0, 0, 0], [0, 6, 7, 0], [0, 8, 9, 0], [0, 0, 0, 0]]> : tensor<4x4xi32> Co-authored-by: Spenser Bauman --- .../Tensor/TransformOps/TensorTransformOps.td | 4 +- .../Dialect/Tensor/Transforms/Transforms.h | 5 +- .../TransformOps/TensorTransformOps.cpp | 15 +- .../Tensor/Transforms/RewriteAsConstant.cpp | 165 +++++++++++++++++- mlir/lib/Dialect/Utils/IndexingUtils.cpp | 2 +- .../Dialect/Tensor/rewrite-as-constant.mlir | 135 ++++++++++++++ 6 files changed, 321 insertions(+), 5 deletions(-) diff --git a/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td index fea5afa0b7bb..81bab1b0c82f 100644 --- a/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td +++ b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td @@ -114,12 +114,14 @@ def ApplyReassociativeReshapeFoldingPatternsOp : Op]> { + let arguments = (ins UnitAttr:$aggressive); let description = [{ Indicates that tensor ops (such as tensor.generate) should be replaced with constants (arith.constant) when possible. }]; - let assemblyFormat = "attr-dict"; + let assemblyFormat = + "(`aggressive` $aggressive^)? attr-dict"; } def Transform_TensorPadOp : Transform_ConcreteOpType<"tensor.pad">; diff --git a/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h index 7dabc266c023..7f983b8b3cfd 100644 --- a/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h +++ b/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h @@ -91,9 +91,12 @@ void populateSimplifyPackAndUnpackPatterns(RewritePatternSet &patterns); /// respectively. void populateFoldIntoPackAndUnpackPatterns(RewritePatternSet &patterns); +using ControlFoldFn = std::function; + /// Populates `patterns` with patterns that replace tensor ops (such as /// tensor.generate) with constants when possible. -void populateRewriteAsConstantPatterns(RewritePatternSet &patterns); +void populateRewriteAsConstantPatterns(RewritePatternSet &patterns, + const ControlFoldFn &controlFn); //===----------------------------------------------------------------------===// // Transform helpers diff --git a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp index 5c6a32ce9a68..33016f84056e 100644 --- a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp +++ b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp @@ -127,7 +127,20 @@ void transform::ApplyReassociativeReshapeFoldingPatternsOp::populatePatterns( void transform::ApplyRewriteTensorOpsAsConstantPatternsOp::populatePatterns( RewritePatternSet &patterns) { - tensor::populateRewriteAsConstantPatterns(patterns); + ControlFoldFn defaultControlFn = [](OpOperand *fusedOperand) { + Operation *producer = fusedOperand->get().getDefiningOp(); + return producer && producer->hasOneUse(); + }; + + ControlFoldFn aggressiveControlFn = [](OpOperand *fusedOperand) { + return true; + }; + + // Add folding with reshape by expansion patterns. + if (getAggressive()) + tensor::populateRewriteAsConstantPatterns(patterns, aggressiveControlFn); + else + tensor::populateRewriteAsConstantPatterns(patterns, defaultControlFn); } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp b/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp index 11e1de543ac9..7c9fced540ad 100644 --- a/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp +++ b/mlir/lib/Dialect/Tensor/Transforms/RewriteAsConstant.cpp @@ -8,9 +8,12 @@ // #include "mlir/Dialect/Tensor/IR/Tensor.h" #include "mlir/Dialect/Tensor/Transforms/Transforms.h" +#include "mlir/Dialect/Utils/IndexingUtils.h" #include "mlir/IR/Matchers.h" #include "mlir/IR/PatternMatch.h" +#include "llvm/ADT/TypeSwitch.h" + using namespace mlir; using namespace mlir::tensor; @@ -45,9 +48,169 @@ struct GenerateToConstant : public OpRewritePattern { } }; +/// Transform a linear index from one indexing space to another given: +/// +/// - the shape of the source indexing space, +/// - the strides of the target indexing space, +/// - a linear index into the source indexing space. +/// +/// This function is logically a sequence of linearize/delinearize over +/// different bases but avoids allocating intermediate SmallVectors. +int64_t transformIndexSpace(ArrayRef inputShape, + ArrayRef outputStrides, + int64_t srcLinearIndex) { + assert(inputShape.size() == outputStrides.size()); + + int64_t dstLinearIndex = 0; + + for (int64_t dim = inputShape.size() - 1; dim >= 0; --dim) { + // Compute the index into the current dimension of the source tensor. + // `quotient` is the remaining linear index after accounting for the + // current dimension. + // + // `remainder` is the index into the source tensor for the current + // dimension. + auto [quotient, remainder] = std::div(srcLinearIndex, inputShape[dim]); + + srcLinearIndex = quotient; + + // Add the contribution of the current dimension to the output using the + // permutation map. + dstLinearIndex += outputStrides[dim] * remainder; + } + + return dstLinearIndex; +} + +template +Value constantFoldPadOp(PatternRewriter &rewriter, Location loc, + DenseElementsAttr input, AttrType padValue, + ArrayRef padLow, ArrayRef padHigh) { + auto inputValues = input.tryGetValues(); + if (failed(inputValues)) + return nullptr; + + auto oldShape = input.getType().getShape(); + + // Compute the output shape of the new value. + auto newShape = + llvm::map_to_vector(llvm::zip(oldShape, padLow, padHigh), + [](std::tuple pack) { + auto [old, low, high] = pack; + return old + low + high; + }); + + int64_t outputSize = computeProduct(newShape); + + // Fully initialize the vector with the padding value. + // The non-padded area will then be copied. + SmallVector values(outputSize, padValue.getValue()); + + // Strides for input and output are used to transform between the indexing + // space of the input and output tensors. + SmallVector outputStrides = computeStrides(newShape); + + // The contribution of the low padding to the offset in the output tensor. + // This is the starting position of the source tensor within the padding + // tensor. + int64_t startingOffset = linearize(padLow, outputStrides); + + // Copy values from the input tensor to the corresponding sub-region + // of the output tensor. + for (auto [inputIndex, inputValue] : llvm::enumerate(*inputValues)) { + auto outputIndex = transformIndexSpace(oldShape, outputStrides, inputIndex); + values[outputIndex + startingOffset] = inputValue; + } + + // Create an attribute for the folded value. + auto newType = input.getType().clone(newShape); + auto newAttr = DenseElementsAttr::get(newType, values); + + Operation *constantOp = + rewriter.getContext() + ->getLoadedDialect() + ->materializeConstant(rewriter, newAttr, newType, loc); + + return constantOp ? constantOp->getResult(0) : nullptr; +} + +struct PadOpToConstant final : public OpRewritePattern { + + PadOpToConstant(MLIRContext *context, const ControlFoldFn &controlFn, + PatternBenefit benefit = 1) + : OpRewritePattern(context, benefit), controlFn{controlFn} {} + + LogicalResult matchAndRewrite(PadOp padTensorOp, + PatternRewriter &rewriter) const override { + if (padTensorOp.getNofold()) + return rewriter.notifyMatchFailure( + padTensorOp, "refusing to fold nofold pad operation"); + + TypedValue input = padTensorOp.getSource(); + RankedTensorType resultType = padTensorOp.getResult().getType(); + + DenseElementsAttr inputAttr = nullptr; + if (!matchPattern(input, m_Constant(&inputAttr))) + return failure(); + + Value paddingValue = padTensorOp.getConstantPaddingValue(); + + // Extract the constant value used for padding or bail out. + Attribute paddingAttr = nullptr; + if (!paddingValue || !matchPattern(paddingValue, m_Constant(&paddingAttr))) + return rewriter.notifyMatchFailure(padTensorOp, + "unable to get constant value"); + + // Try to extract the constant values of the low and high padding. + auto lowPad = getConstantIntValues(padTensorOp.getMixedLowPad()); + auto highPad = getConstantIntValues(padTensorOp.getMixedHighPad()); + + // If the padding cannot be extracted, bail out. + if (!lowPad || !highPad) + return rewriter.notifyMatchFailure(padTensorOp, + "unable to extract constant padding"); + + // We have a potential candidate, consult the control function to + // determine if the op should fold. + if (!controlFn(&padTensorOp.getSourceMutable())) + return rewriter.notifyMatchFailure(padTensorOp, + "not folding due to cost function"); + + Location loc = padTensorOp.getLoc(); + + // Try constant folding the supported cases of integer and float values. + Value newOp = + llvm::TypeSwitch(paddingAttr) + .Case([&](FloatAttr floatAttr) { + return constantFoldPadOp( + rewriter, loc, inputAttr, floatAttr, *lowPad, *highPad); + }) + .Case([&](IntegerAttr integerAttr) { + return constantFoldPadOp( + rewriter, loc, inputAttr, integerAttr, *lowPad, *highPad); + }) + .Default(Value()); + + if (!newOp) + return rewriter.notifyMatchFailure(padTensorOp, + "tensor type not supported"); + + if (newOp.getType() != resultType) + newOp = rewriter.create(loc, resultType, newOp); + + rewriter.replaceOp(padTensorOp, newOp); + return success(); + } + +private: + ControlFoldFn controlFn; +}; + } // namespace void mlir::tensor::populateRewriteAsConstantPatterns( - RewritePatternSet &patterns) { + RewritePatternSet &patterns, const ControlFoldFn &controlFn) { patterns.add(patterns.getContext()); + + patterns.add(patterns.getContext(), controlFn); } diff --git a/mlir/lib/Dialect/Utils/IndexingUtils.cpp b/mlir/lib/Dialect/Utils/IndexingUtils.cpp index 4c960659d80c..aba225be720c 100644 --- a/mlir/lib/Dialect/Utils/IndexingUtils.cpp +++ b/mlir/lib/Dialect/Utils/IndexingUtils.cpp @@ -92,7 +92,7 @@ int64_t mlir::computeProduct(ArrayRef basis) { assert(llvm::all_of(basis, [](int64_t s) { return s > 0; }) && "basis must be nonnegative"); if (basis.empty()) - return 0; + return 1; return std::accumulate(basis.begin(), basis.end(), 1, std::multiplies()); } diff --git a/mlir/test/Dialect/Tensor/rewrite-as-constant.mlir b/mlir/test/Dialect/Tensor/rewrite-as-constant.mlir index 1a1cf9e407d8..35ee6f1caf0d 100644 --- a/mlir/test/Dialect/Tensor/rewrite-as-constant.mlir +++ b/mlir/test/Dialect/Tensor/rewrite-as-constant.mlir @@ -21,3 +21,138 @@ func.func @tensor_generate_constant() -> tensor<2x3x5xf32> { } : tensor<2x3x5xf32> return %0 : tensor<2x3x5xf32> } + +// CHECK-LABEL: func @pad_of_ints( +// CHECK: %[[cst:.*]] = arith.constant dense<[ +// CHECK-SAME{LITERAL}: [0, 0, 0, 0], +// CHECK-SAME{LITERAL}: [0, 6, 7, 0], +// CHECK-SAME{LITERAL}: [0, 8, 9, 0], +// CHECK-SAME{LITERAL}: [0, 0, 0, 0] +// CHECK-SAME{LITERAL}: ]> : tensor<4x4xi32> +// CHECK: %[[cast:.*]] = tensor.cast %[[cst]] : tensor<4x4xi32> to tensor +// CHECK: return %[[cast]] +func.func @pad_of_ints() -> tensor { + %init = arith.constant dense<[[6, 7], [8, 9]]> : tensor<2x2xi32> + %pad_value = arith.constant 0 : i32 + + %c1 = arith.constant 1 : index + + %0 = tensor.pad %init low[%c1, %c1] high[%c1, %c1] { + ^bb0(%arg1: index, %arg2: index): + tensor.yield %pad_value : i32 + } : tensor<2x2xi32> to tensor + + return %0 : tensor +} + +// CHECK-LABEL: func @pad_of_floats( +// CHECK: %[[cst:.*]] = arith.constant dense<[ +// CHECK-SAME{LITERAL}: [0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00], +// CHECK-SAME{LITERAL}: [0.000000e+00, 6.000000e+00, 7.000000e+00, 0.000000e+00], +// CHECK-SAME{LITERAL}: [0.000000e+00, 8.000000e+00, 9.000000e+00, 0.000000e+00], +// CHECK-SAME{LITERAL}: [0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00] +// CHECK-SAME{LITERAL}: ]> : tensor<4x4xf32> +// CHECK: return %[[cst]] + +func.func @pad_of_floats() -> tensor<4x4xf32> { + %init = arith.constant dense<[[6.0, 7.0], [8.0, 9.0]]> : tensor<2x2xf32> + %pad_value = arith.constant 0.0 : f32 + + %0 = tensor.pad %init low[1, 1] high[1, 1] { + ^bb0(%arg1: index, %arg2: index): + tensor.yield %pad_value : f32 + } : tensor<2x2xf32> to tensor<4x4xf32> + + return %0 : tensor<4x4xf32> +} + +// CHECK-LABEL: func @pad_of_ints_no_low_dims( +// CHECK: %[[cst:.*]] = arith.constant dense<[ +// CHECK-SAME{LITERAL}: [6, 7, 0], +// CHECK-SAME{LITERAL}: [8, 9, 0], +// CHECK-SAME{LITERAL}: [0, 0, 0] +// CHECK-SAME{LITERAL}: ]> : tensor<3x3xi32> +// CHECK: return %[[cst]] +func.func @pad_of_ints_no_low_dims() -> tensor<3x3xi32> { + %init = arith.constant dense<[[6, 7], [8, 9]]> : tensor<2x2xi32> + %pad_value = arith.constant 0 : i32 + + %0 = tensor.pad %init low[0, 0] high[1, 1] { + ^bb0(%arg1: index, %arg2: index): + tensor.yield %pad_value : i32 + } : tensor<2x2xi32> to tensor<3x3xi32> + + return %0 : tensor<3x3xi32> +} + +// CHECK-LABEL: func @pad_of_ints_no_high_dims( +// CHECK: %[[cst:.*]] = arith.constant dense<[ +// CHECK-SAME{LITERAL}: [0, 0, 0], +// CHECK-SAME{LITERAL}: [0, 6, 7], +// CHECK-SAME{LITERAL}: [0, 8, 9] +// CHECK-SAME{LITERAL}: ]> : tensor<3x3xi32> +// CHECK: return %[[cst]] +func.func @pad_of_ints_no_high_dims() -> tensor<3x3xi32> { + %init = arith.constant dense<[[6, 7], [8, 9]]> : tensor<2x2xi32> + %pad_value = arith.constant 0 : i32 + + %0 = tensor.pad %init low[1, 1] high[0, 0] { + ^bb0(%arg1: index, %arg2: index): + tensor.yield %pad_value : i32 + } : tensor<2x2xi32> to tensor<3x3xi32> + + return %0 : tensor<3x3xi32> +} + +// CHECK-LABEL: func @pad_multi_use_do_not_fold( +// CHECK: %[[pad:.+]] = tensor.pad +// CHECK: return %[[pad]] +func.func @pad_multi_use_do_not_fold() -> (tensor, tensor<2x2xi32>) { + %init = arith.constant dense<[[6, 7], [8, 9]]> : tensor<2x2xi32> + %pad_value = arith.constant 0 : i32 + + %c1 = arith.constant 1 : index + + %0 = tensor.pad %init low[%c1, %c1] high[%c1, %c1] { + ^bb0(%arg1: index, %arg2: index): + tensor.yield %pad_value : i32 + } : tensor<2x2xi32> to tensor + + return %0, %init : tensor, tensor<2x2xi32> +} + +// ----- + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> + transform.apply_patterns to %func_op { + transform.apply_patterns.tensor.rewrite_as_constant aggressive + } : !transform.op<"func.func"> + transform.yield + } +} + +// CHECK-LABEL: func @pad_aggressive_fold( +// CHECK: %[[init:.*]] = arith.constant dense<7> : tensor<2x2xi32> +// CHECK: %[[cst:.*]] = arith.constant dense<[ +// CHECK-SAME{LITERAL}: [0, 0, 0, 0], +// CHECK-SAME{LITERAL}: [0, 7, 7, 0], +// CHECK-SAME{LITERAL}: [0, 7, 7, 0], +// CHECK-SAME{LITERAL}: [0, 0, 0, 0] +// CHECK-SAME{LITERAL}: ]> : tensor<4x4xi32> +// CHECK: %[[cast:.*]] = tensor.cast %[[cst]] : tensor<4x4xi32> to tensor +// CHECK: return %[[cast]] +func.func @pad_aggressive_fold() -> (tensor, tensor<2x2xi32>) { + %init = arith.constant dense<7> : tensor<2x2xi32> + %pad_value = arith.constant 0 : i32 + + %c1 = arith.constant 1 : index + + %0 = tensor.pad %init low[%c1, %c1] high[%c1, %c1] { + ^bb0(%arg1: index, %arg2: index): + tensor.yield %pad_value : i32 + } : tensor<2x2xi32> to tensor + + return %0, %init : tensor, tensor<2x2xi32> +} -- GitLab From 2a6efe6a49e00a1953d537816a39e5c9883dc3c0 Mon Sep 17 00:00:00 2001 From: Sayhaan Siddiqui <49014204+sayhaan@users.noreply.github.com> Date: Thu, 6 Jun 2024 07:23:05 -0700 Subject: [PATCH 103/462] [BOLT][DWARF][NFC] Refactor GDB Index into a new file (#94405) Create a new class and file for functions that update GDB index. --- bolt/include/bolt/Core/GDBIndex.h | 61 ++++++++++ bolt/lib/Core/CMakeLists.txt | 1 + bolt/lib/Core/GDBIndex.cpp | 185 ++++++++++++++++++++++++++++++ 3 files changed, 247 insertions(+) create mode 100644 bolt/include/bolt/Core/GDBIndex.h create mode 100644 bolt/lib/Core/GDBIndex.cpp diff --git a/bolt/include/bolt/Core/GDBIndex.h b/bolt/include/bolt/Core/GDBIndex.h new file mode 100644 index 000000000000..6604c2a11472 --- /dev/null +++ b/bolt/include/bolt/Core/GDBIndex.h @@ -0,0 +1,61 @@ +//===-- bolt/Core/GDBIndex.h - GDB Index support ----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// This file contains declaration of classes required for generation of +/// .gdb_index section. +/// +//===----------------------------------------------------------------------===// + +#ifndef BOLT_CORE_GDB_INDEX_H +#define BOLT_CORE_GDB_INDEX_H + +#include "bolt/Core/BinaryContext.h" +#include + +namespace llvm { +namespace bolt { + +class GDBIndex { +public: + /// Contains information about TU so we can write out correct entries in GDB + /// index. + struct GDBIndexTUEntry { + uint64_t UnitOffset; + uint64_t TypeHash; + uint64_t TypeDIERelativeOffset; + }; + +private: + BinaryContext &BC; + + /// Entries for GDB Index Types CU List. + using GDBIndexTUEntryType = std::vector; + GDBIndexTUEntryType GDBIndexTUEntryVector; + +public: + GDBIndex(BinaryContext &BC) : BC(BC) {} + + std::mutex GDBIndexMutex; + + /// Adds an GDBIndexTUEntry if .gdb_index section exists. + void addGDBTypeUnitEntry(const GDBIndexTUEntry &&Entry); + + /// Rewrite .gdb_index section if present. + void updateGdbIndexSection(const CUOffsetMap &CUMap, const uint32_t NumCUs, + DebugARangesSectionWriter &ARangesSectionWriter); + + /// Returns all entries needed for Types CU list. + const GDBIndexTUEntryType &getGDBIndexTUEntryVector() const { + return GDBIndexTUEntryVector; + } +}; + +} // namespace bolt +} // namespace llvm + +#endif diff --git a/bolt/lib/Core/CMakeLists.txt b/bolt/lib/Core/CMakeLists.txt index 441df9fe0846..873cf67a5629 100644 --- a/bolt/lib/Core/CMakeLists.txt +++ b/bolt/lib/Core/CMakeLists.txt @@ -25,6 +25,7 @@ add_llvm_library(LLVMBOLTCore DynoStats.cpp Exceptions.cpp FunctionLayout.cpp + GDBIndex.cpp HashUtilities.cpp JumpTable.cpp MCPlusBuilder.cpp diff --git a/bolt/lib/Core/GDBIndex.cpp b/bolt/lib/Core/GDBIndex.cpp new file mode 100644 index 000000000000..9e6d24167d55 --- /dev/null +++ b/bolt/lib/Core/GDBIndex.cpp @@ -0,0 +1,185 @@ +//===- bolt/Core/GDBIndex.cpp - GDB Index support ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "bolt/Core/GDBIndex.h" + +using namespace llvm::bolt; +using namespace llvm::support::endian; + +void GDBIndex::addGDBTypeUnitEntry(const GDBIndexTUEntry &&Entry) { + std::lock_guard Lock(GDBIndexMutex); + if (!BC.getGdbIndexSection()) + return; + GDBIndexTUEntryVector.emplace_back(Entry); +} + +void GDBIndex::updateGdbIndexSection( + const CUOffsetMap &CUMap, const uint32_t NumCUs, + DebugARangesSectionWriter &ARangesSectionWriter) { + if (!BC.getGdbIndexSection()) + return; + + // See https://sourceware.org/gdb/onlinedocs/gdb/Index-Section-Format.html + // for .gdb_index section format. + + StringRef GdbIndexContents = BC.getGdbIndexSection()->getContents(); + + const char *Data = GdbIndexContents.data(); + + // Parse the header. + const uint32_t Version = read32le(Data); + if (Version != 7 && Version != 8) { + errs() << "BOLT-ERROR: can only process .gdb_index versions 7 and 8\n"; + exit(1); + } + + // Some .gdb_index generators use file offsets while others use section + // offsets. Hence we can only rely on offsets relative to each other, + // and ignore their absolute values. + const uint32_t CUListOffset = read32le(Data + 4); + const uint32_t CUTypesOffset = read32le(Data + 8); + const uint32_t AddressTableOffset = read32le(Data + 12); + const uint32_t SymbolTableOffset = read32le(Data + 16); + const uint32_t ConstantPoolOffset = read32le(Data + 20); + Data += 24; + + // Map CUs offsets to indices and verify existing index table. + std::map OffsetToIndexMap; + const uint32_t CUListSize = CUTypesOffset - CUListOffset; + const uint32_t TUListSize = AddressTableOffset - CUTypesOffset; + const unsigned NUmCUsEncoded = CUListSize / 16; + unsigned MaxDWARFVersion = BC.DwCtx->getMaxVersion(); + unsigned NumDWARF5TUs = + getGDBIndexTUEntryVector().size() - BC.DwCtx->getNumTypeUnits(); + bool SkipTypeUnits = false; + // For DWARF5 Types are in .debug_info. + // LLD doesn't generate Types CU List, and in CU list offset + // only includes CUs. + // GDB 11+ includes only CUs in CU list and generates Types + // list. + // GDB 9 includes CUs and TUs in CU list and generates TYpes + // list. The NumCUs is CUs + TUs, so need to modify the check. + // For split-dwarf + // GDB-11, DWARF5: TU units from dwo are not included. + // GDB-11, DWARF4: TU units from dwo are included. + if (MaxDWARFVersion >= 5) + SkipTypeUnits = !TUListSize ? true + : ((NUmCUsEncoded + NumDWARF5TUs) == + BC.DwCtx->getNumCompileUnits()); + + if (!((CUListSize == NumCUs * 16) || + (CUListSize == (NumCUs + NumDWARF5TUs) * 16))) { + errs() << "BOLT-ERROR: .gdb_index: CU count mismatch\n"; + exit(1); + } + DenseSet OriginalOffsets; + for (unsigned Index = 0, Units = BC.DwCtx->getNumCompileUnits(); + Index < Units; ++Index) { + const DWARFUnit *CU = BC.DwCtx->getUnitAtIndex(Index); + if (SkipTypeUnits && CU->isTypeUnit()) + continue; + const uint64_t Offset = read64le(Data); + Data += 16; + if (CU->getOffset() != Offset) { + errs() << "BOLT-ERROR: .gdb_index CU offset mismatch\n"; + exit(1); + } + + OriginalOffsets.insert(Offset); + OffsetToIndexMap[Offset] = Index; + } + + // Ignore old address table. + const uint32_t OldAddressTableSize = SymbolTableOffset - AddressTableOffset; + // Move Data to the beginning of symbol table. + Data += SymbolTableOffset - CUTypesOffset; + + // Calculate the size of the new address table. + uint32_t NewAddressTableSize = 0; + for (const auto &CURangesPair : ARangesSectionWriter.getCUAddressRanges()) { + const SmallVector &Ranges = CURangesPair.second; + NewAddressTableSize += Ranges.size() * 20; + } + + // Difference between old and new table (and section) sizes. + // Could be negative. + int32_t Delta = NewAddressTableSize - OldAddressTableSize; + + size_t NewGdbIndexSize = GdbIndexContents.size() + Delta; + + // Free'd by ExecutableFileMemoryManager. + auto *NewGdbIndexContents = new uint8_t[NewGdbIndexSize]; + uint8_t *Buffer = NewGdbIndexContents; + + write32le(Buffer, Version); + write32le(Buffer + 4, CUListOffset); + write32le(Buffer + 8, CUTypesOffset); + write32le(Buffer + 12, AddressTableOffset); + write32le(Buffer + 16, SymbolTableOffset + Delta); + write32le(Buffer + 20, ConstantPoolOffset + Delta); + Buffer += 24; + + using MapEntry = std::pair; + std::vector CUVector(CUMap.begin(), CUMap.end()); + // Need to sort since we write out all of TUs in .debug_info before CUs. + std::sort(CUVector.begin(), CUVector.end(), + [](const MapEntry &E1, const MapEntry &E2) -> bool { + return E1.second.Offset < E2.second.Offset; + }); + // Writing out CU List + for (auto &CUInfo : CUVector) { + // Skipping TU for DWARF5 when they are not included in CU list. + if (!OriginalOffsets.count(CUInfo.first)) + continue; + write64le(Buffer, CUInfo.second.Offset); + // Length encoded in CU doesn't contain first 4 bytes that encode length. + write64le(Buffer + 8, CUInfo.second.Length + 4); + Buffer += 16; + } + + // Rewrite TU CU List, since abbrevs can be different. + // Entry example: + // 0: offset = 0x00000000, type_offset = 0x0000001e, type_signature = + // 0x418503b8111e9a7b Spec says " triplet, the first value is the CU offset, + // the second value is the type offset in the CU, and the third value is the + // type signature" Looking at what is being generated by gdb-add-index. The + // first entry is TU offset, second entry is offset from it, and third entry + // is the type signature. + if (TUListSize) + for (const GDBIndexTUEntry &Entry : getGDBIndexTUEntryVector()) { + write64le(Buffer, Entry.UnitOffset); + write64le(Buffer + 8, Entry.TypeDIERelativeOffset); + write64le(Buffer + 16, Entry.TypeHash); + Buffer += sizeof(GDBIndexTUEntry); + } + + // Generate new address table. + for (const std::pair &CURangesPair : + ARangesSectionWriter.getCUAddressRanges()) { + const uint32_t CUIndex = OffsetToIndexMap[CURangesPair.first]; + const DebugAddressRangesVector &Ranges = CURangesPair.second; + for (const DebugAddressRange &Range : Ranges) { + write64le(Buffer, Range.LowPC); + write64le(Buffer + 8, Range.HighPC); + write32le(Buffer + 16, CUIndex); + Buffer += 20; + } + } + + const size_t TrailingSize = + GdbIndexContents.data() + GdbIndexContents.size() - Data; + assert(Buffer + TrailingSize == NewGdbIndexContents + NewGdbIndexSize && + "size calculation error"); + + // Copy over the rest of the original data. + memcpy(Buffer, Data, TrailingSize); + + // Register the new section. + BC.registerOrUpdateNoteSection(".gdb_index", NewGdbIndexContents, + NewGdbIndexSize); +} -- GitLab From 6b9753a0ecf7fdea203c6faf23c3ad4bf432273c Mon Sep 17 00:00:00 2001 From: Wei Zhao <60720283+wxz2020@users.noreply.github.com> Date: Thu, 6 Jun 2024 09:27:50 -0500 Subject: [PATCH 104/462] [AArch64] Add support for Qualcomm Oryon processor (#91022) Oryon is an ARM V8 AArch64 CPU from Qualcomm. --------- Co-authored-by: Wei Zhao --- clang/test/Driver/aarch64-oryon-1.c | 19 + clang/test/Misc/target-invalid-cpu-note.c | 4 +- .../llvm/TargetParser/AArch64TargetParser.h | 5 + llvm/lib/Target/AArch64/AArch64.td | 5 + llvm/lib/Target/AArch64/AArch64Processors.td | 30 + llvm/lib/Target/AArch64/AArch64SchedOryon.td | 1659 +++++++++++++++++ llvm/lib/Target/AArch64/AArch64Subtarget.cpp | 7 + llvm/lib/TargetParser/Host.cpp | 1 + llvm/unittests/TargetParser/Host.cpp | 3 + .../TargetParser/TargetParserTest.cpp | 16 +- 10 files changed, 1745 insertions(+), 4 deletions(-) create mode 100644 clang/test/Driver/aarch64-oryon-1.c create mode 100644 llvm/lib/Target/AArch64/AArch64SchedOryon.td diff --git a/clang/test/Driver/aarch64-oryon-1.c b/clang/test/Driver/aarch64-oryon-1.c new file mode 100644 index 000000000000..952ba5df74ba --- /dev/null +++ b/clang/test/Driver/aarch64-oryon-1.c @@ -0,0 +1,19 @@ +// RUN: %clang -target aarch64 -mcpu=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=Phoenix %s +// RUN: %clang -target aarch64 -mlittle-endian -mcpu=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=Phoenix %s +// RUN: %clang -target aarch64_be -mlittle-endian -mcpu=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=Phoenix %s +// RUN: %clang -target aarch64 -mtune=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=Phoenix-TUNE %s +// RUN: %clang -target aarch64 -mlittle-endian -mtune=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=Phoenix-TUNE %s +// RUN: %clang -target aarch64_be -mlittle-endian -mtune=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=Phoenix-TUNE %s +// Phoenix: "-cc1"{{.*}} "-triple" "aarch64{{(--)?}}"{{.*}} "-target-cpu" "oryon-1" "-target-feature" "+v8.6a" +// Phoenix-TUNE: "-cc1"{{.*}} "-triple" "aarch64{{(--)?}}"{{.*}} "-target-cpu" "generic" + +// RUN: %clang -target arm64 -mcpu=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=ARM64-Phoenix %s +// RUN: %clang -target arm64 -mlittle-endian -mcpu=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=ARM64-Phoenix %s +// RUN: %clang -target arm64 -mtune=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=ARM64-Phoenix-TUNE %s +// RUN: %clang -target arm64 -mlittle-endian -mtune=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=ARM64-Phoenix-TUNE %s +// ARM64-Phoenix: "-cc1"{{.*}} "-triple" "arm64{{.*}}" "-target-cpu" "oryon-1" "-target-feature" "+v8.6a" +// ARM64-Phoenix-TUNE: "-cc1"{{.*}} "-triple" "arm64{{.*}}" "-target-cpu" "generic" + +// RUN: %clang -target aarch64 -mcpu=oryon-1 -mtune=cortex-a53 -### -c %s 2>&1 | FileCheck -check-prefix=MCPU-MTUNE-Phoenix %s +// RUN: %clang -target aarch64 -mtune=cortex-a53 -mcpu=oryon-1 -### -c %s 2>&1 | FileCheck -check-prefix=MCPU-MTUNE-Phoenix %s +// MCPU-MTUNE-Phoenix: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "oryon-1" diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c index bad1374b9397..5ec2743c8d41 100644 --- a/clang/test/Misc/target-invalid-cpu-note.c +++ b/clang/test/Misc/target-invalid-cpu-note.c @@ -5,11 +5,11 @@ // RUN: not %clang_cc1 -triple arm64--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix AARCH64 // AARCH64: error: unknown target CPU 'not-a-cpu' -// AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a520ae, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-a720ae, cortex-r82, cortex-r82ae, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-n3, neoverse-512tvb, neoverse-v1, neoverse-v2, neoverse-v3, neoverse-v3ae, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, cobalt-100, grace{{$}} +// AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a520ae, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-a720ae, cortex-r82, cortex-r82ae, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-n3, neoverse-512tvb, neoverse-v1, neoverse-v2, neoverse-v3, neoverse-v3ae, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, oryon-1, cobalt-100, grace{{$}} // RUN: not %clang_cc1 -triple arm64--- -tune-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix TUNE_AARCH64 // TUNE_AARCH64: error: unknown target CPU 'not-a-cpu' -// TUNE_AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a520ae, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-a720ae, cortex-r82, cortex-r82ae, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-n3, neoverse-512tvb, neoverse-v1, neoverse-v2, neoverse-v3, neoverse-v3ae, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, cobalt-100, grace{{$}} +// TUNE_AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a520ae, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-a720ae, cortex-r82, cortex-r82ae, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-n3, neoverse-512tvb, neoverse-v1, neoverse-v2, neoverse-v3, neoverse-v3ae, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, oryon-1, cobalt-100, grace{{$}} // RUN: not %clang_cc1 -triple i386--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix X86 // X86: error: unknown target CPU 'not-a-cpu' diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h index 5025ab2491de..afe6789ceb25 100644 --- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h +++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h @@ -577,6 +577,11 @@ inline constexpr CpuInfo CpuInfos[] = { AArch64::AEK_SHA2, AArch64::AEK_AES, AArch64::AEK_MTE, AArch64::AEK_SB, AArch64::AEK_SSBS, AArch64::AEK_CSSC})}, + {"oryon-1", ARMV8_6A, + (AArch64::ExtensionBitset({AArch64::AEK_AES, AArch64::AEK_CRYPTO, + AArch64::AEK_RAND, AArch64::AEK_SM4, + AArch64::AEK_SHA3, AArch64::AEK_SHA2, + AArch64::AEK_PROFILE}))}, }; // Name alias. diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td index 4b2ce0d73949..5708b6173750 100644 --- a/llvm/lib/Target/AArch64/AArch64.td +++ b/llvm/lib/Target/AArch64/AArch64.td @@ -85,6 +85,10 @@ def SMEUnsupported : AArch64Unsupported { SME2Unsupported.F); } +def MTEUnsupported : AArch64Unsupported { + let F = [HasMTE]; +} + let F = [HasPAuth, HasPAuthLR] in def PAUnsupported : AArch64Unsupported; @@ -109,6 +113,7 @@ include "AArch64SchedNeoverseN1.td" include "AArch64SchedNeoverseN2.td" include "AArch64SchedNeoverseV1.td" include "AArch64SchedNeoverseV2.td" +include "AArch64SchedOryon.td" include "AArch64Processors.td" diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td index 8d16709114df..a759efcd9441 100644 --- a/llvm/lib/Target/AArch64/AArch64Processors.td +++ b/llvm/lib/Target/AArch64/AArch64Processors.td @@ -617,6 +617,27 @@ def TuneAmpere1B : SubtargetFeature<"ampere1b", "ARMProcFamily", "Ampere1B", FeatureLdpAlignedOnly, FeatureStpAlignedOnly]>; +def TuneOryon : SubtargetFeature<"oryon-1", "ARMProcFamily", + "Oryon", + "Nuvia Inc Oryon processors", [ + FeatureCrypto, + FeatureFPARMv8, + FeatureNEON, + FeatureFuseAES, + FeatureFuseAdrpAdd, + FeatureEnableSelectOptimize, + FeatureFuseCryptoEOR, + FeatureFuseAddress, + FeatureSM4, + FeatureSHA2, + FeatureSHA3, + FeatureAES, + FeatureFullFP16, + FeatureFP16FML, + FeaturePerfMon, + FeatureSPE, + FeaturePostRAScheduler, + HasV8_6aOps]>; def ProcessorFeatures { list A53 = [HasV8_0aOps, FeatureCRC, FeatureCrypto, @@ -806,6 +827,11 @@ def ProcessorFeatures { FeatureSHA3, FeatureAES, FeatureCSSC, FeatureWFxT, FeatureFullFP16]; + list Oryon = [HasV8_6aOps, FeatureNEON, FeaturePerfMon, + FeatureCrypto, FeatureRandGen, + FeaturePAuth, FeatureSM4, FeatureSHA2, + FeatureSHA3, FeatureAES]; + // ETE and TRBE are future architecture extensions. We temporarily enable them // by default for users targeting generic AArch64. The extensions do not // affect code generated by the compiler and can be used only by explicitly @@ -988,3 +1014,7 @@ def : ProcessorModel<"ampere1a", Ampere1Model, ProcessorFeatures.Ampere1A, def : ProcessorModel<"ampere1b", Ampere1BModel, ProcessorFeatures.Ampere1B, [TuneAmpere1B]>; + +// Qualcomm Oryon +def : ProcessorModel<"oryon-1", OryonModel, ProcessorFeatures.Oryon, + [TuneOryon]>; diff --git a/llvm/lib/Target/AArch64/AArch64SchedOryon.td b/llvm/lib/Target/AArch64/AArch64SchedOryon.td new file mode 100644 index 000000000000..09d1af248f0e --- /dev/null +++ b/llvm/lib/Target/AArch64/AArch64SchedOryon.td @@ -0,0 +1,1659 @@ +//=- AArch64SchedOryon.td - Qualcomm Oryon CPU 001 ---*- tablegen -*-=// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the scheduling model for Qualcomm Oryon +// family of processors. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// Pipeline Description. + +def OryonModel : SchedMachineModel { + let IssueWidth = 14; + let MicroOpBufferSize = 376; + let LoadLatency = 4; + let MispredictPenalty = 13; // 13 cycles for mispredicted branch. + let LoopMicroOpBufferSize = 0; // Do not have a LoopMicroOpBuffer + let PostRAScheduler = 1; // Using PostRA sched. + let CompleteModel = 1; + + list UnsupportedFeatures = !listconcat(SVEUnsupported.F, + SMEUnsupported.F, + MTEUnsupported.F, + PAUnsupported.F, + [HasPAuth, HasCSSC]); +} + +let SchedModel = OryonModel in { + +// Issue ports. +// IXU has 6 ports p0 ~ p5 +// LSU has 4 ports p6 ~ p9(ls0 ~ ls3), p10/p11(std0, std1) has to work with ls0~ls3 +// VXU has 4 ports p12 ~ p15 + +// cross IXU/LSU/VXU resource group for FMOV P41 of VXU +// I2V +def ORYONI4FP0 : ProcResource<1>; +def ORYONI5FP1 : ProcResource<1>; +// V2I +def ORYONFP0I4 : ProcResource<1>; +def ORYONFP1I5 : ProcResource<1>; + +// store 1 for normal store instructions +def ORYONST0 : ProcResource<1>; +// store 2 for normal store instructions +def ORYONST1 : ProcResource<1>; + +// Port 0: ALU/Indirect/Direct Branch. +def ORYONP0 : ProcResource<1>; + +// Port 1: ALU/Direct Branch. +def ORYONP1 : ProcResource<1>; + +// Port 2: ALU. +def ORYONP2 : ProcResource<1>; + +// Port 3: ALU. +def ORYONP3 : ProcResource<1>; + +// Port 4: ALU. +def ORYONP4 : ProcResource<1> { + let Super = ORYONI4FP0; + let Super = ORYONFP0I4; } + +// Port 5: ALU. +def ORYONP5 : ProcResource<1> { + let Super = ORYONI5FP1; + let Super = ORYONFP1I5; } + +// Port 6: Load/Store. LS0 +def ORYONP6 : ProcResource<1> { + let Super = ORYONST0; } + +// Port 7: Load/store. LS1 +def ORYONP7 : ProcResource<1> { + let Super = ORYONST0; } + +// Port 8: Load/Store. LS2 +def ORYONP8 : ProcResource<1> { + let Super = ORYONST1; } + +// Port 9: Load/store. LS3 +def ORYONP9 : ProcResource<1> { + let Super = ORYONST1; } + +// Port 10: Load/Store. STD0 +def ORYONP10SD0 : ProcResource<1> { + let Super = ORYONST0; } + +// Port 11: Load/store. STD1 +def ORYONP11SD1 : ProcResource<1> { + let Super = ORYONST1; } + +// Port 12: FP/Neon/SIMD/Crypto. +def ORYONP12FP0 : ProcResource<1> { + let Super = ORYONI4FP0; + let Super = ORYONFP0I4; } + +// Port 13: FP/Neon/SIMD/Crypto. +def ORYONP13FP1 : ProcResource<1> { + let Super = ORYONI5FP1; + let Super = ORYONFP1I5; } + +// Port 14: FP/Neon/SIMD/Crypto. +def ORYONP14FP2 : ProcResource<1>; + +// Port 15: FP/Neon/SIMD/Crypto. +def ORYONP15FP3 : ProcResource<1>; + +// Define groups for the functional units on each issue port. Each group +// created will be used by a WriteRes. + +// Integer add/shift/logical/misc. instructions on port I0/I1/I2/I3/I4/I5. +def ORYONI012345 : ProcResGroup<[ORYONP0, ORYONP1, ORYONP2, + ORYONP3, ORYONP4, ORYONP5]> { + let BufferSize = 120; +} + +// Direct Conditional Branch instructions on ports I0/I1. +def ORYONI01 : ProcResGroup<[ORYONP0, ORYONP1]> { + let BufferSize = 40; +} + +// Indirect/crypto Conditional Branch instructions on ports I0. +def ORYONI0 : ProcResGroup<[ORYONP0]> { + let BufferSize = 20; +} + +// Crypto/CRC/PAU instructions on ports I2. +def ORYONI2 : ProcResGroup<[ORYONP2]> { + let BufferSize = 20; +} + +// Multiply/Multiply-ADD instructions on ports I4/I5. +def ORYONI45 : ProcResGroup<[ORYONP4, ORYONP5]> { + let BufferSize = 40; +} + +// Divide instructions on ports I5. +def ORYONI5 : ProcResGroup<[ORYONP5]> { + let BufferSize = 20; +} + +// Comparison instructions on ports I0/I1/I2/I3. +def ORYONI0123 : ProcResGroup<[ORYONP0, ORYONP1, + ORYONP2, ORYONP3]> { + let BufferSize = 80; +} + +// Load instructions on ports P6/P7/P8/P9. +def ORYONLD : ProcResGroup<[ORYONP6, ORYONP7, ORYONP8, ORYONP9]> { + let BufferSize = 64; +} + +// Store instructions on combo of STA/STD pipes +def ORYONST : ProcResGroup<[ORYONST0, ORYONST1]> { + let BufferSize = 64; +} + +// Arithmetic and CRYP-AED ASIMD/FP instructions on ports FP0/FP1/FP2/FP3. +def ORYONFP0123 : ProcResGroup<[ORYONP12FP0, ORYONP13FP1, + ORYONP14FP2, ORYONP15FP3]> { + let BufferSize = 192; +} + +// FP Comparison and F/I move instructions on ports FP0/FP1. +def ORYONFP01 : ProcResGroup<[ORYONP12FP0, ORYONP13FP1]> { + let BufferSize = 96; +} + +// FDIV instructions on ports FP3. +def ORYONFP3 : ProcResGroup<[ORYONP15FP3]> { + let BufferSize = 48; +} + +// CRYP-SHA instructions on ports FP1. +def ORYONFP1 : ProcResGroup<[ORYONP14FP2]> { + let BufferSize = 48; +} + +def ORYONFP2 : ProcResGroup<[ORYONP14FP2]> { + let BufferSize = 48; +} + +// Reciprocal, Squre root on FP0. +def ORYONFP0 : ProcResGroup<[ORYONP12FP0]> { + let BufferSize = 48; +} + +// cross IXU/LSU/VXU resource group for FMOV P41 of VXU +// I2V +def ORYONI2V : ProcResGroup<[ORYONI4FP0, ORYONI5FP1]> { + let BufferSize = 40; +} + +// V2I +def ORYONV2I : ProcResGroup<[ORYONFP0I4, ORYONFP1I5]> { + let BufferSize = 96; +} + +// Define commonly used write types for InstRW specializations. +// All definitions follow the format: ORYONWrite_Cyc_. + +// Because of the complexity of Oryon CPU, we skip the following +// generic definitions and define each instruction specifically + +// These WriteRes entries are not used in the Falkor sched model. +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } +def : WriteRes { let Unsupported = 1; } + +// These ReadAdvance entries will be defined in later implementation +def : ReadAdvance; +def : ReadAdvance; +def : ReadAdvance; +def : ReadAdvance; +def : ReadAdvance; +def : ReadAdvance; +def : ReadAdvance; +def : ReadAdvance; +def : ReadAdvance; +def : ReadAdvance; + + +//IXU resource definition +// 1 cycles NO pipe +def ORYONWrite_1Cyc_NONE : SchedWriteRes<[]>; + +// 1 cycles on I01. +def ORYONWrite_1Cyc_I01 : SchedWriteRes<[ORYONI01]>; + +def ORYONWrite_1Cyc_2Uops_I01 : SchedWriteRes<[ORYONI01]> { + let NumMicroOps = 2; +} + +def ORYONWrite_1Cyc_I0 : SchedWriteRes<[ORYONI0]>; + +// 7 cycles on I2. PAC*/AUT* instructions +def ORYONWrite_7Cyc_I2 : SchedWriteRes<[ORYONI2]> { + let Latency = 7; +} + +// 7 cycles on I2. PAC*/AUT* instructions +def ORYONWrite_7Cyc_3Uops_I2 : SchedWriteRes<[ORYONI2]> { + let Latency = 7; + let NumMicroOps = 3; +} + +// 9 (7+1+1) cycles on I2 and I0/I1, I0. Authentication branch instructions +// these instructions are broken down to three uops +// a. PtrAuth on pipe 2 taking 7 cycles +// b. Link Register Update on pipes 0 and 1 taking 1 cycle +// c. Indirect branch on pipe 0 taking 1 cycle + +def ORYONWrite_9Cyc_I012 : SchedWriteRes<[ORYONI2, ORYONI01]> { + let Latency = 9; + let NumMicroOps = 3; +} + +// 3 cycles on I2. CRC32 and CRC32C instructions +def ORYONWrite_3Cyc_I2 : SchedWriteRes<[ORYONI2]> { + let Latency = 3; +} + +// 1 cycle on I012345 +def ORYONWrite_1Cyc_I012345 : SchedWriteRes<[ORYONI012345]>; + +// 1 cycle on I0123 +def ORYONWrite_1Cyc_I0123 : SchedWriteRes<[ORYONI0123]>; + +// 1 cycle on 2 of I012345 +def ORYONWrite_1Cyc_I012345_I012345 : +SchedWriteRes<[ORYONI012345, ORYONI012345]> ; + +// 2 cycle on 2 of I0123 with ReleaseAtCycles +def ORYONWrite_2Cyc_I0123_I0123_RC : +SchedWriteRes<[ORYONI0123, ORYONI0123]> { + let Latency = 2; + let ReleaseAtCycles = [2,2]; +} + +// 2 cycle on 2 of I012345 +def ORYONWrite_2Cyc_I012345_I012345_RC : +SchedWriteRes<[ORYONI012345, ORYONI012345]> { + let Latency = 2; + let ReleaseAtCycles = [2,2]; +} + +// 3 cycle on 2 of I45 +def ORYONWrite_3Cyc_I45_I45_RC : +SchedWriteRes<[ORYONI45, ORYONI45]> { + let Latency = 3; + let ReleaseAtCycles = [2,2]; +} + +// 3 cycle on I45 +def ORYONWrite_3Cyc_I45 : SchedWriteRes<[ORYONI45]> { + let Latency = 3; +} + +// 7 cycle on I2 32-bit integer division +def ORYONWrite_7Cyc_I2_RC : SchedWriteRes<[ORYONI2]> { + let Latency = 7; + let ReleaseAtCycles = [2]; +} + +// 9 cycle on I2 64-bit integer division +def ORYONWrite_9Cyc_I2_RC : SchedWriteRes<[ORYONI2]> { + let Latency = 9; + let ReleaseAtCycles = [2]; +} + +// LSU resource definition +// need to define WriteLDAdr, WriteAdrAdr, WriteLDHi, WriteSTX +// 4 cycle on LS(P6789) +def ORYONWrite_4Cyc_LD : SchedWriteRes<[ORYONLD]> { + let Latency = 4; +} + +// 4 cycle for Post/Pre inc/dec access, also covers all pair loads Post/Pre +def ORYONWrite_4Cyc_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> { + let Latency = 4; +} + +// 5 (4+1) for VXU SIMD access/could also include FP +// resource might not be correct, as VXU resource not included +def ORYONWrite_5Cyc_LD : SchedWriteRes<[ORYONLD]> { + let Latency = 5; +} + +def ORYONWrite_5Cyc_2Uops_LD : SchedWriteRes<[ORYONLD]> { + let Latency = 5; + let NumMicroOps = 2; +} + +def ORYONWrite_5Cyc_3Uops_LD : SchedWriteRes<[ORYONLD]> { + let Latency = 5; + let NumMicroOps = 3; +} + +def ORYONWrite_5Cyc_4Uops_LD : SchedWriteRes<[ORYONLD]> { + let Latency = 5; + let NumMicroOps = 4; +} + +def ORYONWrite_5Cyc_5Uops_LD : SchedWriteRes<[ORYONLD]> { + let Latency = 5; + let NumMicroOps = 5; +} + +def ORYONWrite_5Cyc_6Uops_LD : SchedWriteRes<[ORYONLD]> { + let Latency = 5; + let NumMicroOps = 6; +} + +def ORYONWrite_5Cyc_8Uops_LD : SchedWriteRes<[ORYONLD]> { + let Latency = 5; + let NumMicroOps = 8; +} + +def ORYONWrite_5Cyc_10Uops_LD : SchedWriteRes<[ORYONLD]> { + let Latency = 5; + let NumMicroOps = 10; +} + +// 6 cycle for Post/Pre inc/dec access +def ORYONWrite_5Cyc_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> { + let Latency = 5; +} + +def ORYONWrite_5Cyc_2Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> { + let Latency = 5; + let NumMicroOps = 2; +} + +def ORYONWrite_5Cyc_3Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> { + let Latency = 5; + let NumMicroOps = 3; +} + +def ORYONWrite_5Cyc_4Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> { + let Latency = 5; + let NumMicroOps = 4; +} + +def ORYONWrite_5Cyc_5Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> { + let Latency = 5; + let NumMicroOps = 5; +} + +def ORYONWrite_5Cyc_6Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> { + let Latency = 5; + let NumMicroOps = 6; +} + +def ORYONWrite_5Cyc_8Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> { + let Latency = 5; + let NumMicroOps = 8; +} + +def ORYONWrite_5Cyc_10Uops_LD_I012345 : SchedWriteRes<[ORYONLD, ORYONI012345]> { + let Latency = 5; + let NumMicroOps = 10; +} + +// 1 cycle for all generic stores +def ORYONWrite_1Cyc_ST : SchedWriteRes<[ORYONST]>; + +def ORYONWrite_1Cyc_2Uops_ST : SchedWriteRes<[ORYONST]> { + let NumMicroOps = 2; +} + +def ORYONWrite_1Cyc_3Uops_ST : SchedWriteRes<[ORYONST]> { + let NumMicroOps = 3; +} + +def ORYONWrite_1Cyc_4Uops_ST : SchedWriteRes<[ORYONST]> { + let NumMicroOps = 4; +} + +def ORYONWrite_1Cyc_5Uops_ST : SchedWriteRes<[ORYONST]> { + let NumMicroOps = 5; +} + +def ORYONWrite_1Cyc_6Uops_ST : SchedWriteRes<[ORYONST]> { + let NumMicroOps = 6; +} + +def ORYONWrite_1Cyc_8Uops_ST : SchedWriteRes<[ORYONST]> { + let NumMicroOps = 8; +} + +def ORYONWrite_1Cyc_10Uops_ST : SchedWriteRes<[ORYONST]> { + let NumMicroOps = 10; +} + +// 1 cycle for neon write: float + ASIMD with Post/Pre Inc/Dec access +// also includes Pair store until further informed +def ORYONWrite_1Cyc_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> { + let NumMicroOps = 3; +} + +def ORYONWrite_1Cyc_2Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> { + let NumMicroOps = 2; +} + +def ORYONWrite_1Cyc_3Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> { + let NumMicroOps = 3; +} + +def ORYONWrite_1Cyc_4Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> { + let NumMicroOps = 4; +} + +def ORYONWrite_1Cyc_5Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> { + let NumMicroOps = 5; +} + +def ORYONWrite_1Cyc_6Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> { + let NumMicroOps = 6; +} + +def ORYONWrite_1Cyc_8Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> { + let NumMicroOps = 8; +} + +def ORYONWrite_1Cyc_10Uops_ST_I012345 : SchedWriteRes<[ORYONST, ORYONI012345]> { + let NumMicroOps = 10; +} + +// VXU resource definition + +// I2V instruction has 1 uOp +// I2v with convert has 2 uOps +// all I2V, V2I's throughputs are 2 +// On VXU doc, p37 -- latencies and throughput +// P41, resource taken, P42, uOps +def ORYONWrite_I2V_4Cyc_I45 : SchedWriteRes<[ORYONI2V]> { + let Latency = 4; +} + +// inline a FCVT, so add one more uOp +def ORYONWrite_I2V_7Cyc_I45 : SchedWriteRes<[ORYONI2V]> { + let Latency = 7; + let NumMicroOps = 2; +} + +// V2I move instruction has 1/2 uOps, P42 in VXU doc +// Latency is 3, FCVT is also 3 cycle +// move + convert is 6 (3+3) cycles +// throughput is 2 +def ORYONWrite_V2I_3Cyc_FP01 : SchedWriteRes<[ORYONV2I]> { + let Latency = 3; +} + +// inline a FCVT, so add one more uOp +def ORYONWrite_V2I_6Cyc_FP01 : SchedWriteRes<[ORYONV2I]> { + let Latency = 6; + let NumMicroOps = 2; +} + +def ORYONWrite_V2V_2Cyc_FP0123 : SchedWriteRes<[ORYONFP0123]> { + let Latency = 2; +} + +def ORYONWrite_V2V_3Cyc_FP0123 : SchedWriteRes<[ORYONFP0123]> { + let Latency = 3; +} + +def ORYONWrite_V2V_6Cyc_FP01 : SchedWriteRes<[ORYONFP0123]> { + let Latency = 6; + let NumMicroOps = 3; +} + +def ORYONWrite_4Cyc_FP0123 : SchedWriteRes<[ORYONFP0123]> { + let Latency = 4; +} + +def ORYONWrite_3Cyc_FP0 : SchedWriteRes<[ORYONFP0]> { + let Latency = 3; +} + +def ORYONWrite_3Cyc_FP0123 : SchedWriteRes<[ORYONFP0123]> { + let Latency = 3; +} + +def ORYONWrite_3Cyc_2Uops_FP0123 : SchedWriteRes<[ORYONFP0123]> { + let Latency = 3; + let NumMicroOps = 2; +} + +def ORYONWrite_2Cyc_FP0123 : SchedWriteRes<[ORYONFP0123]> { + let Latency = 2; +} + +def ORYONWrite_2Cyc_FP01 : SchedWriteRes<[ORYONFP01]> { + let Latency = 2; +} + +// 2 cycle on FP1 +def ORYONWrite_2Cyc_FP1 : SchedWriteRes<[ORYONFP1]> { + let Latency = 2; +} + +// 3 cycle on FP1 +def ORYONWrite_3Cyc_FP1 : SchedWriteRes<[ORYONFP1]> { + let Latency = 3; +} + +// 4 cycle , 0.5 throughput on FP1 +def ORYONWrite_4Cyc_FP1_RC4 : SchedWriteRes<[ORYONFP1]> { + let Latency = 4; + let ReleaseAtCycles = [4]; +} + +// 5 cycle , 1 throughput on FP1 +def ORYONWrite_5Cyc_FP1 : SchedWriteRes<[ORYONFP1]> { + let Latency = 5; +} + +// 8 cycle , 2 throughput on FP0123 +def ORYONWrite_8Cyc_FP0123_RC : SchedWriteRes<[ORYONFP0123]> { + let Latency = 8; + let ReleaseAtCycles = [2]; +} + +def ORYONWrite_6Cyc_FP3 : SchedWriteRes<[ORYONFP3]> { + let Latency = 6; +} + +def ORYONWrite_7Cyc_FP3 : SchedWriteRes<[ORYONFP3]> { + let Latency = 7; +} + +def ORYONWrite_8Cyc_FP3 : SchedWriteRes<[ORYONFP3]> { + let Latency = 8; +} + +def ORYONWrite_9Cyc_FP3 : SchedWriteRes<[ORYONFP3]> { + let Latency = 9; +} + +def ORYONWrite_10Cyc_FP3 : SchedWriteRes<[ORYONFP3]> { + let Latency = 10; +} + +def ORYONWrite_8Cyc_FP3_RC : SchedWriteRes<[ORYONFP3]> { + let Latency = 8; + let ReleaseAtCycles = [2]; +} + +def ORYONWrite_10Cyc_FP3_RC : SchedWriteRes<[ORYONFP3]> { + let Latency = 10; + let ReleaseAtCycles = [2]; +} + +def ORYONWrite_13Cyc_FP3_RC : SchedWriteRes<[ORYONFP3]> { + let Latency = 13; + let ReleaseAtCycles = [2]; +} + +def ORYONWrite_4Cyc_FP0123_RC : +SchedWriteRes<[ORYONFP0123]> { + let Latency = 4; + let ReleaseAtCycles = [2]; +} + +def ORYONWrite_4Cyc_FP0123_FP0123_RC : +SchedWriteRes<[ORYONFP0123, ORYONFP0123]> { + let Latency = 4; + let NumMicroOps = 2; + let ReleaseAtCycles = [2,2]; +} + +def ORYONWrite_4Cyc_FP0123_FP0123_FP0123_RC : +SchedWriteRes<[ORYONFP0123, ORYONFP0123, ORYONFP0123]> { + let Latency = 4; + let NumMicroOps = 3; + let ReleaseAtCycles = [3,3,3]; +} + +def ORYONWrite_6Cyc_FP0123_FP0123_FP0123_FP0123_RC : +SchedWriteRes<[ORYONFP0123, ORYONFP0123, ORYONFP0123, ORYONFP0123]> { + let Latency = 6; + let NumMicroOps = 4; + let ReleaseAtCycles = [6,6,6,6]; +} + +//===----------------------------------------------------------------------===// +// Instruction Tables in IXU +//===----------------------------------------------------------------------===// + +//--- +// Arithmetic Instructions +//--- + +//1, 1, 6 +def : InstRW<[ORYONWrite_1Cyc_I012345], + (instregex "^ADD(W|X)r(i|r|x)", "^SUB(W|X)r(i|r|x)")>; + +//2,2,3 +def : InstRW<[ORYONWrite_2Cyc_I012345_I012345_RC], + (instregex "^ADD(W|X)rs", "^SUB(W|X)rs")>; + +//1,1,4 alias CMP, CMN on page 75 +def : InstRW<[ORYONWrite_1Cyc_I0123], + (instregex "^ADDS(W|X)r(i|r|x)(64)?", "^SUBS(W|X)r(i|r|x)")>; + +//2,2,2 alias CMP, CMN on page 75 +def : InstRW<[ORYONWrite_2Cyc_I0123_I0123_RC], + (instregex "^ADDS(W|X)rs", "^SUBS(W|X)rs")>; + +//1,1,4 +def : InstRW<[ORYONWrite_1Cyc_I0123], + (instregex "^ADC(W|X)r","^SBC(W|X)r", + "^ADCS(W|X)r","^SBCS(W|X)r")>; + +//1,1,2 +def : InstRW<[ORYONWrite_1Cyc_2Uops_I01], + (instrs ADR,ADRP)>; + +//1,1,4 +def : InstRW<[ORYONWrite_1Cyc_I0123], + (instregex "^CSEL(W|X)r", "^CSINV(W|X)r", + "^CSNEG(W|X)r", "^CSINC(W|X)r")>; + +//--- +//Compare Instruciton +//--- + +// We have CCMP, CCMN as LLVM DAG node +// CMP is an alias of SUBS as above +// CMN is an alias of ADDS as above +// We also have no way to get shift compare node in LLVM +//2,2,1.5 CMP, CMN + +//1,1,4 +def : InstRW<[ORYONWrite_1Cyc_I0123], + (instregex "^CCMP(W|X)(i|r)", "^CCMN(W|X)(i|r)")>; + +//--- +// Branch +//--- + +def : InstRW<[ORYONWrite_1Cyc_NONE], (instrs B)>; +def : InstRW<[ORYONWrite_1Cyc_I01], (instrs BL)>; +def : InstRW<[ORYONWrite_1Cyc_I01], + (instrs Bcc, CBZW, CBZX, CBNZW, CBNZX, TBZW, TBZX, TBNZW, TBNZX)>; +def : InstRW<[ORYONWrite_1Cyc_I0], (instrs BR, BLR)>; +def : InstRW<[ORYONWrite_1Cyc_I0], (instrs RET)>; + +// 3 uOp, 1 cycle for branch, 7 cycle for Authentication, +// 1 cycle for updating link register +// V8.3a PAC +def : InstRW<[ORYONWrite_9Cyc_I012], + (instrs BLRAA, BLRAAZ, BLRAB, BLRABZ, + BRAA, BRAAZ, BRAB, BRABZ)>; +def : InstRW<[ORYONWrite_9Cyc_I012], (instrs RETAA, RETAB, ERETAA, ERETAB)>; + +def : InstRW<[ORYONWrite_7Cyc_3Uops_I2], (instregex "^LDRAA", "^LDRAB")>; + +// Logical Instructions +//--- + +//1,1,4 TST is an alias of ANDS +def : InstRW<[ORYONWrite_1Cyc_I0123], + (instregex "^ANDS(W|X)r(i|r|x)", "^BICS(W|X)r(i|r|x)")>; + +//2,2,2 TST shift is an alias +def : InstRW<[ORYONWrite_2Cyc_I0123_I0123_RC], + (instregex "^ANDS(W|X)rs", "^BICS(W|X)rs")>; + +//1,1,6 +def : InstRW<[ORYONWrite_1Cyc_I012345], + (instregex "^AND(W|X)r(i|r|x)", "^EOR(W|X)r(i|r|x)", + "^ORR(W|X)r(i|r|x)", "^BIC(W|X)r(i|r|x)", + "^EON(W|X)r(i|r|x)", "^ORN(W|X)r(i|r|x)")>; + +//2,2,3 +def : InstRW<[ORYONWrite_2Cyc_I012345_I012345_RC], + (instregex "^AND(W|X)rs", "^EOR(W|X)rs", "^ORR(W|X)rs", + "^BIC(W|X)rs", "^EON(W|X)rs", "^ORN(W|X)rs")>; + + +//--- +// Shift Instructions +//--- + +//1,1,6 +def : InstRW<[ORYONWrite_1Cyc_I012345], + (instregex "^ASRV(W|X)r", "^LSLV(W|X)r", + "^LSRV(W|X)r", "^RORV(W|X)r", + "RMIF")>; + +//--- +// Move-Data Bit-field and Sign_Extension Instructions +//--- + +//1,1,6 +def : InstRW<[ORYONWrite_1Cyc_I012345], + (instregex "^MOVK(W|X)i", "^MOVN(W|X)i", + "^MOVZ(W|X)i", "^SBFM(W|X)ri", + "^UBFM(W|X)ri", "^BFM(W|X)ri", + "^SXT(W|B|H|X)", "^UXT(H|B)")>; + +// COPY instruction is an LLVM internal DAG node, needs further study +def : InstRW<[ORYONWrite_1Cyc_I012345], (instrs COPY)>; + +//--- +// Reverse Instructions +//--- + +//1,1,6 +def : InstRW<[ORYONWrite_1Cyc_I012345], + (instregex "^RBIT(W|X)r", "^REV(16|32|64)?(W|X)r")>; + + +//--- +// Flag Manipulate Instructions +//--- + +//1,1,4 +def : InstRW<[ORYONWrite_1Cyc_I0123], + (instregex "^SETF8", "^SETF16", "^CFINV")>; + +//--- +// Miscellaneous Instructions +//--- + +//1,1,6 +def : InstRW<[ORYONWrite_1Cyc_I012345], + (instregex "^CLS(W|X)r$", "^CLZ(W|X)r$", "^EXTR(W|X)rri")>; + + +//--- +// Multiply Instructions +//--- + +//1,3,2 +def : InstRW<[ORYONWrite_3Cyc_I45], + (instregex "^MADD(W|X)rrr", "^MSUB(W|X)rrr", + "^(S|U)MADDLrrr", "^(S|U)MSUBLrrr", + "^(S|U)MULHrr")>; + +//--- +// Divide Instructions +//--- + +def : InstRW<[ORYONWrite_7Cyc_I2_RC], + (instregex "^(S|U)DIVWr")>; + +def : InstRW<[ORYONWrite_9Cyc_I2_RC], + (instregex "^(S|U)DIVXr")>; + + +//--- +// Cryptgraphy Instructions +// +//1,3,1 on I2 +def : InstRW<[ORYONWrite_3Cyc_I2], + (instregex "^CRC32(B|H|W|X)rr", "^CRC32C(B|H|W|X)rr")>; + +//--- +// PAU instructions +//--- + +// on p47 of IXU document, we have 7 cycles for all PAU instructions +// here we just assume all signing and pauth instructions are 7 cycles +// assume all are 7 cycles here + +// signing instrucitons +def : InstRW<[ORYONWrite_7Cyc_I2], (instrs PACIA, PACIB, + PACDA, PACDB, + PACIZA, PACIZB, + PACDZA, PACDZB, + PACGA)>; +// authentication instrucitons +def : InstRW<[ORYONWrite_7Cyc_I2], (instrs AUTIA, AUTIB, + AUTDA, AUTDB, + AUTIZA, AUTIZB, + AUTDZA, AUTDZB)>; +def : InstRW<[ORYONWrite_7Cyc_I2], (instrs XPACI, XPACD)>; + +//===----------------------------------------------------------------------===// +// Instruction Tables in LSU +//===----------------------------------------------------------------------===// + +// 4 cycle Load-to-use from L1D$ +// Neon load with 5 cycle +// 6 cycle to STA ? +// STD cycle ? +// NEON STD + 2 + +// Load Instructions +// FP Load Instructions + +// Load pair, immed pre-index, normal +// Load pair, immed pre-index, signed words +// Load pair, immed post-index, normal +// Load pair, immed post-index, signed words +// NOTE: Handled by WriteLD, WriteLDHi, WriteAdr. + +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDNPDi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDNPQi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDNPSi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDNPWi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDNPXi)>; + +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDPDi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDPQi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDPSi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDPSWi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDPWi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDPXi)>; + +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRBui)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRDui)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRHui)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRQui)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSui)>; + +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRDl)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRQl)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRWl)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRXl)>; + +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRBi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRHi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRWi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRXi)>; + +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRSBWi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRSBXi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRSHWi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRSHXi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDTRSWi)>; + +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], + (instrs LDPDpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], + (instrs LDPQpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], + (instrs LDPSpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], + (instrs LDPWpre)>; + +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRBpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRDpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRHpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRQpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRWpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRXpre)>; + +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSBWpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSBXpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSBWpost)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSBXpost)>; + +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSHWpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSHXpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSHWpost)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSHXpost)>; + +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRBBpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRBBpost)>; + +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRHHpre)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRHHpost)>; + +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], + (instrs LDPDpost)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], + (instrs LDPQpost)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], + (instrs LDPSpost)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], + (instrs LDPWpost)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], + (instrs LDPXpost)>; + +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRBpost)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRDpost)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRHpost)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRQpost)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRSpost)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRWpost)>; +def : InstRW<[ORYONWrite_4Cyc_LD_I012345], (instrs LDRXpost)>; + +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRBroW)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRDroW)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRHroW)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRHHroW)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRQroW)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSroW)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSHWroW)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSHXroW)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRWroW)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRXroW)>; + +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRBroX)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRDroX)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRHHroX)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRHroX)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRQroX)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSroX)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSHWroX)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRSHXroX)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRWroX)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDRXroX)>; + +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURBi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURBBi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURDi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURHi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURHHi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURQi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURSi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURXi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURSBWi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURSBXi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURSHWi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURSHXi)>; +def : InstRW<[ORYONWrite_4Cyc_LD], (instrs LDURSWi)>; + + + +// Store register, immed post-index +// NOTE: Handled by WriteST, ReadAdrBase + +// Store register, immed pre-index +// NOTE: Handled by WriteST + +// Store pair, immed post-index, W-form +// Store pair, immed post-indx, X-form +// Store pair, immed pre-index, W-form +// Store pair, immed pre-index, X-form +// NOTE: Handled by WriteSTP. + +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURBi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURBBi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURDi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURHi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURHHi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURQi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURSi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURWi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STURXi)>; + +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STTRBi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STTRHi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STTRWi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STTRXi)>; + +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STNPDi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STNPQi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STNPXi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STNPWi)>; + +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STPDi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STPQi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STPXi)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STPWi)>; + +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STRBui)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STRDui)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STRHui)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STRQui)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STRXui)>; +def : InstRW<[ORYONWrite_1Cyc_ST], (instrs STRWui)>; + +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instrs STPDpre, STPDpost)>; +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instrs STPSpre, STPSpost)>; +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instrs STPWpre, STPWpost)>; +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instrs STPXpre, STPXpost)>; + +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instrs STRBpre, STRBpost)>; +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instrs STRBBpre, STRBBpost)>; +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instrs STRDpre, STRDpost)>; +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instrs STRHpre, STRHpost)>; +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instrs STRHHpre, STRHHpost)>; +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instrs STRQpre, STRQpost)>; +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instrs STRSpre, STRSpost)>; +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instrs STRWpre, STRWpost)>; +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instrs STRXpre, STRXpost)>; + +def : InstRW<[ORYONWrite_1Cyc_ST], + (instrs STRBroW, STRBroX)>; +def : InstRW<[ORYONWrite_1Cyc_ST], + (instrs STRDroW, STRDroX)>; +def : InstRW<[ORYONWrite_1Cyc_ST], + (instrs STRHroW, STRHroX)>; +def : InstRW<[ORYONWrite_1Cyc_ST], + (instrs STRHHroW, STRHHroX)>; +def : InstRW<[ORYONWrite_1Cyc_ST], + (instrs STRQroW, STRQroX)>; +def : InstRW<[ORYONWrite_1Cyc_ST], + (instrs STRSroW, STRSroX)>; +def : InstRW<[ORYONWrite_1Cyc_ST], + (instrs STRWroW, STRWroX)>; +def : InstRW<[ORYONWrite_1Cyc_ST], + (instrs STRXroW, STRXroX)>; + +// ASIMD Load instructions, 4 cycle access + 2 cycle NEON access +// ASIMD load, 1 element, multiple, 1 reg, D-form 1uOps +// ASIMD load, 1 element, multiple, 1 reg, Q-form 1uOps +def : InstRW<[ORYONWrite_5Cyc_LD], + (instregex "^LD1Onev(8b|4h|2s|1d|16b|8h|4s|2d)$")>; + +def : InstRW<[ORYONWrite_5Cyc_LD_I012345], + (instregex "^LD1Onev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>; + +// ASIMD load, 1 element, multiple, 2 reg, D-form 3 uOps +// ASIMD load, 1 element, multiple, 2 reg, Q-form 2 uOps +def : InstRW<[ORYONWrite_5Cyc_3Uops_LD], + (instregex "^LD1Twov(8b|4h|2s|1d)$")>; + +def : InstRW<[ORYONWrite_5Cyc_2Uops_LD], + (instregex "^LD1Twov(16b|8h|4s|2d)$")>; + +def : InstRW<[ORYONWrite_5Cyc_3Uops_LD_I012345], + (instregex "^LD1Twov(8b|4h|2s|1d)_POST$")>; + +def : InstRW<[ORYONWrite_5Cyc_2Uops_LD_I012345], + (instregex "^LD1Twov(16b|8h|4s|2d)_POST$")>; + +// ASIMD load, 1 element, multiple, 3 reg, D-form 4 uOps +// ASIMD load, 1 element, multiple, 3 reg, Q-form 3 uOps +def : InstRW<[ORYONWrite_5Cyc_4Uops_LD], + (instregex "^LD1Threev(8b|4h|2s|1d)$")>; + +def : InstRW<[ORYONWrite_5Cyc_3Uops_LD], + (instregex "^LD1Threev(16b|8h|4s|2d)$")>; + +def : InstRW<[ORYONWrite_5Cyc_4Uops_LD_I012345], + (instregex "^LD1Threev(8b|4h|2s|1d)_POST$")>; + +def : InstRW<[ORYONWrite_5Cyc_3Uops_LD_I012345], + (instregex "^LD1Threev(16b|8h|4s|2d)_POST$")>; + +// ASIMD load, 1 element, multiple, 4 reg, D-form 6 uOps +// ASIMD load, 1 element, multiple, 4 reg, Q-form 4 uOps +def : InstRW<[ORYONWrite_5Cyc_6Uops_LD], + (instregex "^LD1Fourv(8b|4h|2s|1d)$")>; +def : InstRW<[ORYONWrite_5Cyc_4Uops_LD], + (instregex "^LD1Fourv(16b|8h|4s|2d)$")>; + +def : InstRW<[ORYONWrite_5Cyc_6Uops_LD_I012345], + (instregex "^LD1Fourv(8b|4h|2s|1d)_POST$")>; +def : InstRW<[ORYONWrite_5Cyc_4Uops_LD_I012345], + (instregex "^LD1Fourv(16b|8h|4s|2d)_POST$")>; + +// ASIMD load, 1 element, one lane, B/H/S 2uOps +// ASIMD load, 1 element, one lane, D 2UOps +def : InstRW<[ORYONWrite_5Cyc_2Uops_LD], (instregex "^LD1i(8|16|32|64)$")>; +def : InstRW<[ORYONWrite_5Cyc_2Uops_LD_I012345], + (instregex "^LD1i(8|16|32|64)_POST$")>; + +// ASIMD load, 1 element, all lanes, D-form, B/H/S 2uOps +// ASIMD load, 1 element, all lanes, D-form, D 2uOps +// ASIMD load, 1 element, all lanes, Q-form 2uOps +def : InstRW<[ORYONWrite_5Cyc_2Uops_LD], + (instregex "^LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>; +def : InstRW<[ORYONWrite_5Cyc_2Uops_LD_I012345], + (instregex "^LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>; + +// ASIMD load, 2 element, multiple, D-form, B/H/S 3 uOps +// ASIMD load, 2 element, multiple, Q-form, D 4 uOps +def : InstRW<[ORYONWrite_5Cyc_3Uops_LD], + (instregex "^LD2Twov(8b|4h|2s)$")>; +def : InstRW<[ORYONWrite_5Cyc_4Uops_LD], + (instregex "^LD2Twov(16b|8h|4s|2d)$")>; +def : InstRW<[ORYONWrite_5Cyc_3Uops_LD_I012345], + (instregex "^LD2Twov(8b|4h|2s)_POST$")>; +def : InstRW<[ORYONWrite_5Cyc_4Uops_LD_I012345], + (instregex "^LD2Twov(16b|8h|4s|2d)_POST$")>; + +// ASIMD load, 2 element, one lane, B/H 3 uOps +// ASIMD load, 2 element, one lane, S 3 uOps +// ASIMD load, 2 element, one lane, D 3 uOps +def : InstRW<[ORYONWrite_5Cyc_3Uops_LD], (instregex "^LD2i(8|16|32|64)$")>; +def : InstRW<[ORYONWrite_5Cyc_3Uops_LD_I012345], + (instregex "^LD2i(8|16|32|64)_POST$")>; + +// ASIMD load, 2 element, all lanes, D-form, B/H/S 3 uOps +// ASIMD load, 2 element, all lanes, D-form, D 3 uOps +// ASIMD load, 2 element, all lanes, Q-form 3 uOps +def : InstRW<[ORYONWrite_5Cyc_3Uops_LD], + (instregex "^LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>; +def : InstRW<[ORYONWrite_5Cyc_3Uops_LD_I012345], + (instregex "^LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>; + +// ASIMD load, 3 element, multiple, D-form, B/H/S 5 uOps +// ASIMD load, 3 element, multiple, Q-form, B/H/S 6 uOps +// ASIMD load, 3 element, multiple, Q-form, D 6 uOps +def : InstRW<[ORYONWrite_5Cyc_5Uops_LD], + (instregex "^LD3Threev(8b|4h|2s)$")>; +def : InstRW<[ORYONWrite_5Cyc_6Uops_LD], + (instregex "^LD3Threev(16b|8h|4s|2d)$")>; +def : InstRW<[ORYONWrite_5Cyc_5Uops_LD_I012345], + (instregex "^LD3Threev(8b|4h|2s)_POST$")>; +def : InstRW<[ORYONWrite_5Cyc_6Uops_LD_I012345], + (instregex "^LD3Threev(16b|8h|4s|2d)_POST$")>; + +// ASIMD load, 3 element, one lone, B/H 4 uOps +// ASIMD load, 3 element, one lane, S 4 uOps +// ASIMD load, 3 element, one lane, D 5 uOps +def : InstRW<[ORYONWrite_5Cyc_4Uops_LD], (instregex "^LD3i(8|16|32)$")>; +def : InstRW<[ORYONWrite_5Cyc_5Uops_LD], (instregex "^LD3i(64)$")>; +def : InstRW<[ORYONWrite_5Cyc_4Uops_LD_I012345], + (instregex "^LD3i(8|16|32)_POST$")>; +def : InstRW<[ORYONWrite_5Cyc_5Uops_LD_I012345], + (instregex "^LD3i(64)_POST$")>; + +// ASIMD load, 3 element, all lanes, D-form, B/H/S 4 uOps +// ASIMD load, 3 element, all lanes, D-form, D 5 uOps +// ASIMD load, 3 element, all lanes, Q-form, B/H/S 4 uOps +// ASIMD load, 3 element, all lanes, Q-form, D 5 uOps +def : InstRW<[ORYONWrite_5Cyc_4Uops_LD], + (instregex "^LD3Rv(8b|4h|2s|16b|8h|4s)$")>; +def : InstRW<[ORYONWrite_5Cyc_5Uops_LD], + (instregex "^LD3Rv(1d|2d)$")>; +def : InstRW<[ORYONWrite_5Cyc_4Uops_LD_I012345], + (instregex "^LD3Rv(8b|4h|2s|16b|8h|4s)_POST$")>; +def : InstRW<[ORYONWrite_5Cyc_5Uops_LD_I012345], + (instregex "^LD3Rv(1d|2d)_POST$")>; + +// ASIMD load, 4 element, multiple, D-form, B/H/S 6 uOps +// ASIMD load, 4 element, multiple, Q-form, B/H/S 10 uOps +// ASIMD load, 4 element, multiple, Q-form, D 8 uOps +def : InstRW<[ORYONWrite_5Cyc_6Uops_LD], + (instregex "^LD4Fourv(8b|4h|2s)$")>; +def : InstRW<[ORYONWrite_5Cyc_10Uops_LD], + (instregex "^LD4Fourv(16b|8h|4s)$")>; +def : InstRW<[ORYONWrite_5Cyc_8Uops_LD], + (instregex "^LD4Fourv(2d)$")>; +def : InstRW<[ORYONWrite_5Cyc_6Uops_LD_I012345], + (instregex "^LD4Fourv(8b|4h|2s)_POST$")>; +def : InstRW<[ORYONWrite_5Cyc_10Uops_LD_I012345], + (instregex "^LD4Fourv(16b|8h|4s)_POST$")>; +def : InstRW<[ORYONWrite_5Cyc_8Uops_LD_I012345], + (instregex "^LD4Fourv(2d)_POST$")>; + +// ASIMD load, 4 element, one lane, B/H 5 uOps +// ASIMD load, 4 element, one lane, S 5 uOps +// ASIMD load, 4 element, one lane, D 6 uOps +def : InstRW<[ORYONWrite_5Cyc_5Uops_LD], (instregex "^LD4i(8|16|32)$")>; +def : InstRW<[ORYONWrite_5Cyc_6Uops_LD], (instregex "^LD4i(64)$")>; +def : InstRW<[ORYONWrite_5Cyc_5Uops_LD_I012345], + (instregex "^LD4i(8|16|32)_POST$")>; +def : InstRW<[ORYONWrite_5Cyc_6Uops_LD_I012345], + (instregex "^LD4i(64)_POST$")>; + +// ASIMD load, 4 element, all lanes, D-form, B/H/S 5 uOps +// ASIMD load, 4 element, all lanes, D-form, D 6 uOps +// ASIMD load, 4 element, all lanes, Q-form, B/H/S 5 uOps +// ASIMD load, 4 element, all lanes, Q-form, D 6 uOps +def : InstRW<[ORYONWrite_5Cyc_5Uops_LD], + (instregex "^LD4Rv(8b|4h|2s|16b|8h|4s)$")>; +def : InstRW<[ORYONWrite_5Cyc_6Uops_LD], + (instregex "^LD4Rv(1d|2d)$")>; +def : InstRW<[ORYONWrite_5Cyc_5Uops_LD_I012345], + (instregex "^LD4Rv(8b|4h|2s|16b|8h|4s)_POST$")>; +def : InstRW<[ORYONWrite_5Cyc_6Uops_LD_I012345], + (instregex "^LD4Rv(1d|2d)_POST$")>; + +// ASIMD Store Instructions +// ASIMD store, 1 element, multiple, 1 reg, D-form 1 uOps +// ASIMD store, 1 element, multiple, 1 reg, Q-form 1 uops +def : InstRW<[ORYONWrite_1Cyc_ST], + (instregex "^ST1Onev(8b|4h|2s|1d|16b|8h|4s|2d)$")>; +def : InstRW<[ORYONWrite_1Cyc_ST_I012345], + (instregex "^ST1Onev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>; + +// ASIMD store, 1 element, multiple, 2 reg, D-form 2 uOps +// ASIMD store, 1 element, multiple, 2 reg, Q-form 2 uOps +def : InstRW<[ORYONWrite_1Cyc_2Uops_ST], + (instregex "^ST1Twov(8b|4h|2s|1d|16b|8h|4s|2d)$")>; +def : InstRW<[ORYONWrite_1Cyc_2Uops_ST_I012345], + (instregex "^ST1Twov(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>; + +// ASIMD store, 1 element, multiple, 3 reg, D-form 3 uOps +// ASIMD store, 1 element, multiple, 3 reg, Q-form 3 uOps +def : InstRW<[ORYONWrite_1Cyc_3Uops_ST], + (instregex "^ST1Threev(8b|4h|2s|1d|16b|8h|4s|2d)$")>; +def : InstRW<[ORYONWrite_1Cyc_3Uops_ST_I012345], + (instregex "^ST1Threev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>; + +// ASIMD store, 1 element, multiple, 4 reg, D-form 4 uOps +// ASIMD store, 1 element, multiple, 4 reg, Q-form 4 uOps +def : InstRW<[ORYONWrite_1Cyc_4Uops_ST], + (instregex "^ST1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)$")>; +def : InstRW<[ORYONWrite_1Cyc_4Uops_ST_I012345], + (instregex "^ST1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>; + +// ASIMD store, 1 element, one lane, B/H/S 2 uOps +// ASIMD store, 1 element, one lane, D 2 uOps +def : InstRW<[ORYONWrite_1Cyc_2Uops_ST], + (instregex "^ST1i(8|16|32|64)$")>; +def : InstRW<[ORYONWrite_1Cyc_2Uops_ST_I012345], + (instregex "^ST1i(8|16|32|64)_POST$")>; + +// ASIMD store, 2 element, multiple, D-form, B/H/S 2 uOps +// ASIMD store, 2 element, multiple, Q-form, B/H/S 4 uOps +// ASIMD store, 2 element, multiple, Q-form, D 4 uOps +def : InstRW<[ORYONWrite_1Cyc_2Uops_ST], + (instregex "^ST2Twov(8b|4h|2s)$")>; +def : InstRW<[ORYONWrite_1Cyc_4Uops_ST], + (instregex "^ST2Twov(16b|8h|4s|2d)$")>; +def : InstRW<[ORYONWrite_1Cyc_2Uops_ST_I012345], + (instregex "^ST2Twov(8b|4h|2s)_POST$")>; +def : InstRW<[ORYONWrite_1Cyc_4Uops_ST_I012345], + (instregex "^ST2Twov(16b|8h|4s|2d)_POST$")>; + +// ASIMD store, 2 element, one lane, B/H/S 2 uOps +// ASIMD store, 2 element, one lane, D 2 uOps +def : InstRW<[ORYONWrite_1Cyc_2Uops_ST], + (instregex "^ST2i(8|16|32|64)$")>; +def : InstRW<[ORYONWrite_1Cyc_2Uops_ST_I012345], + (instregex "^ST2i(8|16|32|64)_POST$")>; + +// ASIMD store, 3 element, multiple, D-form, B/H/S 4 uOps +// ASIMD store, 3 element, multiple, Q-form, B/H/S 6 uOps +// ASIMD store, 3 element, multiple, Q-form, D 6 uOps +def : InstRW<[ORYONWrite_1Cyc_4Uops_ST], + (instregex "^ST3Threev(8b|4h|2s)$")>; +def : InstRW<[ORYONWrite_1Cyc_6Uops_ST], + (instregex "^ST3Threev(16b|8h|4s|2d)$")>; +def : InstRW<[ORYONWrite_1Cyc_4Uops_ST_I012345], + (instregex "^ST3Threev(8b|4h|2s)_POST$")>; +def : InstRW<[ORYONWrite_1Cyc_6Uops_ST_I012345], + (instregex "^ST3Threev(16b|8h|4s|2d)_POST$")>; + +// ASIMD store, 3 element, one lane, B/H 2 uOps +// ASIMD store, 3 element, one lane, S 2 uOps +// ASIMD store, 3 element, one lane, D 4 uOps +def : InstRW<[ORYONWrite_1Cyc_2Uops_ST], (instregex "^ST3i(8|16|32)$")>; +def : InstRW<[ORYONWrite_1Cyc_4Uops_ST], (instregex "^ST3i(64)$")>; +def : InstRW<[ORYONWrite_1Cyc_2Uops_ST_I012345], + (instregex "^ST3i(8|16|32)_POST$")>; +def : InstRW<[ORYONWrite_1Cyc_4Uops_ST_I012345], + (instregex "^ST3i(64)_POST$")>; + + +// ASIMD store, 4 element, multiple, D-form, B/H/S 5 uOps +// ASIMD store, 4 element, multiple, Q-form, B/H/S 10 uOps +// ASIMD store, 4 element, multiple, Q-form, D 8 uOps +def : InstRW<[ORYONWrite_1Cyc_5Uops_ST], + (instregex "^ST4Fourv(8b|4h|2s)$")>; +def : InstRW<[ORYONWrite_1Cyc_10Uops_ST], + (instregex "^ST4Fourv(16b|8h|4s)$")>; +def : InstRW<[ORYONWrite_1Cyc_8Uops_ST], + (instregex "^ST4Fourv(2d)$")>; +def : InstRW<[ORYONWrite_1Cyc_5Uops_ST_I012345], + (instregex "^ST4Fourv(8b|4h|2s)_POST$")>; +def : InstRW<[ORYONWrite_1Cyc_10Uops_ST_I012345], + (instregex "^ST4Fourv(16b|8h|4s)_POST$")>; +def : InstRW<[ORYONWrite_1Cyc_8Uops_ST_I012345], + (instregex "^ST4Fourv(2d)_POST$")>; + +// ASIMD store, 4 element, one lane, B/H 3 uOps +// ASIMD store, 4 element, one lane, S 3 uOps +// ASIMD store, 4 element, one lane, D 4 uOps +def : InstRW<[ORYONWrite_1Cyc_3Uops_ST], (instregex "^ST4i(8|16|32)$")>; +def : InstRW<[ORYONWrite_1Cyc_4Uops_ST], (instregex "^ST4i(64)$")>; +def : InstRW<[ORYONWrite_1Cyc_3Uops_ST_I012345], + (instregex "^ST4i(8|16|32)_POST$")>; +def : InstRW<[ORYONWrite_1Cyc_4Uops_ST_I012345], + (instregex "^ST4i(64)_POST$")>; + + +//===----------------------------------------------------------------------===// +// Instruction Tables in VXU +//===----------------------------------------------------------------------===// +// all uOps are not clearly written in the VXU document + +// I2V +def : InstRW<[ORYONWrite_I2V_4Cyc_I45], (instregex "^FMOV[HSD][WX]r", "^FMOVDXHighr")>; + +// I2V with convert +def : InstRW<[ORYONWrite_I2V_7Cyc_I45], (instregex "^[SU]CVTF[SU][XW][HSD]ri")>; + +// V2I +def : InstRW<[ORYONWrite_V2I_3Cyc_FP01], (instregex "^FMOV[WX][HSD]r", "FMOVXDHighr")>; + +// V2I with convert 2nd [SU] necessary? +def : InstRW<[ORYONWrite_V2I_6Cyc_FP01], (instregex "^FCVT[AMNPZ][SU][SU][XW][HSD]r")>; + +// float to float move immediate, row 7 in big chart +def : InstRW<[ORYONWrite_V2V_2Cyc_FP0123], (instregex "^FMOV[HSD]r")>; +def : InstRW<[ORYONWrite_V2V_2Cyc_FP0123], (instregex "^FMOV[HSD]i")>; + +// float to float conversion within VXU, precision conversion +def : InstRW<[ORYONWrite_V2V_6Cyc_FP01], (instregex "^FJCVTZS")>; +def : InstRW<[ORYONWrite_V2V_3Cyc_FP0123], (instregex "^FCVT[HSD][HSD]r", + "^FRINT(A|I|M|N|P|X|Z)(Sr|Dr)")>; + +// floating comparison write to NZCV +def : InstRW<[ORYONWrite_2Cyc_FP01], (instregex "^FCMP(E)?[HSD]r[ir]")>; +def : InstRW<[ORYONWrite_2Cyc_FP01], (instregex "^FCCMP(E)?[HSD]rr")>; + +// floating point conditional select +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^FCSEL")>; + +// floating multiply-add +def : InstRW<[ORYONWrite_4Cyc_FP0123], (instregex "^(F|FN)MADD", "^(F|FN)MSUB")>; + +// floating unary, cycle/throughput? xls row14 +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^F(ABS|NEG)[SD]r")>; + +//floating division/square root +def : InstRW<[ORYONWrite_7Cyc_FP3], (instregex "^FDIVHrr")>; +def : InstRW<[ORYONWrite_8Cyc_FP3], (instregex "^FDIVSrr")>; +def : InstRW<[ORYONWrite_10Cyc_FP3], (instregex "^FDIVDrr")>; + +def : InstRW<[ORYONWrite_8Cyc_FP3_RC], (instregex "^FSQRTHr")>; +def : InstRW<[ORYONWrite_10Cyc_FP3_RC], (instregex "^FSQRTSr")>; +def : InstRW<[ORYONWrite_13Cyc_FP3_RC], (instregex "^FSQRTDr")>; + +//========== +// SIMD move instructions +//========== + +// ASIMD DUP element +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^DUPv.+lane")>; +// ASIMD DUP general thoughput undecided, 3? FP0123 +// VXU doc, p42, 2 uOps +def : InstRW<[ORYONWrite_3Cyc_2Uops_FP0123], (instregex "^DUPv.+gpr")>; + +// ASIMD insert, element to element +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^INSv.+lane")>; +// ASIMD insert, gen reg 3? FP0123? +def : InstRW<[ORYONWrite_3Cyc_2Uops_FP0123], (instregex "^INSv.+gpr")>; + +// ASIMD move, FP immed +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^FMOVv")>; + +// ASIMD transfer, element to gen reg +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^[SU]MOVv")>; + +//========== +// SIMD arithmetic instructions +//========== +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^ADDv", "^SUBv", + "^BIFv", "^BITv", "^BSLv", + "^ANDv", "^BICv", "^EORv", + "^ORRv", "^ORNv")>; + + +def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^FABDv", "^FADDv", "^FSUBv")>; + +// floating division +def : InstRW<[ORYONWrite_6Cyc_FP3], (instregex "^FDIVv.*16$")>; +def : InstRW<[ORYONWrite_7Cyc_FP3], (instregex "^FDIVv.*32$")>; +def : InstRW<[ORYONWrite_9Cyc_FP3], (instregex "^FDIVv.*64$")>; + +def : InstRW<[ORYONWrite_4Cyc_FP0123], (instregex "^FMUL(X)?v", + "^FRECPSv", "^FRSQRTSv")>; + +def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^MLAv","^MLSv", "^MULv", + "^PMULv", "UABAv")>; + +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "SABAv", "SABDv", + "^(SH|UH)(ADD|SUB)v", + "^S(MAX|MIN)v", + "^(SQ|UQ)(ADD|SUB)v", + "^(SQ|SQR|UQ|UQR)SHLv", + "^(SR|UR)HADDv", + "^(SR|UR)SHLv", + "^UABDv", + "^U(MAX|MIN)v")>; +// IMAX or UMAX in the above line +//========== +// SIMD compare instructions +//========== + +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^CMEQv","^CMGEv","^CMGTv", + "^CMLEv","^CMLTv", "^CMHIv", + "^CMHSv", + "^FCMEQv", "^FCMGEv", + "^FCMGTv", "^FCMLEv", + "^FCMLTv", + "^FACGEv", "^FACGTv")>; + +//========== +// SIMD widening and narrowing arithmetic instructions +//========== +// NO need to list ADDHN2, RADDHN2, RSUBHN2 as they are not distinguished +// from ADDHN, RADDHN, RSUBHN in td file(v16i8, v8i16, v4i32). +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^ADDHNv", + "^SUBHNv", + "^RADDHNv", + "^RSUBHNv", + "^SABD(L|L2)v", "^UABD(L|L2)v", + "^(S|U)(ADD|SUB)(L|L2|W|W2)v")>; + +def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^PMUL(L|L2)v","^SABA(L|L2)v", + "^(S|U|SQ)(MLA|MSL|MUL)(L|L2)v")>; + +//========== +// SIMD unary arithmetic instructions +//========== +//^MVNv is an alias of ^NOTv +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^ABSv", "^CLSv","^CLZv", "^CNTv", + "^NEGv", "^NOTv", + "^RBITv", "^REV(16|32|64)v", + "^SQ(ABS|NEG)v", "^SQ(XT|XTU)(N|N2)v", + "^(SU|US)QADDv", + "^UQXT(N|N2)v", "^XTN2?v")>; + +def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^FCVT(L|L2|N|N2|XN|XN2)v", + "^FRINT[AIMNPXZ]v", + "^FRSQRTEv", + "^(S|U)ADALPv", + "^(S|U)ADDLPv")>; + + +def : InstRW<[ORYONWrite_3Cyc_FP0], (instregex "^URECPEv", "^URSQRTEv", + "^FRECPEv", "^FRECPXv")>; + +def : InstRW<[ORYONWrite_8Cyc_FP3_RC], (instregex "^FSQRTv.*16$")>; +def : InstRW<[ORYONWrite_10Cyc_FP3_RC], (instregex "^FSQRTv.*32$")>; +def : InstRW<[ORYONWrite_13Cyc_FP3_RC], (instregex "^FSQRTv.*64$")>; + +//========== +// SIMD binary elememt arithmetic instructions +//========== + +def : InstRW<[ORYONWrite_4Cyc_FP0123], (instregex "^FMLAv", "^FMLSv")>; + +def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^SQDMULHv", + "^SQRD(MLA|MLS|MUL)Hv")>; + +//========== +// SIMD permute instructions +//========== + +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^EXTv", "^TRN(1|2)v", + "^UZP(1|2)v", "^ZIP(1|2)v")>; + +//========== +// SIMD immediate instructions +//========== + +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^MOVIv", "^MVNIv")>; + +//========== +// SIMD shift(immediate) instructions +//========== +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^RSHR(N|N2)v", "^SHLv", + "^(SHL|SHR)(N|N2)v", + "^SLIv", + "^(SQ|SQR)SHR(U)?(N|N2)v", + "^(UQ|UQR)SHR(N|N2)v", + "^SQSHLUv", + "^SRIv", + "^(S|SR|U|UR)SHRv", + "^(S|SR|U|UR)SRAv", + "^(S|U)SHL(L|L2)v")>; + +//========== +// SIMD floating-point and integer conversion instructions +//========== +// same as above conversion + +//========== +// SIMD reduce (acoss vector lanes) instructions +//========== + +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^ADDVv", + "^(FMAX|FMIN)(V|NMV)v", + "^(S|U)ADDLVv", + "^(S|U)(MAX|MIN)Vv")>; +//========== +// SIMD pairwise arithmetic instructions +//========== + +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^ADDPv", "^FADDPv", + "^(FMAX|FMIN)(NMP|P)v", + "^(S|U)(MIN|MAX)Pv")>; +//========== +// SIMD dot prodcut instructions +//========== + +def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^(U|S)DOTv")>; + +//========== +// SIMD table lookup instructions +//========== +// TBL 1-reg/2-reg; TBX 1-reg, 1uOp, throughput=4 latency=2 +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instrs TBLv8i8One, TBLv16i8One, + TBXv8i8One, TBXv16i8One, + TBLv8i8Two, TBLv16i8Two)>; + +// TBL 3-reg/4-reg, 3uops, throughtput=4/3=1.33 latency=4 +def : InstRW<[ORYONWrite_4Cyc_FP0123_FP0123_FP0123_RC], + (instrs TBLv8i8Three, TBLv16i8Three, + TBLv8i8Four, TBLv16i8Four)>; + + +// TBX 2-reg 2 uOps, throughput=2 latency=4 +def : InstRW<[ORYONWrite_4Cyc_FP0123_FP0123_RC], (instrs TBXv8i8Two, TBXv16i8Two)>; + +// TBX 3-reg/4-reg, 4uOps, throughput=1, latency=6 +def : InstRW<[ORYONWrite_6Cyc_FP0123_FP0123_FP0123_FP0123_RC], + (instrs TBXv8i8Three, TBXv16i8Three, + TBXv8i8Four, TBXv16i8Four)>; + + +//========== +// SIMD complex number arithmetic instructions +//========== + +def : InstRW<[ORYONWrite_4Cyc_FP0123], (instregex "^FCADDv", "^FCMLAv")>; + +//========== +// SIMD cryptographic instructions +//========== +// 3,4 on IMLA, CRYP +def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^AES[DE]", + "^SM3(TT1|TT2)(A|B)")>; + +// 2,4 on CRYP +def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^AESI?MC", + "^EOR3", + "^RAX1", + "^XAR", + "^BCAX", + "^SM3SS1", + "^SM3PART(W1|W2)")>; +// 5,1 on CRYP +def : InstRW<[ORYONWrite_5Cyc_FP1], (instregex "^SM4E", + "^SM4EKEY")>; + +// 2,1 on CRYP +def : InstRW<[ORYONWrite_2Cyc_FP1], (instregex "^SHA1(H|SU0|SU1)", + "^SHA256SU0", + "^SHA512(SU0|SU1)")>; + +// 3,1 on CRYP +def : InstRW<[ORYONWrite_3Cyc_FP1], (instregex "^SHA256SU1", + "^SHA512(H|H2)")>; + +// 4,0.25 on CRYP +def : InstRW<[ORYONWrite_4Cyc_FP1_RC4], (instregex "^SHA1(C|P|M)", + "^SHA256(H|H2)")>; + +//========== +// SIMD v8.6 instructions +//========== +// 4,2 on IMLA +def : InstRW<[ORYONWrite_4Cyc_FP0123_RC], (instregex "^(S|U|US)MMLA$")>; + +// 4,0.5 on IMLA +def : InstRW<[ORYONWrite_8Cyc_FP0123_RC], (instregex "^BFMMLA$")>; + +// 4,0.5 on IMLA +def : InstRW<[ORYONWrite_8Cyc_FP0123_RC], (instregex "^BFMLAL(B|T)")>; + +// 3,4 +def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^(US|SU)DOTv")>; + +// 3,1 +def : InstRW<[ORYONWrite_4Cyc_FP0123], (instregex "^BF(16)?DOTv")>; + +// 3,4 +def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^BFCVT(N|N2)?$")>; + + +} // SchedModel = OryonModel diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp index 8bc26eeef34d..93ea729e2550 100644 --- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp @@ -299,6 +299,13 @@ void AArch64Subtarget::initializeProperties(bool HasMinSize) { PrefLoopAlignment = Align(64); MaxInterleaveFactor = 4; break; + case Oryon: + CacheLineSize = 64; + PrefFunctionAlignment = Align(16); + MaxInterleaveFactor = 4; + PrefetchDistance = 128; + MinPrefetchStride = 1024; + break; } if (AArch64MinimumJumpTableEntries.getNumOccurrences() > 0 || !HasMinSize) diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp index 68155acd9e5b..b3b8486c604b 100644 --- a/llvm/lib/TargetParser/Host.cpp +++ b/llvm/lib/TargetParser/Host.cpp @@ -302,6 +302,7 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) { .Case("0x805", "cortex-a76") // Kryo 4xx/5xx Silver .Case("0xc00", "falkor") .Case("0xc01", "saphira") + .Case("0x001", "oryon-1") .Default("generic"); if (Implementer == "0x53") { // Samsung Electronics Co., Ltd. // The Exynos chips have a convoluted ID scheme that doesn't seem to follow diff --git a/llvm/unittests/TargetParser/Host.cpp b/llvm/unittests/TargetParser/Host.cpp index 6aa1d7a087eb..61921a99e171 100644 --- a/llvm/unittests/TargetParser/Host.cpp +++ b/llvm/unittests/TargetParser/Host.cpp @@ -125,6 +125,9 @@ TEST(getLinuxHostCPUName, AArch64) { EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0xc0\n" "CPU part : 0xac5"), "ampere1b"); + EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x51\n" + "CPU part : 0x001"), + "oryon-1"); // MSM8992/4 weirdness StringRef MSM8992ProcCpuInfo = R"( diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp index 797d7dfbca20..571031d07fcc 100644 --- a/llvm/unittests/TargetParser/TargetParserTest.cpp +++ b/llvm/unittests/TargetParser/TargetParserTest.cpp @@ -1815,11 +1815,23 @@ INSTANTIATE_TEST_SUITE_P( {AArch64::AEK_CRC, AArch64::AEK_AES, AArch64::AEK_SHA2, AArch64::AEK_FP, AArch64::AEK_SIMD, AArch64::AEK_FP16, AArch64::AEK_RAS, AArch64::AEK_LSE, AArch64::AEK_RDM}), - "8.2-A")), + "8.2-A"), + ARMCPUTestParams( + "oryon-1", "armv8.6-a", "crypto-neon-fp-armv8", + (AArch64::ExtensionBitset( + {AArch64::AEK_CRC, AArch64::AEK_FP, AArch64::AEK_PAUTH, + AArch64::AEK_FCMA, AArch64::AEK_JSCVT, AArch64::AEK_SIMD, + AArch64::AEK_RAS, AArch64::AEK_LSE, AArch64::AEK_RDM, + AArch64::AEK_RCPC, AArch64::AEK_DOTPROD, AArch64::AEK_SM4, + AArch64::AEK_SHA3, AArch64::AEK_BF16, AArch64::AEK_SHA2, + AArch64::AEK_AES, AArch64::AEK_I8MM, AArch64::AEK_RAND, + AArch64::AEK_PROFILE, AArch64::AEK_CRYPTO})), + "8.6-A")), + ARMCPUTestParams::PrintToStringParamName); // Note: number of CPUs includes aliases. -static constexpr unsigned NumAArch64CPUArchs = 76; +static constexpr unsigned NumAArch64CPUArchs = 77; TEST(TargetParserTest, testAArch64CPUArchList) { SmallVector List; -- GitLab From 505cd125a1edad4e59776e03e9a755aa598ed6f2 Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Thu, 6 Jun 2024 16:31:10 +0200 Subject: [PATCH 105/462] [SPIR-V] Add validation to the test case with get_image_array_size/get_image_dim calls (#94467) This PR is to add validation to the test case with get_image_array_size/get_image_dim calls (transcoding/check_ro_qualifier.ll). This test case didn't pass validation because of invalid emission of OpCompositeExtract instruction (Result Type must be the same type as Composite.). In order to fix the problem this PR improves type inference in general and partially addresses issues: * https://github.com/llvm/llvm-project/issues/91998 * https://github.com/llvm/llvm-project/issues/91997 A reproducer from the description of the latter issue is added as a new test case as a part of this PR. --- llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 47 ++++-- llvm/lib/Target/SPIRV/SPIRVBuiltins.td | 1 + llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 149 +++++++++++------- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 16 +- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 6 +- .../test/CodeGen/SPIRV/event-wait-ptr-type.ll | 16 +- ...Intrinsics-no-duplicate-spv_assign_type.ll | 4 +- .../transcoding/OpGroupAsyncCopy-strided.ll | 36 +++++ .../SPIRV/transcoding/check_ro_qualifier.ll | 2 +- 9 files changed, 196 insertions(+), 81 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/transcoding/OpGroupAsyncCopy-strided.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp index 956b851fce6c..49838e685a6d 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp @@ -1459,11 +1459,22 @@ static bool generateImageSizeQueryInst(const SPIRV::IncomingCall *Call, Component == 3 ? NumActualRetComponents - 1 : Component; assert(ExtractedComposite < NumActualRetComponents && "Invalid composite index!"); + Register TypeReg = GR->getSPIRVTypeID(Call->ReturnType); + SPIRVType *NewType = nullptr; + if (QueryResultType->getOpcode() == SPIRV::OpTypeVector) { + Register NewTypeReg = QueryResultType->getOperand(1).getReg(); + if (TypeReg != NewTypeReg && + (NewType = GR->getSPIRVTypeForVReg(NewTypeReg)) != nullptr) + TypeReg = NewTypeReg; + } MIRBuilder.buildInstr(SPIRV::OpCompositeExtract) .addDef(Call->ReturnRegister) - .addUse(GR->getSPIRVTypeID(Call->ReturnType)) + .addUse(TypeReg) .addUse(QueryResult) .addImm(ExtractedComposite); + if (NewType != nullptr) + insertAssignInstr(Call->ReturnRegister, nullptr, NewType, GR, MIRBuilder, + MIRBuilder.getMF().getRegInfo()); } else { // More than 1 component is expected, fill a new vector. auto MIB = MIRBuilder.buildInstr(SPIRV::OpVectorShuffle) @@ -2063,16 +2074,30 @@ static bool generateAsyncCopy(const SPIRV::IncomingCall *Call, auto Scope = buildConstantIntReg(SPIRV::Scope::Workgroup, MIRBuilder, GR); switch (Opcode) { - case SPIRV::OpGroupAsyncCopy: - return MIRBuilder.buildInstr(Opcode) - .addDef(Call->ReturnRegister) - .addUse(GR->getSPIRVTypeID(Call->ReturnType)) - .addUse(Scope) - .addUse(Call->Arguments[0]) - .addUse(Call->Arguments[1]) - .addUse(Call->Arguments[2]) - .addUse(buildConstantIntReg(1, MIRBuilder, GR)) - .addUse(Call->Arguments[3]); + case SPIRV::OpGroupAsyncCopy: { + SPIRVType *NewType = + Call->ReturnType->getOpcode() == SPIRV::OpTypeEvent + ? nullptr + : GR->getOrCreateSPIRVTypeByName("spirv.Event", MIRBuilder); + Register TypeReg = GR->getSPIRVTypeID(NewType ? NewType : Call->ReturnType); + unsigned NumArgs = Call->Arguments.size(); + Register EventReg = Call->Arguments[NumArgs - 1]; + bool Res = MIRBuilder.buildInstr(Opcode) + .addDef(Call->ReturnRegister) + .addUse(TypeReg) + .addUse(Scope) + .addUse(Call->Arguments[0]) + .addUse(Call->Arguments[1]) + .addUse(Call->Arguments[2]) + .addUse(Call->Arguments.size() > 4 + ? Call->Arguments[3] + : buildConstantIntReg(1, MIRBuilder, GR)) + .addUse(EventReg); + if (NewType != nullptr) + insertAssignInstr(Call->ReturnRegister, nullptr, NewType, GR, MIRBuilder, + MIRBuilder.getMF().getRegInfo()); + return Res; + } case SPIRV::OpGroupWaitEvents: return MIRBuilder.buildInstr(Opcode) .addUse(Scope) diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td index 24c6c2688642..edc9e1a33d9f 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td @@ -586,6 +586,7 @@ defm : DemangledNativeBuiltin<"__spirv_SpecConstantComposite", OpenCL_std, SpecC // Async Copy and Prefetch builtin records: defm : DemangledNativeBuiltin<"async_work_group_copy", OpenCL_std, AsyncCopy, 4, 4, OpGroupAsyncCopy>; +defm : DemangledNativeBuiltin<"async_work_group_strided_copy", OpenCL_std, AsyncCopy, 5, 5, OpGroupAsyncCopy>; defm : DemangledNativeBuiltin<"__spirv_GroupAsyncCopy", OpenCL_std, AsyncCopy, 6, 6, OpGroupAsyncCopy>; defm : DemangledNativeBuiltin<"wait_group_events", OpenCL_std, AsyncCopy, 2, 2, OpGroupWaitEvents>; defm : DemangledNativeBuiltin<"__spirv_GroupWaitEvents", OpenCL_std, AsyncCopy, 3, 3, OpGroupWaitEvents>; diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 5ef0be1cab72..bbd25dc85f52 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -61,9 +61,6 @@ class SPIRVEmitIntrinsics DenseMap AggrConstTypes; DenseSet AggrStores; - // a registry of created Intrinsic::spv_assign_ptr_type instructions - DenseMap AssignPtrTypeInstr; - // deduce element type of untyped pointers Type *deduceElementType(Value *I); Type *deduceElementTypeHelper(Value *I); @@ -98,14 +95,16 @@ class SPIRVEmitIntrinsics return B.CreateIntrinsic(IntrID, {Types}, Args); } + void buildAssignType(IRBuilder<> &B, Type *ElemTy, Value *Arg); void buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg); + void updateAssignType(CallInst *AssignCI, Value *Arg, Value *OfType); void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B); void processInstrAfterVisit(Instruction *I, IRBuilder<> &B); void insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B); void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B); - void insertAssignTypeInstrForTargetExtTypes(TargetExtType *AssignedType, - Value *V, IRBuilder<> &B); + void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V, + IRBuilder<> &B); void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer, Type *ExpectedElementType, unsigned OperandToReplace, @@ -218,15 +217,39 @@ static inline void reportFatalOnTokenType(const Instruction *I) { false); } +void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty, + Value *Arg) { + Value *OfType = PoisonValue::get(Ty); + CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, + {Arg->getType()}, OfType, Arg, {}, B); + GR->addAssignPtrTypeInstr(Arg, AssignCI); +} + void SPIRVEmitIntrinsics::buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg) { - CallInst *AssignPtrTyCI = - buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {Arg->getType()}, - Constant::getNullValue(ElemTy), Arg, - {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B); + Value *OfType = PoisonValue::get(ElemTy); + CallInst *AssignPtrTyCI = buildIntrWithMD( + Intrinsic::spv_assign_ptr_type, {Arg->getType()}, OfType, Arg, + {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B); GR->addDeducedElementType(AssignPtrTyCI, ElemTy); GR->addDeducedElementType(Arg, ElemTy); - AssignPtrTypeInstr[Arg] = AssignPtrTyCI; + GR->addAssignPtrTypeInstr(Arg, AssignPtrTyCI); +} + +void SPIRVEmitIntrinsics::updateAssignType(CallInst *AssignCI, Value *Arg, + Value *OfType) { + LLVMContext &Ctx = Arg->getContext(); + AssignCI->setArgOperand( + 1, MetadataAsValue::get( + Ctx, MDNode::get(Ctx, ValueAsMetadata::getConstant(OfType)))); + if (cast(AssignCI)->getIntrinsicID() != + Intrinsic::spv_assign_ptr_type) + return; + + // update association with the pointee type + Type *ElemTy = OfType->getType(); + GR->addDeducedElementType(AssignCI, ElemTy); + GR->addDeducedElementType(Arg, ElemTy); } // Set element pointer type to the given value of ValueTy and tries to @@ -513,19 +536,16 @@ void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I) { if (!Ty) { GR->addDeducedElementType(Op, KnownElemTy); // check if there is existing Intrinsic::spv_assign_ptr_type instruction - auto It = AssignPtrTypeInstr.find(Op); - if (It == AssignPtrTypeInstr.end()) { + CallInst *AssignCI = GR->findAssignPtrTypeInstr(Op); + if (AssignCI == nullptr) { Instruction *User = dyn_cast(Op->use_begin()->get()); setInsertPointSkippingPhis(B, User ? User->getNextNode() : I); CallInst *CI = buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {OpTy}, OpTyVal, Op, {B.getInt32(getPointerAddressSpace(OpTy))}, B); - AssignPtrTypeInstr[Op] = CI; + GR->addAssignPtrTypeInstr(Op, CI); } else { - It->second->setArgOperand( - 1, - MetadataAsValue::get( - Ctx, MDNode::get(Ctx, ValueAsMetadata::getConstant(OpTyVal)))); + updateAssignType(AssignCI, Op, OpTyVal); } } else { if (auto *OpI = dyn_cast(Op)) { @@ -559,7 +579,9 @@ void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old, if (isAssignTypeInstr(U)) { B.SetInsertPoint(U); SmallVector Args = {New, U->getOperand(1)}; - B.CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args); + CallInst *AssignCI = + B.CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args); + GR->addAssignPtrTypeInstr(New, AssignCI); U->eraseFromParent(); } else if (isMemInstrToReplace(U) || isa(U) || isa(U)) { @@ -751,33 +773,39 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) { return NewI; } -void SPIRVEmitIntrinsics::insertAssignTypeInstrForTargetExtTypes( +void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt( TargetExtType *AssignedType, Value *V, IRBuilder<> &B) { - // Do not emit spv_assign_type if the V is of the AssignedType already. - if (V->getType() == AssignedType) - return; + Type *VTy = V->getType(); - // Do not emit spv_assign_type if there is one already targetting V. If the - // found spv_assign_type assigns a type different than AssignedType, report an - // error. Builtin types cannot be redeclared or casted. - for (auto User : V->users()) { - auto *II = dyn_cast(User); - if (!II || II->getIntrinsicID() != Intrinsic::spv_assign_type) - continue; + // A couple of sanity checks. + assert(isPointerTy(VTy) && "Expect a pointer type!"); + if (auto PType = dyn_cast(VTy)) + if (PType->getElementType() != AssignedType) + report_fatal_error("Unexpected pointer element type!"); - MetadataAsValue *VMD = cast(II->getOperand(1)); - Type *BuiltinType = - dyn_cast(VMD->getMetadata())->getType(); - if (BuiltinType != AssignedType) - report_fatal_error("Type mismatch " + BuiltinType->getTargetExtName() + - "/" + AssignedType->getTargetExtName() + - " for value " + V->getName(), - false); + CallInst *AssignCI = GR->findAssignPtrTypeInstr(V); + if (!AssignCI) { + buildAssignType(B, AssignedType, V); return; } - Constant *Const = UndefValue::get(AssignedType); - buildIntrWithMD(Intrinsic::spv_assign_type, {V->getType()}, Const, V, {}, B); + Type *CurrentType = + dyn_cast( + cast(AssignCI->getOperand(1))->getMetadata()) + ->getType(); + if (CurrentType == AssignedType) + return; + + // Builtin types cannot be redeclared or casted. + if (CurrentType->isTargetExtTy()) + report_fatal_error("Type mismatch " + CurrentType->getTargetExtName() + + "/" + AssignedType->getTargetExtName() + + " for value " + V->getName(), + false); + + // Our previous guess about the type seems to be wrong, let's update + // inferred type according to a new, more precise type information. + updateAssignType(AssignCI, V, PoisonValue::get(AssignedType)); } void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( @@ -850,7 +878,7 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( ExpectedElementTypeConst, Pointer, {B.getInt32(AddressSpace)}, B); GR->addDeducedElementType(CI, ExpectedElementType); GR->addDeducedElementType(Pointer, ExpectedElementType); - AssignPtrTypeInstr[Pointer] = CI; + GR->addAssignPtrTypeInstr(Pointer, CI); return; } @@ -929,8 +957,7 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I, for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) { Value *ArgOperand = CI->getArgOperand(OpIdx); - if (!isa(ArgOperand->getType()) && - !isa(ArgOperand->getType())) + if (!isPointerTy(ArgOperand->getType())) continue; // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs() @@ -952,8 +979,8 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I, continue; if (ExpectedType->isTargetExtTy()) - insertAssignTypeInstrForTargetExtTypes(cast(ExpectedType), - ArgOperand, B); + insertAssignPtrTypeTargetExt(cast(ExpectedType), + ArgOperand, B); else replacePointerOperandWithPtrCast(CI, ArgOperand, ExpectedType, OpIdx, B); } @@ -1145,7 +1172,7 @@ void SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I, CallInst *CI = buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {I->getType()}, EltTyConst, I, {B.getInt32(AddressSpace)}, B); GR->addDeducedElementType(CI, ElemTy); - AssignPtrTypeInstr[I] = CI; + GR->addAssignPtrTypeInstr(I, CI); } void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, @@ -1164,20 +1191,32 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, TypeToAssign = It->second; } } - Constant *Const = UndefValue::get(TypeToAssign); - buildIntrWithMD(Intrinsic::spv_assign_type, {Ty}, Const, I, {}, B); + buildAssignType(B, TypeToAssign, I); } for (const auto &Op : I->operands()) { if (isa(Op) || isa(Op) || // Check GetElementPtrConstantExpr case. (isa(Op) && isa(Op))) { setInsertPointSkippingPhis(B, I); - if (isa(Op) && Op->getType()->isAggregateType()) - buildIntrWithMD(Intrinsic::spv_assign_type, {B.getInt32Ty()}, Op, - UndefValue::get(B.getInt32Ty()), {}, B); - else if (!isa(Op)) - buildIntrWithMD(Intrinsic::spv_assign_type, {Op->getType()}, Op, Op, {}, - B); + Type *OpTy = Op->getType(); + if (isa(Op) && OpTy->isAggregateType()) { + CallInst *AssignCI = + buildIntrWithMD(Intrinsic::spv_assign_type, {B.getInt32Ty()}, Op, + UndefValue::get(B.getInt32Ty()), {}, B); + GR->addAssignPtrTypeInstr(Op, AssignCI); + } else if (!isa(Op)) { + Type *OpTy = Op->getType(); + if (auto PType = dyn_cast(OpTy)) { + buildAssignPtr(B, PType->getElementType(), Op); + } else if (isPointerTy(OpTy)) { + Type *ElemTy = GR->findDeducedElementType(Op); + buildAssignPtr(B, ElemTy ? ElemTy : deduceElementType(Op), Op); + } else { + CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, + {OpTy}, Op, Op, {}, B); + GR->addAssignPtrTypeInstr(Op, AssignCI); + } + } } } } @@ -1368,14 +1407,12 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) { continue; insertAssignPtrTypeIntrs(I, B); + deduceOperandElementType(I); insertAssignTypeIntrs(I, B); insertPtrCastOrAssignTypeInstr(I, B); insertSpirvDecorations(I, B); } - for (auto &I : instructions(Func)) - deduceOperandElementType(&I); - for (auto *I : Worklist) { TrackConstants = true; if (!I->getType()->isVoidTy() || isa(I)) diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h index ef0973d03d15..db01f68f48de 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -73,8 +73,11 @@ class SPIRVGlobalRegistry { // untyped pointers. DenseMap DeducedElTys; // Maps composite values to deduced types where untyped pointers are replaced - // with typed ones + // with typed ones. DenseMap DeducedNestedTys; + // Maps values to "assign type" calls, thus being a registry of created + // Intrinsic::spv_assign_ptr_type instructions. + DenseMap AssignPtrTypeInstr; // Add a new OpTypeXXX instruction without checking for duplicates. SPIRVType *createSPIRVType(const Type *Type, MachineIRBuilder &MIRBuilder, @@ -149,6 +152,17 @@ public: return It == FunResPointerTypes.end() ? nullptr : It->second; } + // A registry of "assign type" records: + // - Add a record. + void addAssignPtrTypeInstr(Value *Val, CallInst *AssignPtrTyCI) { + AssignPtrTypeInstr[Val] = AssignPtrTyCI; + } + // - Find a record. + CallInst *findAssignPtrTypeInstr(const Value *Val) { + auto It = AssignPtrTypeInstr.find(Val); + return It == AssignPtrTypeInstr.end() ? nullptr : It->second; + } + // Deduced element types of untyped pointers and composites: // - Add a record to the map of deduced element types. void addDeducedElementType(Value *Val, Type *Ty) { DeducedElTys[Val] = Ty; } diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index 3d536085b78a..a0a253c23b1e 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -417,7 +417,8 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR, MachineInstr *Def = MRI.getVRegDef(Reg); assert(Def && "Expecting an instruction that defines the register"); // G_GLOBAL_VALUE already has type info. - if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE) + if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE && + Def->getOpcode() != SPIRV::ASSIGN_TYPE) insertAssignInstr(Reg, nullptr, AssignedPtrType, GR, MIB, MF.getRegInfo()); ToErase.push_back(&MI); @@ -427,7 +428,8 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR, MachineInstr *Def = MRI.getVRegDef(Reg); assert(Def && "Expecting an instruction that defines the register"); // G_GLOBAL_VALUE already has type info. - if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE) + if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE && + Def->getOpcode() != SPIRV::ASSIGN_TYPE) insertAssignInstr(Reg, Ty, nullptr, GR, MIB, MF.getRegInfo()); ToErase.push_back(&MI); } else if (MIOp == TargetOpcode::G_CONSTANT || diff --git a/llvm/test/CodeGen/SPIRV/event-wait-ptr-type.ll b/llvm/test/CodeGen/SPIRV/event-wait-ptr-type.ll index d6fb70bb59a7..ec9afc789944 100644 --- a/llvm/test/CodeGen/SPIRV/event-wait-ptr-type.ll +++ b/llvm/test/CodeGen/SPIRV/event-wait-ptr-type.ll @@ -4,16 +4,16 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} -; CHECK: %[[#EventTy:]] = OpTypeEvent -; CHECK: %[[#StructEventTy:]] = OpTypeStruct %[[#EventTy]] -; CHECK: %[[#GenPtrStructEventTy:]] = OpTypePointer Generic %[[#StructEventTy]] -; CHECK: %[[#FunPtrStructEventTy:]] = OpTypePointer Function %[[#StructEventTy]] -; CHECK: %[[#GenPtrEventTy:]] = OpTypePointer Generic %[[#EventTy:]] +; CHECK-DAG: %[[#EventTy:]] = OpTypeEvent +; CHECK-DAG: %[[#StructEventTy:]] = OpTypeStruct %[[#EventTy]] +; CHECK-DAG: %[[#FunPtrStructEventTy:]] = OpTypePointer Function %[[#StructEventTy]] +; CHECK-DAG: %[[#GenPtrEventTy:]] = OpTypePointer Generic %[[#EventTy]] +; CHECK-DAG: %[[#FunPtrEventTy:]] = OpTypePointer Function %[[#EventTy]] ; CHECK: OpFunction ; CHECK: %[[#Var:]] = OpVariable %[[#FunPtrStructEventTy]] Function -; CHECK-NEXT: %[[#AddrspacecastVar:]] = OpPtrCastToGeneric %[[#GenPtrStructEventTy]] %[[#Var]] -; CHECK-NEXT: %[[#BitcastVar:]] = OpBitcast %[[#GenPtrEventTy]] %[[#AddrspacecastVar]] -; CHECK-NEXT: OpGroupWaitEvents %[[#]] %[[#]] %[[#BitcastVar]] +; CHECK-NEXT: %[[#FunEvent:]] = OpBitcast %[[#FunPtrEventTy]] %[[#Var]] +; CHECK-NEXT: %[[#GenEvent:]] = OpPtrCastToGeneric %[[#GenPtrEventTy]] %[[#FunEvent]] +; CHECK-NEXT: OpGroupWaitEvents %[[#]] %[[#]] %[[#GenEvent]] %"class.sycl::_V1::device_event" = type { target("spirv.Event") } diff --git a/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-duplicate-spv_assign_type.ll b/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-duplicate-spv_assign_type.ll index 7056b9cb1230..9db4f26a27d4 100644 --- a/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-duplicate-spv_assign_type.ll +++ b/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-duplicate-spv_assign_type.ll @@ -3,9 +3,9 @@ ; CHECK: *** IR Dump After SPIRV emit intrinsics (emit-intrinsics) *** define spir_kernel void @test(ptr addrspace(1) %srcimg) { -; CHECK: call void @llvm.spv.assign.type.p1(ptr addrspace(1) %srcimg, metadata target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) undef) +; CHECK: call void @llvm.spv.assign.type.p1(ptr addrspace(1) %srcimg, metadata target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) poison) %call1 = call spir_func <2 x i32> @_Z13get_image_dim14ocl_image2d_ro(ptr addrspace(1) %srcimg) -; CHECK-NOT: call void @llvm.spv.assign.type.p1(ptr addrspace(1) %srcimg, metadata target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) undef) +; CHECK-NOT: call void @llvm.spv.assign.type.p1(ptr addrspace(1) %srcimg, metadata target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) poison) %call2 = call spir_func <2 x i32> @_Z13get_image_dim14ocl_image2d_ro(ptr addrspace(1) %srcimg) ret void ; CHECK: } diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpGroupAsyncCopy-strided.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpGroupAsyncCopy-strided.ll new file mode 100644 index 000000000000..96d6016083f0 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/transcoding/OpGroupAsyncCopy-strided.ll @@ -0,0 +1,36 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-SPIRV-DAG: %[[#LongTy:]] = OpTypeInt 64 0 +; CHECK-SPIRV-DAG: %[[#IntTy:]] = OpTypeInt 32 0 +; CHECK-SPIRV-DAG: %[[#EventTy:]] = OpTypeEvent +; CHECK-SPIRV-DAG: %[[#Scope:]] = OpConstant %[[#IntTy]] 2 +; CHECK-SPIRV-DAG: %[[#Num:]] = OpConstant %[[#LongTy]] 123 +; CHECK-SPIRV-DAG: %[[#Null:]] = OpConstantNull +; CHECK-SPIRV-DAG: %[[#Stride:]] = OpConstant %[[#LongTy]] 1 +; CHECK-SPIRV-DAG: %[[#GenPtrEventTy:]] = OpTypePointer Generic %[[#EventTy]] +; CHECK-SPIRV-DAG: %[[#FunPtrEventTy:]] = OpTypePointer Function %[[#EventTy]] +; CHECK-SPIRV: OpFunction +; CHECK-SPIRV: %[[#Var:]] = OpVariable %[[#]] Function +; CHECK-SPIRV: %[[#ResEvent:]] = OpGroupAsyncCopy %[[#EventTy]] %[[#Scope]] %[[#Null]] %[[#Null]] %[[#Num]] %[[#Stride]] %[[#Null]] +; CHECK-SPIRV: %[[#VarPtrEvent:]] = OpBitcast %[[#FunPtrEventTy]] %[[#Var]] +; CHECK-SPIRV: OpStore %[[#VarPtrEvent]] %[[#ResEvent]] +; CHECK-SPIRV: %[[#VarPtrEvent2:]] = OpBitcast %[[#FunPtrEventTy]] %[[#Var]] +; CHECK-SPIRV: %[[#PtrEventGen:]] = OpPtrCastToGeneric %[[#]] %[[#VarPtrEvent2]] +; CHECK-SPIRV: OpGroupWaitEvents %[[#Scope]] %[[#Num]] %[[#PtrEventGen]] +; CHECK-SPIRV: OpFunctionEnd + +define spir_kernel void @foo() { + %event = alloca ptr, align 8 + %call = call spir_func ptr @_Z29async_work_group_strided_copyPU3AS3hPU3AS1Khmm9ocl_event(ptr null, ptr null, i64 123, i64 1, ptr null) + store ptr %call, ptr %event, align 8 + %event.ascast = addrspacecast ptr %event to ptr addrspace(4) + call spir_func void @_Z17wait_group_eventsiPU3AS49ocl_event(i64 123, ptr addrspace(4) %event.ascast) + ret void +} + +declare spir_func ptr @_Z29async_work_group_strided_copyPU3AS3hPU3AS1Khmm9ocl_event(ptr, ptr, i64, i64, ptr) +declare spir_func void @_Z17wait_group_eventsiPU3AS49ocl_event(i64, ptr addrspace(4)) diff --git a/llvm/test/CodeGen/SPIRV/transcoding/check_ro_qualifier.ll b/llvm/test/CodeGen/SPIRV/transcoding/check_ro_qualifier.ll index 824ca1b2d692..6f61aba23a46 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/check_ro_qualifier.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/check_ro_qualifier.ll @@ -1,5 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV -; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: %[[#IMAGE_TYPE:]] = OpTypeImage ; CHECK-SPIRV: %[[#IMAGE_ARG:]] = OpFunctionParameter %[[#IMAGE_TYPE]] -- GitLab From b7b8d028962e7a7280e3d65019345f2446785e7e Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 6 Jun 2024 15:33:23 +0100 Subject: [PATCH 106/462] [ConstraintElim] Add induction tests with different start values. Extra tests for https://github.com/llvm/llvm-project/pull/94610. --- .../induction-condition-in-loop-exit.ll | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit.ll b/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit.ll index 44ce82b51d70..2f0b51c410a5 100644 --- a/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit.ll +++ b/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 ; RUN: opt -p constraint-elimination -S %s | FileCheck %s +declare void @llvm.assume(i1) + define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_known(ptr %s) { ; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_known( ; CHECK-SAME: ptr [[S:%.*]]) { @@ -40,6 +42,170 @@ exit: ret i1 %t } +define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known_due_to_start_value(ptr %s) { +; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known_due_to_start_value( +; CHECK-SAME: ptr [[S:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 1235, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]] +; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10 +; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 +; CHECK-NEXT: br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[T:%.*]] = icmp ult i32 [[IV]], 1235 +; CHECK-NEXT: ret i1 [[T]] +; +entry: + br label %loop.header + +loop.header: + %iv = phi i32 [ 1235, %entry ], [ %iv.next, %loop.latch ] + %exitcond.not = icmp eq i32 %iv, 1234 + br i1 %exitcond.not, label %exit, label %loop.latch + +loop.latch: + %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv + %0 = load i8, ptr %arrayidx, align 1 + %latch.c = icmp ult i8 %0, 10 + %iv.next = add i32 %iv, 1 + br i1 %latch.c, label %loop.header, label %exit + +exit: + %t = icmp ult i32 %iv, 1235 + ret i1 %t +} + +define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_known_due_to_precond_on_start_value(ptr %s, i32 %start) { +; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_known_due_to_precond_on_start_value( +; CHECK-SAME: ptr [[S:%.*]], i32 [[START:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[PRE_C:%.*]] = icmp ule i32 [[START]], 1234 +; CHECK-NEXT: call void @llvm.assume(i1 [[PRE_C]]) +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]] +; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10 +; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 +; CHECK-NEXT: br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[T:%.*]] = icmp ult i32 [[IV]], 1235 +; CHECK-NEXT: ret i1 [[T]] +; +entry: + %pre.c = icmp ule i32 %start, 1234 + call void @llvm.assume(i1 %pre.c) + br label %loop.header + +loop.header: + %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop.latch ] + %exitcond.not = icmp eq i32 %iv, 1234 + br i1 %exitcond.not, label %exit, label %loop.latch + +loop.latch: + %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv + %0 = load i8, ptr %arrayidx, align 1 + %latch.c = icmp ult i8 %0, 10 + %iv.next = add i32 %iv, 1 + br i1 %latch.c, label %loop.header, label %exit + +exit: + %t = icmp ult i32 %iv, 1235 + ret i1 %t +} + +define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known_due_to_precond_on_start_value(ptr %s, i32 %start) { +; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known_due_to_precond_on_start_value( +; CHECK-SAME: ptr [[S:%.*]], i32 [[START:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[PRE_C:%.*]] = icmp ule i32 [[START]], 1236 +; CHECK-NEXT: call void @llvm.assume(i1 [[PRE_C]]) +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]] +; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10 +; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 +; CHECK-NEXT: br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[T:%.*]] = icmp ult i32 [[IV]], 1236 +; CHECK-NEXT: ret i1 [[T]] +; +entry: + %pre.c = icmp ule i32 %start, 1236 + call void @llvm.assume(i1 %pre.c) + br label %loop.header + +loop.header: + %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop.latch ] + %exitcond.not = icmp eq i32 %iv, 1234 + br i1 %exitcond.not, label %exit, label %loop.latch + +loop.latch: + %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv + %0 = load i8, ptr %arrayidx, align 1 + %latch.c = icmp ult i8 %0, 10 + %iv.next = add i32 %iv, 1 + br i1 %latch.c, label %loop.header, label %exit + +exit: + %t = icmp ult i32 %iv, 1236 + ret i1 %t +} + +define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known_due_to_missing_precond(ptr %s, i32 %start) { +; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_unique_exit_const_compare_not_known_due_to_missing_precond( +; CHECK-SAME: ptr [[S:%.*]], i32 [[START:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 1234 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i32 [[IV]] +; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[LATCH_C:%.*]] = icmp ult i8 [[TMP0]], 10 +; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 +; CHECK-NEXT: br i1 [[LATCH_C]], label %[[LOOP_HEADER]], label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[T:%.*]] = icmp ult i32 [[IV]], 1236 +; CHECK-NEXT: ret i1 [[T]] +; +entry: + br label %loop.header + +loop.header: + %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop.latch ] + %exitcond.not = icmp eq i32 %iv, 1234 + br i1 %exitcond.not, label %exit, label %loop.latch + +loop.latch: + %arrayidx = getelementptr inbounds i8, ptr %s, i32 %iv + %0 = load i8, ptr %arrayidx, align 1 + %latch.c = icmp ult i8 %0, 10 + %iv.next = add i32 %iv, 1 + br i1 %latch.c, label %loop.header, label %exit + +exit: + %t = icmp ult i32 %iv, 1236 + ret i1 %t +} + define i1 @multi_exiting_loop_eq_same_exit_with_out_loop_preds_const_compare_not_known(ptr %s, i1 %pre.c, i32 %x) { ; CHECK-LABEL: define i1 @multi_exiting_loop_eq_same_exit_with_out_loop_preds_const_compare_not_known( ; CHECK-SAME: ptr [[S:%.*]], i1 [[PRE_C:%.*]], i32 [[X:%.*]]) { -- GitLab From 083a26682d1c4d2d6efb051cb2f658564eaa32ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Thu, 6 Jun 2024 15:35:02 +0200 Subject: [PATCH 107/462] [clang][Interp][NFC] Return a valid SourceInfo for Function PCs We already assert that the given PC is in range and that the function has a body, so the SrcMap should generally never be empty. However, when generating destructors, we create quite a few instructions for which we have no source information, which may cause the previous assertion to fail. Return the end of the source map in this case. --- clang/lib/AST/Interp/Function.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clang/lib/AST/Interp/Function.cpp b/clang/lib/AST/Interp/Function.cpp index 1d04998d5dd1..00f5a1fced53 100644 --- a/clang/lib/AST/Interp/Function.cpp +++ b/clang/lib/AST/Interp/Function.cpp @@ -40,7 +40,8 @@ SourceInfo Function::getSource(CodePtr PC) const { unsigned Offset = PC - getCodeBegin(); using Elem = std::pair; auto It = llvm::lower_bound(SrcMap, Elem{Offset, {}}, llvm::less_first()); - assert(It != SrcMap.end()); + if (It == SrcMap.end()) + return SrcMap.back().second; return It->second; } -- GitLab From 84b026690ded7f7728b6d1ba48b233b6ca8317eb Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Thu, 6 Jun 2024 16:44:07 +0200 Subject: [PATCH 108/462] DAG: Pass flags to FoldConstantArithmetic (#93663) There is simply way too much going on inside getNode. The complicated constant folding of vector handling works by looking for build_vector operands, and then tries to getNode the scalar element and then checks if constants were the result. As a side effect, this produces unused scalar operation nodes (previously, without flags). If the vector operation were later scalarized, it would find the flagless constant folding temporary and lose the flag. I don't think this is a reasonable way for constant folding to operate, but for now fix this by ensuring flags on the original operation are preserved in the temporary. This yields a clear code improvement for AMDGPU when f16 isn't legal. The Wasm cases switch from using a libcall to compare and select. We are evidently missing the fcmp+select to fminimum/fmaximum handling, but this would be further improved when that's handled. AArch64 also avoids the libcall, but looks worse and has a different call for some reason. --- llvm/include/llvm/CodeGen/SelectionDAG.h | 3 +- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 7 +- .../AArch64/vecreduce-fmax-legalization.ll | 14 +- .../AArch64/vecreduce-fmin-legalization.ll | 14 +- llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll | 138 ++++------- llvm/test/CodeGen/WebAssembly/simd-arith.ll | 220 ++++++++++-------- 6 files changed, 202 insertions(+), 194 deletions(-) diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 48cb0cdf851c..7b0e5e7d9504 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1893,7 +1893,8 @@ public: const SDNode *N2); SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, - ArrayRef Ops); + ArrayRef Ops, + SDNodeFlags Flags = SDNodeFlags()); /// Fold floating-point operations when all operands are constants and/or /// undefined. diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 09cdec8adb27..e176cf2cc2a6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -6333,7 +6333,8 @@ bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef Ops) { } SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, - EVT VT, ArrayRef Ops) { + EVT VT, ArrayRef Ops, + SDNodeFlags Flags) { // If the opcode is a target-specific ISD node, there's nothing we can // do here and the operand rules may not line up with the below, so // bail early. @@ -6690,7 +6691,7 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, } // Constant fold the scalar operands. - SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps); + SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags); // Legalize the (integer) scalar constant if necessary. if (LegalSVT != SVT) @@ -7261,7 +7262,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, } // Perform trivial constant folding. - if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2})) + if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}, Flags)) return SV; // Canonicalize an UNDEF to the RHS, even over a constant. diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll index 4c02a5240ba6..c993051ccebf 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll @@ -648,7 +648,19 @@ define float @test_v3f32_ninf(<3 x float> %a) nounwind { define fp128 @test_v2f128(<2 x fp128> %a) nounwind { ; CHECK-LABEL: test_v2f128: ; CHECK: // %bb.0: -; CHECK-NEXT: b fmaxl +; CHECK-NEXT: sub sp, sp, #48 +; CHECK-NEXT: str x30, [sp, #32] // 8-byte Folded Spill +; CHECK-NEXT: stp q0, q1, [sp] // 32-byte Folded Spill +; CHECK-NEXT: bl __gttf2 +; CHECK-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload +; CHECK-NEXT: cmp w0, #0 +; CHECK-NEXT: b.le .LBB18_2 +; CHECK-NEXT: // %bb.1: +; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload +; CHECK-NEXT: .LBB18_2: +; CHECK-NEXT: ldr x30, [sp, #32] // 8-byte Folded Reload +; CHECK-NEXT: add sp, sp, #48 +; CHECK-NEXT: ret %b = call nnan fp128 @llvm.vector.reduce.fmax.v2f128(<2 x fp128> %a) ret fp128 %b } diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll index 18d40cb18ba6..0116be51dd69 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll @@ -648,7 +648,19 @@ define float @test_v3f32_ninf(<3 x float> %a) nounwind { define fp128 @test_v2f128(<2 x fp128> %a) nounwind { ; CHECK-LABEL: test_v2f128: ; CHECK: // %bb.0: -; CHECK-NEXT: b fminl +; CHECK-NEXT: sub sp, sp, #48 +; CHECK-NEXT: str x30, [sp, #32] // 8-byte Folded Spill +; CHECK-NEXT: stp q0, q1, [sp] // 32-byte Folded Spill +; CHECK-NEXT: bl __lttf2 +; CHECK-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload +; CHECK-NEXT: cmp w0, #0 +; CHECK-NEXT: b.ge .LBB18_2 +; CHECK-NEXT: // %bb.1: +; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload +; CHECK-NEXT: .LBB18_2: +; CHECK-NEXT: ldr x30, [sp, #32] // 8-byte Folded Reload +; CHECK-NEXT: add sp, sp, #48 +; CHECK-NEXT: ret %b = call nnan fp128 @llvm.vector.reduce.fmin.v2f128(<2 x fp128> %a) ret fp128 %b } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll index 7d7a46259710..0c13c5348879 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll @@ -654,21 +654,16 @@ define <2 x half> @v_maximum_v2f16__nnan(<2 x half> %src0, <2 x half> %src1) { ; GFX7-LABEL: v_maximum_v2f16__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0 -; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1 +; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0 -; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1 -; GFX7-NEXT: v_mov_b32_e32 v5, 0x7fc00000 -; GFX7-NEXT: v_max_f32_e32 v4, v0, v2 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc -; GFX7-NEXT: v_max_f32_e32 v2, v1, v3 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc +; GFX7-NEXT: v_max_f32_e32 v0, v0, v2 +; GFX7-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v2f16__nnan: @@ -847,21 +842,16 @@ define <2 x half> @v_maximum_v2f16__nnan_nsz(<2 x half> %src0, <2 x half> %src1) ; GFX7-LABEL: v_maximum_v2f16__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0 -; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1 +; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0 -; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1 -; GFX7-NEXT: v_mov_b32_e32 v5, 0x7fc00000 -; GFX7-NEXT: v_max_f32_e32 v4, v0, v2 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc -; GFX7-NEXT: v_max_f32_e32 v2, v1, v3 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc +; GFX7-NEXT: v_max_f32_e32 v0, v0, v2 +; GFX7-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v2f16__nnan_nsz: @@ -1216,28 +1206,21 @@ define <3 x half> @v_maximum_v3f16__nnan(<3 x half> %src0, <3 x half> %src1) { ; GFX7-LABEL: v_maximum_v3f16__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5 +; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2 +; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0 -; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1 -; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5 -; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2 +; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5 +; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0 -; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1 -; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2 -; GFX7-NEXT: v_max_f32_e32 v6, v0, v3 -; GFX7-NEXT: v_mov_b32_e32 v7, 0x7fc00000 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc -; GFX7-NEXT: v_max_f32_e32 v3, v1, v4 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc -; GFX7-NEXT: v_max_f32_e32 v3, v2, v5 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc +; GFX7-NEXT: v_max_f32_e32 v0, v0, v3 +; GFX7-NEXT: v_max_f32_e32 v1, v1, v4 +; GFX7-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v3f16__nnan: @@ -1427,28 +1410,21 @@ define <3 x half> @v_maximum_v3f16__nnan_nsz(<3 x half> %src0, <3 x half> %src1) ; GFX7-LABEL: v_maximum_v3f16__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5 +; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2 +; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0 -; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1 -; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5 -; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2 +; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5 +; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0 -; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1 -; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2 -; GFX7-NEXT: v_max_f32_e32 v6, v0, v3 -; GFX7-NEXT: v_mov_b32_e32 v7, 0x7fc00000 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc -; GFX7-NEXT: v_max_f32_e32 v3, v1, v4 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc -; GFX7-NEXT: v_max_f32_e32 v3, v2, v5 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc +; GFX7-NEXT: v_max_f32_e32 v0, v0, v3 +; GFX7-NEXT: v_max_f32_e32 v1, v1, v4 +; GFX7-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v3f16__nnan_nsz: @@ -1671,35 +1647,26 @@ define <4 x half> @v_maximum_v4f16__nnan(<4 x half> %src0, <4 x half> %src1) { ; GFX7-LABEL: v_maximum_v4f16__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7 +; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6 +; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3 +; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2 +; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0 -; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1 -; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6 -; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2 -; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7 -; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3 +; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v7 +; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v6 +; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0 -; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1 -; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v6 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2 -; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v7 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3 -; GFX7-NEXT: v_max_f32_e32 v8, v0, v4 -; GFX7-NEXT: v_mov_b32_e32 v9, 0x7fc00000 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc -; GFX7-NEXT: v_max_f32_e32 v4, v1, v5 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc -; GFX7-NEXT: v_max_f32_e32 v4, v2, v6 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc -; GFX7-NEXT: v_max_f32_e32 v4, v3, v7 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc +; GFX7-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX7-NEXT: v_max_f32_e32 v1, v1, v5 +; GFX7-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX7-NEXT: v_max_f32_e32 v3, v3, v7 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v4f16__nnan: @@ -1924,35 +1891,26 @@ define <4 x half> @v_maximum_v4f16__nnan_nsz(<4 x half> %src0, <4 x half> %src1) ; GFX7-LABEL: v_maximum_v4f16__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7 +; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6 +; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3 +; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2 +; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0 -; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1 -; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6 -; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2 -; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7 -; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3 +; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v7 +; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v6 +; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0 -; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1 -; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v6 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2 -; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v7 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3 -; GFX7-NEXT: v_max_f32_e32 v8, v0, v4 -; GFX7-NEXT: v_mov_b32_e32 v9, 0x7fc00000 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc -; GFX7-NEXT: v_max_f32_e32 v4, v1, v5 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc -; GFX7-NEXT: v_max_f32_e32 v4, v2, v6 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc -; GFX7-NEXT: v_max_f32_e32 v4, v3, v7 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc +; GFX7-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX7-NEXT: v_max_f32_e32 v1, v1, v5 +; GFX7-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX7-NEXT: v_max_f32_e32 v3, v3, v7 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v4f16__nnan_nsz: diff --git a/llvm/test/CodeGen/WebAssembly/simd-arith.ll b/llvm/test/CodeGen/WebAssembly/simd-arith.ll index 761a75418a00..67388b688e3b 100644 --- a/llvm/test/CodeGen/WebAssembly/simd-arith.ll +++ b/llvm/test/CodeGen/WebAssembly/simd-arith.ll @@ -11788,27 +11788,35 @@ define <4 x float> @minnum_intrinsic_v4f32(<4 x float> %x, <4 x float> %y) { ; NO-SIMD128-LABEL: minnum_intrinsic_v4f32: ; NO-SIMD128: .functype minnum_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> () ; NO-SIMD128-NEXT: # %bb.0: -; NO-SIMD128-NEXT: call $push0=, fminf, $4, $8 -; NO-SIMD128-NEXT: f32.store 12($0), $pop0 -; NO-SIMD128-NEXT: call $push1=, fminf, $3, $7 -; NO-SIMD128-NEXT: f32.store 8($0), $pop1 -; NO-SIMD128-NEXT: call $push2=, fminf, $2, $6 -; NO-SIMD128-NEXT: f32.store 4($0), $pop2 -; NO-SIMD128-NEXT: call $push3=, fminf, $1, $5 -; NO-SIMD128-NEXT: f32.store 0($0), $pop3 +; NO-SIMD128-NEXT: f32.lt $push0=, $4, $8 +; NO-SIMD128-NEXT: f32.select $push1=, $4, $8, $pop0 +; NO-SIMD128-NEXT: f32.store 12($0), $pop1 +; NO-SIMD128-NEXT: f32.lt $push2=, $3, $7 +; NO-SIMD128-NEXT: f32.select $push3=, $3, $7, $pop2 +; NO-SIMD128-NEXT: f32.store 8($0), $pop3 +; NO-SIMD128-NEXT: f32.lt $push4=, $2, $6 +; NO-SIMD128-NEXT: f32.select $push5=, $2, $6, $pop4 +; NO-SIMD128-NEXT: f32.store 4($0), $pop5 +; NO-SIMD128-NEXT: f32.lt $push6=, $1, $5 +; NO-SIMD128-NEXT: f32.select $push7=, $1, $5, $pop6 +; NO-SIMD128-NEXT: f32.store 0($0), $pop7 ; NO-SIMD128-NEXT: return ; ; NO-SIMD128-FAST-LABEL: minnum_intrinsic_v4f32: ; NO-SIMD128-FAST: .functype minnum_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> () ; NO-SIMD128-FAST-NEXT: # %bb.0: -; NO-SIMD128-FAST-NEXT: call $push0=, fminf, $1, $5 -; NO-SIMD128-FAST-NEXT: f32.store 0($0), $pop0 -; NO-SIMD128-FAST-NEXT: call $push1=, fminf, $2, $6 -; NO-SIMD128-FAST-NEXT: f32.store 4($0), $pop1 -; NO-SIMD128-FAST-NEXT: call $push2=, fminf, $3, $7 -; NO-SIMD128-FAST-NEXT: f32.store 8($0), $pop2 -; NO-SIMD128-FAST-NEXT: call $push3=, fminf, $4, $8 -; NO-SIMD128-FAST-NEXT: f32.store 12($0), $pop3 +; NO-SIMD128-FAST-NEXT: f32.lt $push0=, $1, $5 +; NO-SIMD128-FAST-NEXT: f32.select $push1=, $1, $5, $pop0 +; NO-SIMD128-FAST-NEXT: f32.store 0($0), $pop1 +; NO-SIMD128-FAST-NEXT: f32.lt $push2=, $2, $6 +; NO-SIMD128-FAST-NEXT: f32.select $push3=, $2, $6, $pop2 +; NO-SIMD128-FAST-NEXT: f32.store 4($0), $pop3 +; NO-SIMD128-FAST-NEXT: f32.lt $push4=, $3, $7 +; NO-SIMD128-FAST-NEXT: f32.select $push5=, $3, $7, $pop4 +; NO-SIMD128-FAST-NEXT: f32.store 8($0), $pop5 +; NO-SIMD128-FAST-NEXT: f32.lt $push6=, $4, $8 +; NO-SIMD128-FAST-NEXT: f32.select $push7=, $4, $8, $pop6 +; NO-SIMD128-FAST-NEXT: f32.store 12($0), $pop7 ; NO-SIMD128-FAST-NEXT: return %a = call nnan <4 x float> @llvm.minnum.v4f32(<4 x float> %x, <4 x float> %y) ret <4 x float> %a @@ -11830,26 +11838,26 @@ define <4 x float> @minnum_nsz_intrinsic_v4f32(<4 x float> %x, <4 x float> %y) { ; NO-SIMD128-LABEL: minnum_nsz_intrinsic_v4f32: ; NO-SIMD128: .functype minnum_nsz_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> () ; NO-SIMD128-NEXT: # %bb.0: -; NO-SIMD128-NEXT: call $push0=, fminf, $4, $8 +; NO-SIMD128-NEXT: f32.min $push0=, $4, $8 ; NO-SIMD128-NEXT: f32.store 12($0), $pop0 -; NO-SIMD128-NEXT: call $push1=, fminf, $3, $7 +; NO-SIMD128-NEXT: f32.min $push1=, $3, $7 ; NO-SIMD128-NEXT: f32.store 8($0), $pop1 -; NO-SIMD128-NEXT: call $push2=, fminf, $2, $6 +; NO-SIMD128-NEXT: f32.min $push2=, $2, $6 ; NO-SIMD128-NEXT: f32.store 4($0), $pop2 -; NO-SIMD128-NEXT: call $push3=, fminf, $1, $5 +; NO-SIMD128-NEXT: f32.min $push3=, $1, $5 ; NO-SIMD128-NEXT: f32.store 0($0), $pop3 ; NO-SIMD128-NEXT: return ; ; NO-SIMD128-FAST-LABEL: minnum_nsz_intrinsic_v4f32: ; NO-SIMD128-FAST: .functype minnum_nsz_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> () ; NO-SIMD128-FAST-NEXT: # %bb.0: -; NO-SIMD128-FAST-NEXT: call $push0=, fminf, $1, $5 +; NO-SIMD128-FAST-NEXT: f32.min $push0=, $1, $5 ; NO-SIMD128-FAST-NEXT: f32.store 0($0), $pop0 -; NO-SIMD128-FAST-NEXT: call $push1=, fminf, $2, $6 +; NO-SIMD128-FAST-NEXT: f32.min $push1=, $2, $6 ; NO-SIMD128-FAST-NEXT: f32.store 4($0), $pop1 -; NO-SIMD128-FAST-NEXT: call $push2=, fminf, $3, $7 +; NO-SIMD128-FAST-NEXT: f32.min $push2=, $3, $7 ; NO-SIMD128-FAST-NEXT: f32.store 8($0), $pop2 -; NO-SIMD128-FAST-NEXT: call $push3=, fminf, $4, $8 +; NO-SIMD128-FAST-NEXT: f32.min $push3=, $4, $8 ; NO-SIMD128-FAST-NEXT: f32.store 12($0), $pop3 ; NO-SIMD128-FAST-NEXT: return %a = call nnan nsz <4 x float> @llvm.minnum.v4f32(<4 x float> %x, <4 x float> %y) @@ -11875,16 +11883,16 @@ define <4 x float> @fminnumv432_non_zero_intrinsic(<4 x float> %x) { ; NO-SIMD128: .functype fminnumv432_non_zero_intrinsic (i32, f32, f32, f32, f32) -> () ; NO-SIMD128-NEXT: # %bb.0: ; NO-SIMD128-NEXT: f32.const $push0=, -0x1p0 -; NO-SIMD128-NEXT: call $push1=, fminf, $4, $pop0 +; NO-SIMD128-NEXT: f32.min $push1=, $4, $pop0 ; NO-SIMD128-NEXT: f32.store 12($0), $pop1 ; NO-SIMD128-NEXT: f32.const $push7=, -0x1p0 -; NO-SIMD128-NEXT: call $push2=, fminf, $3, $pop7 +; NO-SIMD128-NEXT: f32.min $push2=, $3, $pop7 ; NO-SIMD128-NEXT: f32.store 8($0), $pop2 ; NO-SIMD128-NEXT: f32.const $push6=, -0x1p0 -; NO-SIMD128-NEXT: call $push3=, fminf, $2, $pop6 +; NO-SIMD128-NEXT: f32.min $push3=, $2, $pop6 ; NO-SIMD128-NEXT: f32.store 4($0), $pop3 ; NO-SIMD128-NEXT: f32.const $push5=, -0x1p0 -; NO-SIMD128-NEXT: call $push4=, fminf, $1, $pop5 +; NO-SIMD128-NEXT: f32.min $push4=, $1, $pop5 ; NO-SIMD128-NEXT: f32.store 0($0), $pop4 ; NO-SIMD128-NEXT: return ; @@ -11892,16 +11900,16 @@ define <4 x float> @fminnumv432_non_zero_intrinsic(<4 x float> %x) { ; NO-SIMD128-FAST: .functype fminnumv432_non_zero_intrinsic (i32, f32, f32, f32, f32) -> () ; NO-SIMD128-FAST-NEXT: # %bb.0: ; NO-SIMD128-FAST-NEXT: f32.const $push0=, -0x1p0 -; NO-SIMD128-FAST-NEXT: call $push1=, fminf, $1, $pop0 +; NO-SIMD128-FAST-NEXT: f32.min $push1=, $1, $pop0 ; NO-SIMD128-FAST-NEXT: f32.store 0($0), $pop1 ; NO-SIMD128-FAST-NEXT: f32.const $push7=, -0x1p0 -; NO-SIMD128-FAST-NEXT: call $push2=, fminf, $2, $pop7 +; NO-SIMD128-FAST-NEXT: f32.min $push2=, $2, $pop7 ; NO-SIMD128-FAST-NEXT: f32.store 4($0), $pop2 ; NO-SIMD128-FAST-NEXT: f32.const $push6=, -0x1p0 -; NO-SIMD128-FAST-NEXT: call $push3=, fminf, $3, $pop6 +; NO-SIMD128-FAST-NEXT: f32.min $push3=, $3, $pop6 ; NO-SIMD128-FAST-NEXT: f32.store 8($0), $pop3 ; NO-SIMD128-FAST-NEXT: f32.const $push5=, -0x1p0 -; NO-SIMD128-FAST-NEXT: call $push4=, fminf, $4, $pop5 +; NO-SIMD128-FAST-NEXT: f32.min $push4=, $4, $pop5 ; NO-SIMD128-FAST-NEXT: f32.store 12($0), $pop4 ; NO-SIMD128-FAST-NEXT: return %a = call nnan <4 x float> @llvm.minnum.v4f32(<4 x float> %x, <4 x float>) @@ -11979,34 +11987,38 @@ define <4 x float> @fminnumv432_one_zero_intrinsic(<4 x float> %x) { ; NO-SIMD128: .functype fminnumv432_one_zero_intrinsic (i32, f32, f32, f32, f32) -> () ; NO-SIMD128-NEXT: # %bb.0: ; NO-SIMD128-NEXT: f32.const $push0=, -0x1p0 -; NO-SIMD128-NEXT: call $push1=, fminf, $4, $pop0 +; NO-SIMD128-NEXT: f32.min $push1=, $4, $pop0 ; NO-SIMD128-NEXT: f32.store 12($0), $pop1 -; NO-SIMD128-NEXT: f32.const $push7=, -0x1p0 -; NO-SIMD128-NEXT: call $push2=, fminf, $3, $pop7 +; NO-SIMD128-NEXT: f32.const $push9=, -0x1p0 +; NO-SIMD128-NEXT: f32.min $push2=, $3, $pop9 ; NO-SIMD128-NEXT: f32.store 8($0), $pop2 ; NO-SIMD128-NEXT: f32.const $push3=, 0x0p0 -; NO-SIMD128-NEXT: call $push4=, fminf, $2, $pop3 -; NO-SIMD128-NEXT: f32.store 4($0), $pop4 -; NO-SIMD128-NEXT: f32.const $push6=, -0x1p0 -; NO-SIMD128-NEXT: call $push5=, fminf, $1, $pop6 -; NO-SIMD128-NEXT: f32.store 0($0), $pop5 +; NO-SIMD128-NEXT: f32.const $push8=, 0x0p0 +; NO-SIMD128-NEXT: f32.lt $push4=, $2, $pop8 +; NO-SIMD128-NEXT: f32.select $push5=, $2, $pop3, $pop4 +; NO-SIMD128-NEXT: f32.store 4($0), $pop5 +; NO-SIMD128-NEXT: f32.const $push7=, -0x1p0 +; NO-SIMD128-NEXT: f32.min $push6=, $1, $pop7 +; NO-SIMD128-NEXT: f32.store 0($0), $pop6 ; NO-SIMD128-NEXT: return ; ; NO-SIMD128-FAST-LABEL: fminnumv432_one_zero_intrinsic: ; NO-SIMD128-FAST: .functype fminnumv432_one_zero_intrinsic (i32, f32, f32, f32, f32) -> () ; NO-SIMD128-FAST-NEXT: # %bb.0: ; NO-SIMD128-FAST-NEXT: f32.const $push0=, -0x1p0 -; NO-SIMD128-FAST-NEXT: call $push1=, fminf, $1, $pop0 +; NO-SIMD128-FAST-NEXT: f32.min $push1=, $1, $pop0 ; NO-SIMD128-FAST-NEXT: f32.store 0($0), $pop1 -; NO-SIMD128-FAST-NEXT: f32.const $push2=, 0x0p0 -; NO-SIMD128-FAST-NEXT: call $push3=, fminf, $2, $pop2 -; NO-SIMD128-FAST-NEXT: f32.store 4($0), $pop3 +; NO-SIMD128-FAST-NEXT: f32.const $push9=, -0x1p0 +; NO-SIMD128-FAST-NEXT: f32.min $push2=, $3, $pop9 +; NO-SIMD128-FAST-NEXT: f32.store 8($0), $pop2 +; NO-SIMD128-FAST-NEXT: f32.const $push3=, 0x0p0 +; NO-SIMD128-FAST-NEXT: f32.const $push8=, 0x0p0 +; NO-SIMD128-FAST-NEXT: f32.lt $push4=, $2, $pop8 +; NO-SIMD128-FAST-NEXT: f32.select $push5=, $2, $pop3, $pop4 +; NO-SIMD128-FAST-NEXT: f32.store 4($0), $pop5 ; NO-SIMD128-FAST-NEXT: f32.const $push7=, -0x1p0 -; NO-SIMD128-FAST-NEXT: call $push4=, fminf, $3, $pop7 -; NO-SIMD128-FAST-NEXT: f32.store 8($0), $pop4 -; NO-SIMD128-FAST-NEXT: f32.const $push6=, -0x1p0 -; NO-SIMD128-FAST-NEXT: call $push5=, fminf, $4, $pop6 -; NO-SIMD128-FAST-NEXT: f32.store 12($0), $pop5 +; NO-SIMD128-FAST-NEXT: f32.min $push6=, $4, $pop7 +; NO-SIMD128-FAST-NEXT: f32.store 12($0), $pop6 ; NO-SIMD128-FAST-NEXT: return %a = call nnan <4 x float> @llvm.minnum.v4f32(<4 x float> %x, <4 x float>) ret <4 x float> %a @@ -12126,27 +12138,35 @@ define <4 x float> @maxnum_intrinsic_v4f32(<4 x float> %x, <4 x float> %y) { ; NO-SIMD128-LABEL: maxnum_intrinsic_v4f32: ; NO-SIMD128: .functype maxnum_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> () ; NO-SIMD128-NEXT: # %bb.0: -; NO-SIMD128-NEXT: call $push0=, fmaxf, $4, $8 -; NO-SIMD128-NEXT: f32.store 12($0), $pop0 -; NO-SIMD128-NEXT: call $push1=, fmaxf, $3, $7 -; NO-SIMD128-NEXT: f32.store 8($0), $pop1 -; NO-SIMD128-NEXT: call $push2=, fmaxf, $2, $6 -; NO-SIMD128-NEXT: f32.store 4($0), $pop2 -; NO-SIMD128-NEXT: call $push3=, fmaxf, $1, $5 -; NO-SIMD128-NEXT: f32.store 0($0), $pop3 +; NO-SIMD128-NEXT: f32.gt $push0=, $4, $8 +; NO-SIMD128-NEXT: f32.select $push1=, $4, $8, $pop0 +; NO-SIMD128-NEXT: f32.store 12($0), $pop1 +; NO-SIMD128-NEXT: f32.gt $push2=, $3, $7 +; NO-SIMD128-NEXT: f32.select $push3=, $3, $7, $pop2 +; NO-SIMD128-NEXT: f32.store 8($0), $pop3 +; NO-SIMD128-NEXT: f32.gt $push4=, $2, $6 +; NO-SIMD128-NEXT: f32.select $push5=, $2, $6, $pop4 +; NO-SIMD128-NEXT: f32.store 4($0), $pop5 +; NO-SIMD128-NEXT: f32.gt $push6=, $1, $5 +; NO-SIMD128-NEXT: f32.select $push7=, $1, $5, $pop6 +; NO-SIMD128-NEXT: f32.store 0($0), $pop7 ; NO-SIMD128-NEXT: return ; ; NO-SIMD128-FAST-LABEL: maxnum_intrinsic_v4f32: ; NO-SIMD128-FAST: .functype maxnum_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> () ; NO-SIMD128-FAST-NEXT: # %bb.0: -; NO-SIMD128-FAST-NEXT: call $push0=, fmaxf, $1, $5 -; NO-SIMD128-FAST-NEXT: f32.store 0($0), $pop0 -; NO-SIMD128-FAST-NEXT: call $push1=, fmaxf, $2, $6 -; NO-SIMD128-FAST-NEXT: f32.store 4($0), $pop1 -; NO-SIMD128-FAST-NEXT: call $push2=, fmaxf, $3, $7 -; NO-SIMD128-FAST-NEXT: f32.store 8($0), $pop2 -; NO-SIMD128-FAST-NEXT: call $push3=, fmaxf, $4, $8 -; NO-SIMD128-FAST-NEXT: f32.store 12($0), $pop3 +; NO-SIMD128-FAST-NEXT: f32.gt $push0=, $1, $5 +; NO-SIMD128-FAST-NEXT: f32.select $push1=, $1, $5, $pop0 +; NO-SIMD128-FAST-NEXT: f32.store 0($0), $pop1 +; NO-SIMD128-FAST-NEXT: f32.gt $push2=, $2, $6 +; NO-SIMD128-FAST-NEXT: f32.select $push3=, $2, $6, $pop2 +; NO-SIMD128-FAST-NEXT: f32.store 4($0), $pop3 +; NO-SIMD128-FAST-NEXT: f32.gt $push4=, $3, $7 +; NO-SIMD128-FAST-NEXT: f32.select $push5=, $3, $7, $pop4 +; NO-SIMD128-FAST-NEXT: f32.store 8($0), $pop5 +; NO-SIMD128-FAST-NEXT: f32.gt $push6=, $4, $8 +; NO-SIMD128-FAST-NEXT: f32.select $push7=, $4, $8, $pop6 +; NO-SIMD128-FAST-NEXT: f32.store 12($0), $pop7 ; NO-SIMD128-FAST-NEXT: return %a = call nnan <4 x float> @llvm.maxnum.v4f32(<4 x float> %x, <4 x float> %y) ret <4 x float> %a @@ -12168,26 +12188,26 @@ define <4 x float> @maxnum_nsz_intrinsic_v4f32(<4 x float> %x, <4 x float> %y) { ; NO-SIMD128-LABEL: maxnum_nsz_intrinsic_v4f32: ; NO-SIMD128: .functype maxnum_nsz_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> () ; NO-SIMD128-NEXT: # %bb.0: -; NO-SIMD128-NEXT: call $push0=, fmaxf, $4, $8 +; NO-SIMD128-NEXT: f32.max $push0=, $4, $8 ; NO-SIMD128-NEXT: f32.store 12($0), $pop0 -; NO-SIMD128-NEXT: call $push1=, fmaxf, $3, $7 +; NO-SIMD128-NEXT: f32.max $push1=, $3, $7 ; NO-SIMD128-NEXT: f32.store 8($0), $pop1 -; NO-SIMD128-NEXT: call $push2=, fmaxf, $2, $6 +; NO-SIMD128-NEXT: f32.max $push2=, $2, $6 ; NO-SIMD128-NEXT: f32.store 4($0), $pop2 -; NO-SIMD128-NEXT: call $push3=, fmaxf, $1, $5 +; NO-SIMD128-NEXT: f32.max $push3=, $1, $5 ; NO-SIMD128-NEXT: f32.store 0($0), $pop3 ; NO-SIMD128-NEXT: return ; ; NO-SIMD128-FAST-LABEL: maxnum_nsz_intrinsic_v4f32: ; NO-SIMD128-FAST: .functype maxnum_nsz_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> () ; NO-SIMD128-FAST-NEXT: # %bb.0: -; NO-SIMD128-FAST-NEXT: call $push0=, fmaxf, $1, $5 +; NO-SIMD128-FAST-NEXT: f32.max $push0=, $1, $5 ; NO-SIMD128-FAST-NEXT: f32.store 0($0), $pop0 -; NO-SIMD128-FAST-NEXT: call $push1=, fmaxf, $2, $6 +; NO-SIMD128-FAST-NEXT: f32.max $push1=, $2, $6 ; NO-SIMD128-FAST-NEXT: f32.store 4($0), $pop1 -; NO-SIMD128-FAST-NEXT: call $push2=, fmaxf, $3, $7 +; NO-SIMD128-FAST-NEXT: f32.max $push2=, $3, $7 ; NO-SIMD128-FAST-NEXT: f32.store 8($0), $pop2 -; NO-SIMD128-FAST-NEXT: call $push3=, fmaxf, $4, $8 +; NO-SIMD128-FAST-NEXT: f32.max $push3=, $4, $8 ; NO-SIMD128-FAST-NEXT: f32.store 12($0), $pop3 ; NO-SIMD128-FAST-NEXT: return %a = call nnan nsz <4 x float> @llvm.maxnum.v4f32(<4 x float> %x, <4 x float> %y) @@ -12265,34 +12285,38 @@ define <4 x float> @maxnum_one_zero_intrinsic_v4f32(<4 x float> %x, <4 x float> ; NO-SIMD128: .functype maxnum_one_zero_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> () ; NO-SIMD128-NEXT: # %bb.0: ; NO-SIMD128-NEXT: f32.const $push0=, -0x1p0 -; NO-SIMD128-NEXT: call $push1=, fmaxf, $4, $pop0 +; NO-SIMD128-NEXT: f32.max $push1=, $4, $pop0 ; NO-SIMD128-NEXT: f32.store 12($0), $pop1 -; NO-SIMD128-NEXT: f32.const $push7=, -0x1p0 -; NO-SIMD128-NEXT: call $push2=, fmaxf, $3, $pop7 +; NO-SIMD128-NEXT: f32.const $push9=, -0x1p0 +; NO-SIMD128-NEXT: f32.max $push2=, $3, $pop9 ; NO-SIMD128-NEXT: f32.store 8($0), $pop2 ; NO-SIMD128-NEXT: f32.const $push3=, 0x0p0 -; NO-SIMD128-NEXT: call $push4=, fmaxf, $2, $pop3 -; NO-SIMD128-NEXT: f32.store 4($0), $pop4 -; NO-SIMD128-NEXT: f32.const $push6=, -0x1p0 -; NO-SIMD128-NEXT: call $push5=, fmaxf, $1, $pop6 -; NO-SIMD128-NEXT: f32.store 0($0), $pop5 +; NO-SIMD128-NEXT: f32.const $push8=, 0x0p0 +; NO-SIMD128-NEXT: f32.gt $push4=, $2, $pop8 +; NO-SIMD128-NEXT: f32.select $push5=, $2, $pop3, $pop4 +; NO-SIMD128-NEXT: f32.store 4($0), $pop5 +; NO-SIMD128-NEXT: f32.const $push7=, -0x1p0 +; NO-SIMD128-NEXT: f32.max $push6=, $1, $pop7 +; NO-SIMD128-NEXT: f32.store 0($0), $pop6 ; NO-SIMD128-NEXT: return ; ; NO-SIMD128-FAST-LABEL: maxnum_one_zero_intrinsic_v4f32: ; NO-SIMD128-FAST: .functype maxnum_one_zero_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> () ; NO-SIMD128-FAST-NEXT: # %bb.0: ; NO-SIMD128-FAST-NEXT: f32.const $push0=, -0x1p0 -; NO-SIMD128-FAST-NEXT: call $push1=, fmaxf, $1, $pop0 +; NO-SIMD128-FAST-NEXT: f32.max $push1=, $1, $pop0 ; NO-SIMD128-FAST-NEXT: f32.store 0($0), $pop1 -; NO-SIMD128-FAST-NEXT: f32.const $push2=, 0x0p0 -; NO-SIMD128-FAST-NEXT: call $push3=, fmaxf, $2, $pop2 -; NO-SIMD128-FAST-NEXT: f32.store 4($0), $pop3 +; NO-SIMD128-FAST-NEXT: f32.const $push9=, -0x1p0 +; NO-SIMD128-FAST-NEXT: f32.max $push2=, $3, $pop9 +; NO-SIMD128-FAST-NEXT: f32.store 8($0), $pop2 +; NO-SIMD128-FAST-NEXT: f32.const $push3=, 0x0p0 +; NO-SIMD128-FAST-NEXT: f32.const $push8=, 0x0p0 +; NO-SIMD128-FAST-NEXT: f32.gt $push4=, $2, $pop8 +; NO-SIMD128-FAST-NEXT: f32.select $push5=, $2, $pop3, $pop4 +; NO-SIMD128-FAST-NEXT: f32.store 4($0), $pop5 ; NO-SIMD128-FAST-NEXT: f32.const $push7=, -0x1p0 -; NO-SIMD128-FAST-NEXT: call $push4=, fmaxf, $3, $pop7 -; NO-SIMD128-FAST-NEXT: f32.store 8($0), $pop4 -; NO-SIMD128-FAST-NEXT: f32.const $push6=, -0x1p0 -; NO-SIMD128-FAST-NEXT: call $push5=, fmaxf, $4, $pop6 -; NO-SIMD128-FAST-NEXT: f32.store 12($0), $pop5 +; NO-SIMD128-FAST-NEXT: f32.max $push6=, $4, $pop7 +; NO-SIMD128-FAST-NEXT: f32.store 12($0), $pop6 ; NO-SIMD128-FAST-NEXT: return %a = call nnan <4 x float> @llvm.maxnum.v4f32(<4 x float> %x, <4 x float>) ret <4 x float> %a @@ -12317,16 +12341,16 @@ define <4 x float> @maxnum_non_zero_intrinsic_v4f32(<4 x float> %x, <4 x float> ; NO-SIMD128: .functype maxnum_non_zero_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> () ; NO-SIMD128-NEXT: # %bb.0: ; NO-SIMD128-NEXT: f32.const $push0=, -0x1p0 -; NO-SIMD128-NEXT: call $push1=, fmaxf, $4, $pop0 +; NO-SIMD128-NEXT: f32.max $push1=, $4, $pop0 ; NO-SIMD128-NEXT: f32.store 12($0), $pop1 ; NO-SIMD128-NEXT: f32.const $push7=, -0x1p0 -; NO-SIMD128-NEXT: call $push2=, fmaxf, $3, $pop7 +; NO-SIMD128-NEXT: f32.max $push2=, $3, $pop7 ; NO-SIMD128-NEXT: f32.store 8($0), $pop2 ; NO-SIMD128-NEXT: f32.const $push3=, 0x1p0 -; NO-SIMD128-NEXT: call $push4=, fmaxf, $2, $pop3 +; NO-SIMD128-NEXT: f32.max $push4=, $2, $pop3 ; NO-SIMD128-NEXT: f32.store 4($0), $pop4 ; NO-SIMD128-NEXT: f32.const $push6=, -0x1p0 -; NO-SIMD128-NEXT: call $push5=, fmaxf, $1, $pop6 +; NO-SIMD128-NEXT: f32.max $push5=, $1, $pop6 ; NO-SIMD128-NEXT: f32.store 0($0), $pop5 ; NO-SIMD128-NEXT: return ; @@ -12334,16 +12358,16 @@ define <4 x float> @maxnum_non_zero_intrinsic_v4f32(<4 x float> %x, <4 x float> ; NO-SIMD128-FAST: .functype maxnum_non_zero_intrinsic_v4f32 (i32, f32, f32, f32, f32, f32, f32, f32, f32) -> () ; NO-SIMD128-FAST-NEXT: # %bb.0: ; NO-SIMD128-FAST-NEXT: f32.const $push0=, -0x1p0 -; NO-SIMD128-FAST-NEXT: call $push1=, fmaxf, $1, $pop0 +; NO-SIMD128-FAST-NEXT: f32.max $push1=, $1, $pop0 ; NO-SIMD128-FAST-NEXT: f32.store 0($0), $pop1 ; NO-SIMD128-FAST-NEXT: f32.const $push2=, 0x1p0 -; NO-SIMD128-FAST-NEXT: call $push3=, fmaxf, $2, $pop2 +; NO-SIMD128-FAST-NEXT: f32.max $push3=, $2, $pop2 ; NO-SIMD128-FAST-NEXT: f32.store 4($0), $pop3 ; NO-SIMD128-FAST-NEXT: f32.const $push7=, -0x1p0 -; NO-SIMD128-FAST-NEXT: call $push4=, fmaxf, $3, $pop7 +; NO-SIMD128-FAST-NEXT: f32.max $push4=, $3, $pop7 ; NO-SIMD128-FAST-NEXT: f32.store 8($0), $pop4 ; NO-SIMD128-FAST-NEXT: f32.const $push6=, -0x1p0 -; NO-SIMD128-FAST-NEXT: call $push5=, fmaxf, $4, $pop6 +; NO-SIMD128-FAST-NEXT: f32.max $push5=, $4, $pop6 ; NO-SIMD128-FAST-NEXT: f32.store 12($0), $pop5 ; NO-SIMD128-FAST-NEXT: return %a = call nnan <4 x float> @llvm.maxnum.v4f32(<4 x float> %x, <4 x float>) -- GitLab From 554a2fa4b2b18c81dc7f47599c4704577eb3a129 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Thu, 6 Jun 2024 07:45:38 -0700 Subject: [PATCH 109/462] [WebAssembly] Fix element segments in wasm64 object files (#94617) Followup to #94487 --- llvm/lib/MC/WasmObjectWriter.cpp | 3 ++- llvm/test/MC/WebAssembly/reloc-pic64.s | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/llvm/lib/MC/WasmObjectWriter.cpp b/llvm/lib/MC/WasmObjectWriter.cpp index 5207af803823..788e92f94b26 100644 --- a/llvm/lib/MC/WasmObjectWriter.cpp +++ b/llvm/lib/MC/WasmObjectWriter.cpp @@ -1022,7 +1022,8 @@ void WasmObjectWriter::writeElemSection( encodeULEB128(TableNumber, W->OS); // the table number // init expr for starting offset - W->OS << char(wasm::WASM_OPCODE_I32_CONST); + W->OS << char(is64Bit() ? wasm::WASM_OPCODE_I64_CONST + : wasm::WASM_OPCODE_I32_CONST); encodeSLEB128(InitialTableOffset, W->OS); W->OS << char(wasm::WASM_OPCODE_END); diff --git a/llvm/test/MC/WebAssembly/reloc-pic64.s b/llvm/test/MC/WebAssembly/reloc-pic64.s index 981d8f0e094c..4c5ec4f30666 100644 --- a/llvm/test/MC/WebAssembly/reloc-pic64.s +++ b/llvm/test/MC/WebAssembly/reloc-pic64.s @@ -110,7 +110,7 @@ hidden_func: # CHECK-NEXT: - Type: ELEM # CHECK-NEXT: Segments: # CHECK-NEXT: Offset: -# CHECK-NEXT: Opcode: I32_CONST +# CHECK-NEXT: Opcode: I64_CONST # CHECK-NEXT: Value: 1 # CHECK-NEXT: Functions: [ 5 ] # CHECK-NEXT: - Type: DATACOUNT -- GitLab From 2df68e0503d1cc8cc5e16f4370f2262ff05f2d2d Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Thu, 6 Jun 2024 07:51:47 -0700 Subject: [PATCH 110/462] [MLIR] Fix generic assembly syntax for ArrayAttr containing hex float (#94583) When a float attribute is printed with Hex, we should not elide the type because it is parsed back as i64 otherwise. --- mlir/lib/IR/AsmPrinter.cpp | 11 ++++++++--- mlir/test/IR/array-of-attr.mlir | 4 ++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp index 29e36210f127..6a362afc52f2 100644 --- a/mlir/lib/IR/AsmPrinter.cpp +++ b/mlir/lib/IR/AsmPrinter.cpp @@ -2061,7 +2061,8 @@ void AsmPrinter::Impl::printLocationInternal(LocationAttr loc, bool pretty, /// Print a floating point value in a way that the parser will be able to /// round-trip losslessly. -static void printFloatValue(const APFloat &apValue, raw_ostream &os) { +static void printFloatValue(const APFloat &apValue, raw_ostream &os, + bool *printedHex = nullptr) { // We would like to output the FP constant value in exponential notation, // but we cannot do this if doing so will lose precision. Check here to // make sure that we only output it in exponential format if we can parse @@ -2102,6 +2103,8 @@ static void printFloatValue(const APFloat &apValue, raw_ostream &os) { // Print special values in hexadecimal format. The sign bit should be included // in the literal. + if (printedHex) + *printedHex = true; SmallVector str; APInt apInt = apValue.bitcastToAPInt(); apInt.toString(str, /*Radix=*/16, /*Signed=*/false, @@ -2275,10 +2278,12 @@ void AsmPrinter::Impl::printAttributeImpl(Attribute attr, return; } else if (auto floatAttr = llvm::dyn_cast(attr)) { - printFloatValue(floatAttr.getValue(), os); + bool printedHex = false; + printFloatValue(floatAttr.getValue(), os, &printedHex); // FloatAttr elides the type if F64. - if (typeElision == AttrTypeElision::May && floatAttr.getType().isF64()) + if (typeElision == AttrTypeElision::May && floatAttr.getType().isF64() && + !printedHex) return; } else if (auto strAttr = llvm::dyn_cast(attr)) { diff --git a/mlir/test/IR/array-of-attr.mlir b/mlir/test/IR/array-of-attr.mlir index 1b6fe5520595..c2a607596582 100644 --- a/mlir/test/IR/array-of-attr.mlir +++ b/mlir/test/IR/array-of-attr.mlir @@ -12,3 +12,7 @@ test.array_of_attr_op // CHECK: test.array_of_attr_op // CHECK-SAME: a = [], b = [], c = [] test.array_of_attr_op a = [], b = [], c = [] + +// CHECK: "test.test_array_float" +// CHECK-SAME: 1.000000e+00 : f32, 1.000000e+00, 0x7FF0000000000000 : f64 +"test.test_array_float"() {test.float_arr = [1.0 : f32, 1.0 : f64, 0x7FF0000000000000 : f64]} : () -> () -- GitLab From 7ef83f5561b34ca07fdef23ca2b3c01c583dbbf5 Mon Sep 17 00:00:00 2001 From: Max191 <44243577+Max191@users.noreply.github.com> Date: Thu, 6 Jun 2024 07:54:27 -0700 Subject: [PATCH 111/462] [mlir] Add pack/unpack transpose foldings for linalg.generic ops, fix bugs (#93055) This PR adds transpose + pack/unpack folding support for transpose ops in the form of `linalg.generic` ops. There were also some bugs with the permutation composing in the previous patterns, so this PR fixes these bugs and adds tests for them as well. --- .../Transforms/PackAndUnpackPatterns.cpp | 123 ++++++++++------ .../Tensor/fold-into-pack-and-unpack.mlir | 139 ++++++++++++++++++ 2 files changed, 221 insertions(+), 41 deletions(-) diff --git a/mlir/lib/Dialect/Tensor/Transforms/PackAndUnpackPatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/PackAndUnpackPatterns.cpp index 5d6e3ec9756a..c681cadcb27c 100644 --- a/mlir/lib/Dialect/Tensor/Transforms/PackAndUnpackPatterns.cpp +++ b/mlir/lib/Dialect/Tensor/Transforms/PackAndUnpackPatterns.cpp @@ -48,6 +48,34 @@ static LogicalResult isPackOn1D(RewriterBase &rewriter, Operation *op, return success(); } +// If the `linalgOp` represents a transpose, return the permutation vector for +// the transpose. Otherwise, return failure. +static FailureOr> +getTransposeOpPermutation(linalg::LinalgOp linalgOp) { + if (auto transposeOp = dyn_cast(linalgOp.getOperation())) + return SmallVector(transposeOp.getPermutation()); + if (linalgOp.getNumParallelLoops() != linalgOp.getNumLoops()) + return failure(); + + if (linalgOp.getNumDpsInputs() != 1 || linalgOp.getNumDpsInits() != 1) + return failure(); + auto mapRange = linalgOp.getIndexingMapsArray(); + if (!mapRange.front().isPermutation() || !mapRange.back().isPermutation() || + mapRange.front() == mapRange.back()) { + return failure(); + } + if (!llvm::hasSingleElement(linalgOp.getBlock()->getOperations())) + return failure(); + AffineMap outMap = mapRange.back(); + AffineMap inMap = mapRange.front(); + // To get the permutation, look at each output index and find which + // dimension in the input we're reading from for that index. + return llvm::map_to_vector(outMap.getResults(), + [&](AffineExpr expr) -> int64_t { + return *inMap.getResultPosition(expr); + }); +} + /// Packing one-dimensional tensor can be expressed as an expand shape op. struct SimplifyPackToExpandShape : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; @@ -246,14 +274,10 @@ static bool checkAndPermute(ArrayRef permutation, for (unsigned int i = 0; i < rank; ++i) { int64_t remappedPosition = permutation[i]; - - if (!inVec.empty()) { - if (remappedPosition >= rank) { - return false; - } + if (remappedPosition >= rank) + return false; + if (!inVec.empty()) remappedPosition = inVec[remappedPosition]; - } - resVec.push_back(remappedPosition); } @@ -263,20 +287,25 @@ static bool checkAndPermute(ArrayRef permutation, /// Fold 'pack' -> 'transpose' into 'pack' since 'pack' already has transpose /// semantics. struct FoldProducerPackWithConsumerLinalgTransposeOp - : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; + : public OpInterfaceRewritePattern { + using OpInterfaceRewritePattern::OpInterfaceRewritePattern; - LogicalResult matchAndRewrite(linalg::TransposeOp transposeOp, + LogicalResult matchAndRewrite(linalg::LinalgOp linalgOp, PatternRewriter &rewriter) const override { - auto packOp = transposeOp.getOperand(0).getDefiningOp(); + auto packOp = linalgOp->getOperand(0).getDefiningOp(); if (!packOp) return failure(); + FailureOr> maybePerm = + getTransposeOpPermutation(linalgOp); + if (failed(maybePerm)) + return failure(); + auto innerDimsPos = packOp.getInnerDimsPos(); auto mixedInnerTiles = packOp.getMixedTiles(); auto outerDimsPerm = packOp.getOuterDimsPerm(); - auto transposePerm = transposeOp.getPermutation(); + auto transposePerm = maybePerm.value(); SmallVector newOuterDimsPermVec; SmallVector newInnerDimsPosVec; SmallVector newMixedInnerTilesVec; @@ -285,7 +314,7 @@ struct FoldProducerPackWithConsumerLinalgTransposeOp if (!checkAndPermute(transposePerm, outerDimsPerm, newOuterDimsPermVec, srcRank)) return rewriter.notifyMatchFailure( - transposeOp, + linalgOp, "Cannot fold in tensor.pack if a tile dimension was transposed " "with a non-tile dimension in linalg.transpose."); @@ -297,11 +326,11 @@ struct FoldProducerPackWithConsumerLinalgTransposeOp } Value output = packOp.createDestinationTensor( - rewriter, transposeOp.getLoc(), packOp.getSource(), - newMixedInnerTilesVec, newInnerDimsPosVec, newOuterDimsPermVec); + rewriter, linalgOp.getLoc(), packOp.getSource(), newMixedInnerTilesVec, + newInnerDimsPosVec, newOuterDimsPermVec); rewriter.replaceOpWithNewOp( - transposeOp, packOp.getSource(), output, newInnerDimsPosVec, + linalgOp, packOp.getSource(), output, newInnerDimsPosVec, newMixedInnerTilesVec, packOp.getPaddingValue(), newOuterDimsPermVec); return success(); @@ -316,12 +345,16 @@ struct FoldConsumerPackWithProducerLinalgTransposeOp LogicalResult matchAndRewrite(PackOp packOp, PatternRewriter &rewriter) const override { - auto transposeOp = packOp.getSource().getDefiningOp(); + auto linalgOp = packOp.getSource().getDefiningOp(); + if (!linalgOp) + return failure(); - if (!transposeOp) + FailureOr> maybePerm = + getTransposeOpPermutation(linalgOp); + if (failed(maybePerm)) return failure(); - auto transposePermutation = transposeOp.getPermutation(); + auto transposePermutation = maybePerm.value(); auto outerDimsPerm = packOp.getOuterDimsPerm(); auto innerDimsPos = packOp.getInnerDimsPos(); SmallVector newInnerDimsPosVec; @@ -337,11 +370,11 @@ struct FoldConsumerPackWithProducerLinalgTransposeOp newInnerDimsPosVec.push_back(transposePermutation[dim]); Value output = packOp.createDestinationTensor( - rewriter, packOp.getLoc(), transposeOp.getOperand(0), + rewriter, packOp.getLoc(), linalgOp->getOperand(0), packOp.getMixedTiles(), newInnerDimsPosVec, newOuterDimsPermVec); rewriter.replaceOpWithNewOp( - packOp, transposeOp.getOperand(0), output, newInnerDimsPosVec, + packOp, linalgOp->getOperand(0), output, newInnerDimsPosVec, packOp.getMixedTiles(), packOp.getPaddingValue(), newOuterDimsPermVec); return success(); @@ -351,34 +384,38 @@ struct FoldConsumerPackWithProducerLinalgTransposeOp /// Fold 'unpack' -> 'transpose' into 'unpack' since 'unpack' already has /// transpose semantics. struct FoldProducerUnPackWithConsumerLinalgTransposeOp - : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; + : public OpInterfaceRewritePattern { + using OpInterfaceRewritePattern::OpInterfaceRewritePattern; - LogicalResult matchAndRewrite(linalg::TransposeOp transposeOp, + LogicalResult matchAndRewrite(linalg::LinalgOp linalgOp, PatternRewriter &rewriter) const override { - auto unPackOp = transposeOp.getOperand(0).getDefiningOp(); + auto unPackOp = linalgOp->getOperand(0).getDefiningOp(); if (!unPackOp) return failure(); - auto transposePermutation = transposeOp.getPermutation(); + FailureOr> maybePerm = + getTransposeOpPermutation(linalgOp); + if (failed(maybePerm)) + return failure(); + auto outerDimsPerm = unPackOp.getOuterDimsPerm(); auto innerDimsPos = unPackOp.getInnerDimsPos(); SmallVector newInnerDimsPosVec; SmallVector newOuterDimsPermVec = - llvm::to_vector(transposePermutation); - - if (!outerDimsPerm.empty()) - applyPermutationToVector(newOuterDimsPermVec, outerDimsPerm); + invertPermutationVector(maybePerm.value()); // Can't use applyPermutationToVector for newInnerDimsPosVec since input and // permutation rank won't necessarily be equal in all cases. for (auto dim : innerDimsPos) - newInnerDimsPosVec.push_back(transposePermutation[dim]); + newInnerDimsPosVec.push_back(newOuterDimsPermVec[dim]); + + if (!outerDimsPerm.empty()) + applyPermutationToVector(newOuterDimsPermVec, outerDimsPerm); // Reuse the destination of the transpose op. rewriter.replaceOpWithNewOp( - transposeOp, unPackOp.getSource(), transposeOp.getDpsInits()[0], + linalgOp, unPackOp.getSource(), linalgOp.getDpsInits()[0], newInnerDimsPosVec, unPackOp.getMixedTiles(), newOuterDimsPermVec); return success(); @@ -393,13 +430,17 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp LogicalResult matchAndRewrite(UnPackOp unPackOp, PatternRewriter &rewriter) const override { - auto transposeOp = - unPackOp.getSource().getDefiningOp(); + auto linalgOp = unPackOp.getSource().getDefiningOp(); + if (!linalgOp) + return failure(); - if (!transposeOp) + FailureOr> maybePerm = + getTransposeOpPermutation(linalgOp); + if (failed(maybePerm)) return failure(); - auto transposePermutation = transposeOp.getPermutation(); + SmallVector inverseTransposePerm = + invertPermutationVector(maybePerm.value()); auto outerDimsPerm = unPackOp.getOuterDimsPerm(); auto innerDimsPos = unPackOp.getInnerDimsPos(); int64_t destRank = unPackOp.getSourceRank() - innerDimsPos.size(); @@ -408,7 +449,7 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp SmallVector newInnerDimsPosVec; SmallVector newMixedInnerTilesVec; - if (!checkAndPermute(transposePermutation, outerDimsPerm, + if (!checkAndPermute(inverseTransposePerm, outerDimsPerm, newOuterDimsPermVec, destRank)) return rewriter.notifyMatchFailure( unPackOp, @@ -416,18 +457,18 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp "with a non-tile dimension in linalg.transpose."); // Process transpose operation for tiled inner dimensions - for (unsigned int i = destRank; i < transposePermutation.size(); ++i) { - int64_t remappedPosition = transposePermutation[i] - destRank; + for (unsigned int i = destRank; i < inverseTransposePerm.size(); ++i) { + int64_t remappedPosition = inverseTransposePerm[i] - destRank; newMixedInnerTilesVec.push_back(mixedInnerTilesVec[remappedPosition]); newInnerDimsPosVec.push_back(innerDimsPos[remappedPosition]); } Value output = unPackOp.createDestinationTensor( - rewriter, unPackOp.getLoc(), transposeOp.getOperand(0), + rewriter, unPackOp.getLoc(), linalgOp->getOperand(0), newMixedInnerTilesVec, newInnerDimsPosVec, newOuterDimsPermVec); rewriter.replaceOpWithNewOp( - unPackOp, transposeOp.getOperand(0), output, newInnerDimsPosVec, + unPackOp, linalgOp->getOperand(0), output, newInnerDimsPosVec, newMixedInnerTilesVec, newOuterDimsPermVec); return success(); diff --git a/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir b/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir index 9a3143f5e550..629a4c213572 100644 --- a/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir +++ b/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir @@ -636,3 +636,142 @@ func.func @tensor_padded_unpack_linalg_transpose_fold(%arg0: tensor<71x7x4x16x16 // CHECK-SAME: into %[[OUT:.+]] : tensor<71x7x4x16x16xf32> -> tensor<100x71x64xf32> // CHECK: return %[[UNPACK]] : tensor<100x71x64xf32> // CHECK: } + +// ----- + +func.func @non_involution_transpose_unpack_fold(%arg0: tensor<2x3x5x4x16xi32>) -> tensor<5x48x8xi32> { + %0 = tensor.empty() : tensor<5x2x3x16x4xi32> + %transposed = linalg.transpose ins(%arg0 : tensor<2x3x5x4x16xi32>) + outs(%0 : tensor<5x2x3x16x4xi32>) + permutation = [2, 0, 1, 4, 3] + %1 = tensor.empty() : tensor<5x48x8xi32> + %unpack = tensor.unpack %transposed + outer_dims_perm = [0, 2, 1] + inner_dims_pos = [1, 2] + inner_tiles = [16, 4] into + %1 : tensor<5x2x3x16x4xi32> -> tensor<5x48x8xi32> + return %unpack : tensor<5x48x8xi32> +} +//CHECK-LABEL: func.func @non_involution_transpose_unpack_fold( +// CHECK-SAME: %[[ARG0:.+]]: tensor<2x3x5x4x16xi32>) -> tensor<5x48x8xi32> { +// CHECK: %[[OUT:.+]] = tensor.empty() : tensor<5x48x8xi32> +// CHECK: %[[UNPACK:.+]] = tensor.unpack %[[ARG0]] +// CHECK-SAME: outer_dims_perm = [2, 1, 0] +// CHECK-SAME: inner_dims_pos = [2, 1] +// CHECK-SAME: inner_tiles = [4, 16] +// CHEKC-SAME: into %[[OUT]] : tensor<2x3x5x4x16xi32> -> tensor<5x48x8xi32> +// CHECK: return %[[UNPACK]] : tensor<5x48x8xi32> +// CHECK: } + +// ----- + +func.func @unpack_non_involution_transpose_fold(%arg0: tensor<57x3x56x1x64xf32>) -> tensor<3648x3x56xf32> { + %0 = tensor.empty() : tensor<3x56x3648xf32> + %unpack = tensor.unpack %arg0 + outer_dims_perm = [2, 0, 1] + inner_dims_pos = [1, 2] + inner_tiles = [1, 64] + into %0 : tensor<57x3x56x1x64xf32> -> tensor<3x56x3648xf32> + + %1 = tensor.empty() : tensor<3648x3x56xf32> + %transposed = linalg.transpose + ins(%unpack : tensor<3x56x3648xf32>) + outs(%1 : tensor<3648x3x56xf32>) + permutation = [2, 0, 1] + return %transposed : tensor<3648x3x56xf32> +} +// CHECK-LABEL: func.func @unpack_non_involution_transpose_fold( +// CHECK-SAME: %[[ARG0:.+]]: tensor<57x3x56x1x64xf32>) -> tensor<3648x3x56xf32> { +// CHECK: %[[OUT:.+]] = tensor.empty() : tensor<3648x3x56xf32> +// CHECK: %[[UNPACK:.+]] = tensor.unpack %[[ARG0]] +// CHECK-SAME: outer_dims_perm = [0, 1, 2] +// CHECK-SAME: inner_dims_pos = [2, 0] +// CHECK-SAME: inner_tiles = [1, 64] +// CHECK-SAME: into %[[OUT:.+]] : tensor<57x3x56x1x64xf32> -> tensor<3648x3x56xf32> +// CHECK: return %[[UNPACK]] : tensor<3648x3x56xf32> +// CHECK: } + +// ----- + +func.func @transpose_unpacked_dims_no_fold(%arg0: tensor<2x16x5x4x3xi32>) -> tensor<5x32x12xi32> { + %0 = tensor.empty() : tensor<5x2x3x16x4xi32> + %transposed = linalg.transpose ins(%arg0 : tensor<2x16x5x4x3xi32>) + outs(%0 : tensor<5x2x3x16x4xi32>) + permutation = [2, 0, 4, 1, 3] + %1 = tensor.empty() : tensor<5x32x12xi32> + %unpack = tensor.unpack %transposed + inner_dims_pos = [1, 2] + inner_tiles = [16, 4] into + %1 : tensor<5x2x3x16x4xi32> -> tensor<5x32x12xi32> + return %unpack : tensor<5x32x12xi32> +} +//CHECK-LABEL: func.func @transpose_unpacked_dims_no_fold( +// CHECK: linalg.transpose +// CHECK: tensor.unpack + +// ----- + +#map = affine_map<(d0, d1, d2, d3, d4)->(d1, d2, d0, d4, d3)> +#map1 = affine_map<(d0, d1, d2, d3, d4)->(d0, d1, d2, d3, d4)> +func.func @generic_transpose_unpack_fold(%arg0: tensor<2x3x5x4x16xi32>) -> tensor<5x48x8xi32> { + %0 = tensor.empty() : tensor<5x2x3x16x4xi32> + %transposed = linalg.generic { + iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel"], + indexing_maps = [#map, #map1]} + ins(%arg0 : tensor<2x3x5x4x16xi32>) + outs(%0 : tensor<5x2x3x16x4xi32>) { + ^bb0(%in : i32, %out : i32): + linalg.yield %in : i32 + } -> tensor<5x2x3x16x4xi32> + %1 = tensor.empty() : tensor<5x48x8xi32> + %unpack = tensor.unpack %transposed + outer_dims_perm = [0, 2, 1] + inner_dims_pos = [1, 2] + inner_tiles = [16, 4] into + %1 : tensor<5x2x3x16x4xi32> -> tensor<5x48x8xi32> + return %unpack : tensor<5x48x8xi32> +} +//CHECK-LABEL: func.func @generic_transpose_unpack_fold( +// CHECK-SAME: %[[ARG0:.+]]: tensor<2x3x5x4x16xi32>) -> tensor<5x48x8xi32> { +// CHECK: %[[OUT:.+]] = tensor.empty() : tensor<5x48x8xi32> +// CHECK: %[[UNPACK:.+]] = tensor.unpack %[[ARG0]] +// CHECK-SAME: outer_dims_perm = [2, 1, 0] +// CHECK-SAME: inner_dims_pos = [2, 1] +// CHECK-SAME: inner_tiles = [4, 16] +// CHEKC-SAME: into %[[OUT]] : tensor<2x3x5x4x16xi32> -> tensor<5x48x8xi32> +// CHECK: return %[[UNPACK]] : tensor<5x48x8xi32> +// CHECK: } + +// ----- + +#map = affine_map<(d0, d1, d2)->(d1, d2, d0)> +#map1 = affine_map<(d0, d1, d2)->(d0, d1, d2)> +func.func @unpack_generic_transpose_fold(%arg0: tensor<57x3x56x1x64xf32>) -> tensor<3648x3x56xf32> { + %0 = tensor.empty() : tensor<3x56x3648xf32> + %unpack = tensor.unpack %arg0 + outer_dims_perm = [2, 0, 1] + inner_dims_pos = [1, 2] + inner_tiles = [1, 64] + into %0 : tensor<57x3x56x1x64xf32> -> tensor<3x56x3648xf32> + + %1 = tensor.empty() : tensor<3648x3x56xf32> + %transposed = linalg.generic { + iterator_types = ["parallel", "parallel", "parallel"], + indexing_maps = [#map, #map1]} + ins(%unpack : tensor<3x56x3648xf32>) + outs(%1 : tensor<3648x3x56xf32>) { + ^bb0(%in : f32, %out : f32): + linalg.yield %in : f32 + } -> tensor<3648x3x56xf32> + return %transposed : tensor<3648x3x56xf32> +} +// CHECK-LABEL: func.func @unpack_generic_transpose_fold( +// CHECK-SAME: %[[ARG0:.+]]: tensor<57x3x56x1x64xf32>) -> tensor<3648x3x56xf32> { +// CHECK: %[[OUT:.+]] = tensor.empty() : tensor<3648x3x56xf32> +// CHECK: %[[UNPACK:.+]] = tensor.unpack %[[ARG0]] +// CHECK-SAME: outer_dims_perm = [0, 1, 2] +// CHECK-SAME: inner_dims_pos = [2, 0] +// CHECK-SAME: inner_tiles = [1, 64] +// CHECK-SAME: into %[[OUT:.+]] : tensor<57x3x56x1x64xf32> -> tensor<3648x3x56xf32> +// CHECK: return %[[UNPACK]] : tensor<3648x3x56xf32> +// CHECK: } -- GitLab From 61ef9fdd14f0e7aec2e7dcc10fddd30eec52ee16 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Thu, 6 Jun 2024 14:58:18 +0000 Subject: [PATCH 112/462] [gn build] Port 2a6efe6a49e0 --- llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn b/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn index 210dd1278509..e88df022be78 100644 --- a/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn +++ b/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn @@ -29,6 +29,7 @@ static_library("Core") { "DynoStats.cpp", "Exceptions.cpp", "FunctionLayout.cpp", + "GDBIndex.cpp", "HashUtilities.cpp", "JumpTable.cpp", "MCPlusBuilder.cpp", -- GitLab From 23d86165aabe351aae967301a4ba7d3781f5fb61 Mon Sep 17 00:00:00 2001 From: Adam Paszke Date: Thu, 6 Jun 2024 17:12:09 +0200 Subject: [PATCH 113/462] [Bazel] Generate LLVM_HAS_XYZ_TARGET macros in llvm config (#94476) Otherwise code that depends on those targets being enabled might not get compiled correctly even if the targets are explicitly included in the configuration (in my case NVVM target for MLIR). --- utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index cfc2ac3ae7fb..aebb05d827f7 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -164,6 +164,10 @@ td_library( includes = ["include"], ) +llvm_config_target_defines = [ + "LLVM_HAS_{}_TARGET=1".format(t) for t in llvm_targets +] + cc_library( name = "config", hdrs = [ @@ -171,7 +175,7 @@ cc_library( "include/llvm/Config/llvm-config.h", ], copts = llvm_copts, - defines = llvm_config_defines, + defines = llvm_config_defines + llvm_config_target_defines, includes = ["include"], textual_hdrs = [ "include/llvm/Config/AsmParsers.def", -- GitLab From c0a8fb2120c6cf1e3006d6cc45eccef6fb5069f5 Mon Sep 17 00:00:00 2001 From: aengelke Date: Thu, 6 Jun 2024 17:13:01 +0200 Subject: [PATCH 114/462] [CodeGen] Use std::bitset for MachineFunctionProperties (#94627) The size of the properties is fixed, so no need for a BitVector. Assigning small, fixed-size bitsets is faster. It's a minor performance improvement. --- llvm/include/llvm/CodeGen/MachineFunction.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h index 9f8e846cac45..6e7292abeddb 100644 --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -18,7 +18,6 @@ #define LLVM_CODEGEN_MACHINEFUNCTION_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/SmallVector.h" @@ -34,6 +33,7 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/Recycler.h" #include "llvm/Target/TargetOptions.h" +#include #include #include #include @@ -217,22 +217,21 @@ public: } MachineFunctionProperties &reset(const MachineFunctionProperties &MFP) { - Properties.reset(MFP.Properties); + Properties &= ~MFP.Properties; return *this; } // Returns true if all properties set in V (i.e. required by a pass) are set // in this. bool verifyRequiredProperties(const MachineFunctionProperties &V) const { - return !V.Properties.test(Properties); + return (Properties | ~V.Properties).all(); } /// Print the MachineFunctionProperties in human-readable form. void print(raw_ostream &OS) const; private: - BitVector Properties = - BitVector(static_cast(Property::LastProperty)+1); + std::bitset(Property::LastProperty) + 1> Properties; }; struct SEHHandler { -- GitLab From ac5e278067033636d51bbee41a6ddb9869d0455b Mon Sep 17 00:00:00 2001 From: aengelke Date: Thu, 6 Jun 2024 17:14:39 +0200 Subject: [PATCH 115/462] [X86] Skip AMX type lowering when AMX is not used (#92910) The pass iterates over the IR multiple times, but most code doesn't use AMX. Therefore, do a single iteration in advance to check whether a function uses AMX at all, and exit early if it doesn't. This makes the function-has-AMX path slightly more expensive, but AMX users probably care a lot less about compile time than JIT users (which tend to not use AMX). For us, it reduces the time spent in this pass from 0.62% to 0.12%. Ideally, we wouldn't even need to iterate over the function to determine that it doesn't use AMX. --- llvm/lib/Target/X86/X86LowerAMXType.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp index b69058787a4e..079ac983a8a0 100644 --- a/llvm/lib/Target/X86/X86LowerAMXType.cpp +++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp @@ -92,6 +92,14 @@ static bool isAMXIntrinsic(Value *I) { return false; } +static bool containsAMXCode(Function &F) { + for (BasicBlock &BB : F) + for (Instruction &I : BB) + if (I.getType()->isX86_AMXTy()) + return true; + return false; +} + static AllocaInst *createAllocaInstAtEntry(IRBuilder<> &Builder, BasicBlock *BB, Type *Ty) { Function &F = *BB->getParent(); @@ -1230,6 +1238,14 @@ public: } bool runOnFunction(Function &F) override { + // Performance optimization: most code doesn't use AMX, so return early if + // there are no instructions that produce AMX values. This is sufficient, as + // AMX arguments and constants are not allowed -- so any producer of an AMX + // value must be an instruction. + // TODO: find a cheaper way for this, without looking at all instructions. + if (!containsAMXCode(F)) + return false; + bool C = false; TargetMachine *TM = &getAnalysis().getTM(); TargetLibraryInfo *TLI = -- GitLab From 5be0d00daa5737b6d2165d0c3ec29ce8af068053 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Sat, 1 Jun 2024 21:01:52 +0200 Subject: [PATCH 116/462] RegisterCoalescer: Remove unnecessary maybe_unused 2214026e957397cc6385f778b28d570485a31856 didn't fix an unused variable warning correctly. --- llvm/lib/CodeGen/RegisterCoalescer.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp index 3397bd0a6060..a808a541103f 100644 --- a/llvm/lib/CodeGen/RegisterCoalescer.cpp +++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp @@ -1339,14 +1339,13 @@ bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP, if (SrcIdx && DstIdx) return false; - [[maybe_unused]] const unsigned DefSubIdx = DefMI->getOperand(0).getSubReg(); + const unsigned DefSubIdx = DefMI->getOperand(0).getSubReg(); const TargetRegisterClass *DefRC = TII->getRegClass(MCID, 0, TRI, *MF); if (!DefMI->isImplicitDef()) { if (DstReg.isPhysical()) { Register NewDstReg = DstReg; - unsigned NewDstIdx = TRI->composeSubRegIndices(CP.getSrcIdx(), - DefMI->getOperand(0).getSubReg()); + unsigned NewDstIdx = TRI->composeSubRegIndices(CP.getSrcIdx(), DefSubIdx); if (NewDstIdx) NewDstReg = TRI->getSubReg(DstReg, NewDstIdx); -- GitLab From 3c6d004068a8b7ff036edab6dbbba7ccc2786dae Mon Sep 17 00:00:00 2001 From: OverMighty Date: Thu, 6 Jun 2024 17:16:59 +0200 Subject: [PATCH 117/462] [libc][math][c23] Fix implicit conversion in smoke tests for {fmax,fmin}f16 (#94624) --- libc/test/src/math/smoke/CMakeLists.txt | 10 ++++++++++ libc/test/src/math/smoke/FMaxTest.h | 10 ++++++---- libc/test/src/math/smoke/FMinTest.h | 10 ++++++---- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt index 8919b54262b0..16c6537df2ab 100644 --- a/libc/test/src/math/smoke/CMakeLists.txt +++ b/libc/test/src/math/smoke/CMakeLists.txt @@ -1705,6 +1705,7 @@ add_fp_unittest( FMinTest.h DEPENDS libc.src.math.fminf + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1718,6 +1719,7 @@ add_fp_unittest( FMinTest.h DEPENDS libc.src.math.fmin + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1731,6 +1733,7 @@ add_fp_unittest( FMinTest.h DEPENDS libc.src.math.fminl + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1744,6 +1747,7 @@ add_fp_unittest( FMinTest.h DEPENDS libc.src.math.fminf128 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1757,6 +1761,7 @@ add_fp_unittest( FMinTest.h DEPENDS libc.src.math.fminf16 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1770,6 +1775,7 @@ add_fp_unittest( FMaxTest.h DEPENDS libc.src.math.fmaxf + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1783,6 +1789,7 @@ add_fp_unittest( FMaxTest.h DEPENDS libc.src.math.fmax + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1796,6 +1803,7 @@ add_fp_unittest( FMaxTest.h DEPENDS libc.src.math.fmaxl + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1809,6 +1817,7 @@ add_fp_unittest( FMaxTest.h DEPENDS libc.src.math.fmaxf128 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1822,6 +1831,7 @@ add_fp_unittest( FMaxTest.h DEPENDS libc.src.math.fmaxf16 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) diff --git a/libc/test/src/math/smoke/FMaxTest.h b/libc/test/src/math/smoke/FMaxTest.h index df8e35e0bd16..f4c78b5d04b5 100644 --- a/libc/test/src/math/smoke/FMaxTest.h +++ b/libc/test/src/math/smoke/FMaxTest.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXTEST_H #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXTEST_H +#include "src/__support/CPP/algorithm.h" #include "test/UnitTest/FEnvSafeTest.h" #include "test/UnitTest/FPMatcher.h" #include "test/UnitTest/Test.h" @@ -55,10 +56,11 @@ public: } void testRange(FMaxFunc func) { - constexpr StorageType COUNT = 100'001; - constexpr StorageType STEP = STORAGE_MAX / COUNT; - for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT; - ++i, v += STEP, w -= STEP) { + constexpr int COUNT = 100'001; + constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max( + static_cast(STORAGE_MAX / COUNT), StorageType(1)); + StorageType v = 0, w = STORAGE_MAX; + for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) { FPBits xbits(v), ybits(w); if (xbits.is_inf_or_nan()) continue; diff --git a/libc/test/src/math/smoke/FMinTest.h b/libc/test/src/math/smoke/FMinTest.h index f71b558cd3da..629aaab729a8 100644 --- a/libc/test/src/math/smoke/FMinTest.h +++ b/libc/test/src/math/smoke/FMinTest.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINTEST_H #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINTEST_H +#include "src/__support/CPP/algorithm.h" #include "test/UnitTest/FEnvSafeTest.h" #include "test/UnitTest/FPMatcher.h" #include "test/UnitTest/Test.h" @@ -55,10 +56,11 @@ public: } void testRange(FMinFunc func) { - constexpr StorageType COUNT = 100'001; - constexpr StorageType STEP = STORAGE_MAX / COUNT; - for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT; - ++i, v += STEP, w -= STEP) { + constexpr int COUNT = 100'001; + constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max( + static_cast(STORAGE_MAX / COUNT), StorageType(1)); + StorageType v = 0, w = STORAGE_MAX; + for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) { FPBits xbits(v), ybits(w); if (xbits.is_inf_or_nan()) continue; -- GitLab From dd1cd02a438dc372e1f8457096d01fabb93c9ced Mon Sep 17 00:00:00 2001 From: OverMighty Date: Thu, 6 Jun 2024 17:20:29 +0200 Subject: [PATCH 118/462] [libc][math][c23] Add {fmaximum,fminimum}{,_mag,_mag_num,_num} C23 math functions (#94510) #93566 --- libc/config/linux/aarch64/entrypoints.txt | 8 + libc/config/linux/x86_64/entrypoints.txt | 8 + libc/docs/c23.rst | 14 +- libc/docs/math/index.rst | 16 +- libc/spec/stdc.td | 8 + libc/src/math/CMakeLists.txt | 8 + libc/src/math/fmaximum_mag_numf16.h | 20 +++ libc/src/math/fmaximum_magf16.h | 20 +++ libc/src/math/fmaximum_numf16.h | 20 +++ libc/src/math/fmaximumf16.h | 20 +++ libc/src/math/fminimum_mag_numf16.h | 20 +++ libc/src/math/fminimum_magf16.h | 20 +++ libc/src/math/fminimum_numf16.h | 20 +++ libc/src/math/fminimumf16.h | 20 +++ libc/src/math/generic/CMakeLists.txt | 106 +++++++++++- libc/src/math/generic/fmaximum_mag_numf16.cpp | 19 +++ libc/src/math/generic/fmaximum_magf16.cpp | 19 +++ libc/src/math/generic/fmaximum_numf16.cpp | 19 +++ libc/src/math/generic/fmaximumf16.cpp | 19 +++ libc/src/math/generic/fminimum_mag_numf16.cpp | 19 +++ libc/src/math/generic/fminimum_magf16.cpp | 19 +++ libc/src/math/generic/fminimum_numf16.cpp | 19 +++ libc/src/math/generic/fminimumf16.cpp | 19 +++ libc/test/src/math/smoke/CMakeLists.txt | 151 +++++++++++++++++- libc/test/src/math/smoke/FMaximumMagNumTest.h | 15 +- libc/test/src/math/smoke/FMaximumMagTest.h | 15 +- libc/test/src/math/smoke/FMaximumNumTest.h | 15 +- libc/test/src/math/smoke/FMaximumTest.h | 15 +- libc/test/src/math/smoke/FMinimumMagNumTest.h | 15 +- libc/test/src/math/smoke/FMinimumMagTest.h | 15 +- libc/test/src/math/smoke/FMinimumNumTest.h | 15 +- libc/test/src/math/smoke/FMinimumTest.h | 15 +- .../math/smoke/fmaximum_mag_numf16_test.cpp | 13 ++ .../src/math/smoke/fmaximum_magf16_test.cpp | 13 ++ .../src/math/smoke/fmaximum_numf16_test.cpp | 13 ++ libc/test/src/math/smoke/fmaximumf16_test.cpp | 13 ++ .../math/smoke/fminimum_mag_numf16_test.cpp | 13 ++ .../src/math/smoke/fminimum_magf16_test.cpp | 13 ++ .../src/math/smoke/fminimum_numf16_test.cpp | 13 ++ libc/test/src/math/smoke/fminimumf16_test.cpp | 13 ++ 40 files changed, 781 insertions(+), 74 deletions(-) create mode 100644 libc/src/math/fmaximum_mag_numf16.h create mode 100644 libc/src/math/fmaximum_magf16.h create mode 100644 libc/src/math/fmaximum_numf16.h create mode 100644 libc/src/math/fmaximumf16.h create mode 100644 libc/src/math/fminimum_mag_numf16.h create mode 100644 libc/src/math/fminimum_magf16.h create mode 100644 libc/src/math/fminimum_numf16.h create mode 100644 libc/src/math/fminimumf16.h create mode 100644 libc/src/math/generic/fmaximum_mag_numf16.cpp create mode 100644 libc/src/math/generic/fmaximum_magf16.cpp create mode 100644 libc/src/math/generic/fmaximum_numf16.cpp create mode 100644 libc/src/math/generic/fmaximumf16.cpp create mode 100644 libc/src/math/generic/fminimum_mag_numf16.cpp create mode 100644 libc/src/math/generic/fminimum_magf16.cpp create mode 100644 libc/src/math/generic/fminimum_numf16.cpp create mode 100644 libc/src/math/generic/fminimumf16.cpp create mode 100644 libc/test/src/math/smoke/fmaximum_mag_numf16_test.cpp create mode 100644 libc/test/src/math/smoke/fmaximum_magf16_test.cpp create mode 100644 libc/test/src/math/smoke/fmaximum_numf16_test.cpp create mode 100644 libc/test/src/math/smoke/fmaximumf16_test.cpp create mode 100644 libc/test/src/math/smoke/fminimum_mag_numf16_test.cpp create mode 100644 libc/test/src/math/smoke/fminimum_magf16_test.cpp create mode 100644 libc/test/src/math/smoke/fminimum_numf16_test.cpp create mode 100644 libc/test/src/math/smoke/fminimumf16_test.cpp diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index 13a4445aca76..33ecff813a1f 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -506,7 +506,15 @@ if(LIBC_TYPES_HAS_FLOAT16) libc.src.math.fdimf16 libc.src.math.floorf16 libc.src.math.fmaxf16 + libc.src.math.fmaximumf16 + libc.src.math.fmaximum_magf16 + libc.src.math.fmaximum_mag_numf16 + libc.src.math.fmaximum_numf16 libc.src.math.fminf16 + libc.src.math.fminimumf16 + libc.src.math.fminimum_magf16 + libc.src.math.fminimum_mag_numf16 + libc.src.math.fminimum_numf16 libc.src.math.fromfpf16 libc.src.math.fromfpxf16 libc.src.math.llrintf16 diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 9db044e953f3..ebacb1c59cee 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -539,7 +539,15 @@ if(LIBC_TYPES_HAS_FLOAT16) libc.src.math.fdimf16 libc.src.math.floorf16 libc.src.math.fmaxf16 + libc.src.math.fmaximumf16 + libc.src.math.fmaximum_magf16 + libc.src.math.fmaximum_mag_numf16 + libc.src.math.fmaximum_numf16 libc.src.math.fminf16 + libc.src.math.fminimumf16 + libc.src.math.fminimum_magf16 + libc.src.math.fminimum_mag_numf16 + libc.src.math.fminimum_numf16 libc.src.math.fromfpf16 libc.src.math.fromfpxf16 libc.src.math.llrintf16 diff --git a/libc/docs/c23.rst b/libc/docs/c23.rst index fa7a59893f89..71232cc004c7 100644 --- a/libc/docs/c23.rst +++ b/libc/docs/c23.rst @@ -62,12 +62,14 @@ Additions: * nextup* |check| * nextdown* |check| * canonicalize* |check| - * fmaximum* - * fminimum* - * fmaximum_mag* - * fminimum_mag* - * fmaximum_mag_num* - * fminimum_mag_num* + * fmaximum* |check| + * fminimum* |check| + * fmaximum_mag* |check| + * fminimum_mag* |check| + * fmaximum_num* |check| + * fminimum_num* |check| + * fmaximum_mag_num* |check| + * fminimum_mag_num* |check| * fadd* * fsub* * fmul* diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst index 8c3c07a35b93..b9507f0887cd 100644 --- a/libc/docs/math/index.rst +++ b/libc/docs/math/index.rst @@ -138,23 +138,23 @@ Basic Operations +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ | fmax | |check| | |check| | |check| | |check| | |check| | 7.12.12.2 | F.10.9.2 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| fmaximum | |check| | |check| | |check| | | |check| | 7.12.12.4 | F.10.9.4 | +| fmaximum | |check| | |check| | |check| | |check| | |check| | 7.12.12.4 | F.10.9.4 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| fmaximum_mag | |check| | |check| | |check| | | |check| | 7.12.12.6 | F.10.9.4 | +| fmaximum_mag | |check| | |check| | |check| | |check| | |check| | 7.12.12.6 | F.10.9.4 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| fmaximum_mag_num | |check| | |check| | |check| | | |check| | 7.12.12.10 | F.10.9.5 | +| fmaximum_mag_num | |check| | |check| | |check| | |check| | |check| | 7.12.12.10 | F.10.9.5 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| fmaximum_num | |check| | |check| | |check| | | |check| | 7.12.12.8 | F.10.9.5 | +| fmaximum_num | |check| | |check| | |check| | |check| | |check| | 7.12.12.8 | F.10.9.5 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ | fmin | |check| | |check| | |check| | |check| | |check| | 7.12.12.3 | F.10.9.3 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| fminimum | |check| | |check| | |check| | | |check| | 7.12.12.5 | F.10.9.4 | +| fminimum | |check| | |check| | |check| | |check| | |check| | 7.12.12.5 | F.10.9.4 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| fminimum_mag | |check| | |check| | |check| | | |check| | 7.12.12.7 | F.10.9.4 | +| fminimum_mag | |check| | |check| | |check| | |check| | |check| | 7.12.12.7 | F.10.9.4 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| fminimum_mag_num | |check| | |check| | |check| | | |check| | 7.12.12.11 | F.10.9.5 | +| fminimum_mag_num | |check| | |check| | |check| | |check| | |check| | 7.12.12.11 | F.10.9.5 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ -| fminimum_num | |check| | |check| | |check| | | |check| | 7.12.12.9 | F.10.9.5 | +| fminimum_num | |check| | |check| | |check| | |check| | |check| | 7.12.12.9 | F.10.9.5 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ | fmod | |check| | |check| | |check| | | |check| | 7.12.10.1 | F.10.7.1 | +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+ diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index f867a3d27403..210f2a132516 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -427,41 +427,49 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"fmaximum", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fmaximumf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fmaximuml", RetValSpec, [ArgSpec, ArgSpec]>, + GuardedFunctionSpec<"fmaximumf16", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, GuardedFunctionSpec<"fmaximumf128", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, FunctionSpec<"fmaximum_num", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fmaximum_numf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fmaximum_numl", RetValSpec, [ArgSpec, ArgSpec]>, + GuardedFunctionSpec<"fmaximum_numf16", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, GuardedFunctionSpec<"fmaximum_numf128", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, FunctionSpec<"fmaximum_mag", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fmaximum_magf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fmaximum_magl", RetValSpec, [ArgSpec, ArgSpec]>, + GuardedFunctionSpec<"fmaximum_magf16", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, GuardedFunctionSpec<"fmaximum_magf128", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, FunctionSpec<"fmaximum_mag_num", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fmaximum_mag_numf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fmaximum_mag_numl", RetValSpec, [ArgSpec, ArgSpec]>, + GuardedFunctionSpec<"fmaximum_mag_numf16", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, GuardedFunctionSpec<"fmaximum_mag_numf128", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, FunctionSpec<"fminimum", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fminimumf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fminimuml", RetValSpec, [ArgSpec, ArgSpec]>, + GuardedFunctionSpec<"fminimumf16", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, GuardedFunctionSpec<"fminimumf128", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, FunctionSpec<"fminimum_num", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fminimum_numf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fmaximum_numl", RetValSpec, [ArgSpec, ArgSpec]>, + GuardedFunctionSpec<"fminimum_numf16", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, GuardedFunctionSpec<"fminimum_numf128", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, FunctionSpec<"fminimum_mag", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fminimum_magf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fminimum_magl", RetValSpec, [ArgSpec, ArgSpec]>, + GuardedFunctionSpec<"fminimum_magf16", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, GuardedFunctionSpec<"fminimum_magf128", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, FunctionSpec<"fminimum_mag_num", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fminimum_mag_numf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"fminimum_mag_numl", RetValSpec, [ArgSpec, ArgSpec]>, + GuardedFunctionSpec<"fminimum_mag_numf16", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT16">, GuardedFunctionSpec<"fminimum_mag_numf128", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, FunctionSpec<"fma", RetValSpec, [ArgSpec, ArgSpec, ArgSpec]>, diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt index 498466692fcc..7a349ddc5372 100644 --- a/libc/src/math/CMakeLists.txt +++ b/libc/src/math/CMakeLists.txt @@ -135,41 +135,49 @@ add_math_entrypoint_object(fminf16) add_math_entrypoint_object(fmaximum) add_math_entrypoint_object(fmaximumf) add_math_entrypoint_object(fmaximuml) +add_math_entrypoint_object(fmaximumf16) add_math_entrypoint_object(fmaximumf128) add_math_entrypoint_object(fmaximum_num) add_math_entrypoint_object(fmaximum_numf) add_math_entrypoint_object(fmaximum_numl) +add_math_entrypoint_object(fmaximum_numf16) add_math_entrypoint_object(fmaximum_numf128) add_math_entrypoint_object(fmaximum_mag) add_math_entrypoint_object(fmaximum_magf) add_math_entrypoint_object(fmaximum_magl) +add_math_entrypoint_object(fmaximum_magf16) add_math_entrypoint_object(fmaximum_magf128) add_math_entrypoint_object(fmaximum_mag_num) add_math_entrypoint_object(fmaximum_mag_numf) add_math_entrypoint_object(fmaximum_mag_numl) +add_math_entrypoint_object(fmaximum_mag_numf16) add_math_entrypoint_object(fmaximum_mag_numf128) add_math_entrypoint_object(fminimum) add_math_entrypoint_object(fminimumf) add_math_entrypoint_object(fminimuml) +add_math_entrypoint_object(fminimumf16) add_math_entrypoint_object(fminimumf128) add_math_entrypoint_object(fminimum_num) add_math_entrypoint_object(fminimum_numf) add_math_entrypoint_object(fminimum_numl) +add_math_entrypoint_object(fminimum_numf16) add_math_entrypoint_object(fminimum_numf128) add_math_entrypoint_object(fminimum_mag) add_math_entrypoint_object(fminimum_magf) add_math_entrypoint_object(fminimum_magl) +add_math_entrypoint_object(fminimum_magf16) add_math_entrypoint_object(fminimum_magf128) add_math_entrypoint_object(fminimum_mag_num) add_math_entrypoint_object(fminimum_mag_numf) add_math_entrypoint_object(fminimum_mag_numl) +add_math_entrypoint_object(fminimum_mag_numf16) add_math_entrypoint_object(fminimum_mag_numf128) add_math_entrypoint_object(fmod) diff --git a/libc/src/math/fmaximum_mag_numf16.h b/libc/src/math/fmaximum_mag_numf16.h new file mode 100644 index 000000000000..4c963d4dccc7 --- /dev/null +++ b/libc/src/math/fmaximum_mag_numf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for fmaximum_mag_numf16 -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMF16_H +#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 fmaximum_mag_numf16(float16 x, float16 y); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMF16_H diff --git a/libc/src/math/fmaximum_magf16.h b/libc/src/math/fmaximum_magf16.h new file mode 100644 index 000000000000..e5f57d3b7f1d --- /dev/null +++ b/libc/src/math/fmaximum_magf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for fmaximum_magf16 ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGF16_H +#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 fmaximum_magf16(float16 x, float16 y); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGF16_H diff --git a/libc/src/math/fmaximum_numf16.h b/libc/src/math/fmaximum_numf16.h new file mode 100644 index 000000000000..b450a4595648 --- /dev/null +++ b/libc/src/math/fmaximum_numf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for fmaximum_numf16 ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMF16_H +#define LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 fmaximum_numf16(float16 x, float16 y); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMF16_H diff --git a/libc/src/math/fmaximumf16.h b/libc/src/math/fmaximumf16.h new file mode 100644 index 000000000000..806339fde683 --- /dev/null +++ b/libc/src/math/fmaximumf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for fmaximumf16 -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUMF16_H +#define LLVM_LIBC_SRC_MATH_FMAXIMUMF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 fmaximumf16(float16 x, float16 y); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_FMAXIMUMF16_H diff --git a/libc/src/math/fminimum_mag_numf16.h b/libc/src/math/fminimum_mag_numf16.h new file mode 100644 index 000000000000..0fd314b2f5a2 --- /dev/null +++ b/libc/src/math/fminimum_mag_numf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for fminimum_mag_numf16 -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMF16_H +#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 fminimum_mag_numf16(float16 x, float16 y); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMF16_H diff --git a/libc/src/math/fminimum_magf16.h b/libc/src/math/fminimum_magf16.h new file mode 100644 index 000000000000..27673555403c --- /dev/null +++ b/libc/src/math/fminimum_magf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for fminimum_magf16 ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAGF16_H +#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAGF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 fminimum_magf16(float16 x, float16 y); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAGF16_H diff --git a/libc/src/math/fminimum_numf16.h b/libc/src/math/fminimum_numf16.h new file mode 100644 index 000000000000..598ff9d3c32d --- /dev/null +++ b/libc/src/math/fminimum_numf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for fminimum_numf16 ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_NUMF16_H +#define LLVM_LIBC_SRC_MATH_FMINIMUM_NUMF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 fminimum_numf16(float16 x, float16 y); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_NUMF16_H diff --git a/libc/src/math/fminimumf16.h b/libc/src/math/fminimumf16.h new file mode 100644 index 000000000000..86dd240ae406 --- /dev/null +++ b/libc/src/math/fminimumf16.h @@ -0,0 +1,20 @@ +//===-- Implementation header for fminimumf16 -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_FMINIMUMF16_H +#define LLVM_LIBC_SRC_MATH_FMINIMUMF16_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float16 fminimumf16(float16 x, float16 y); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_FMINIMUMF16_H diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt index c183f09fa5e3..b1d786fc6b29 100644 --- a/libc/src/math/generic/CMakeLists.txt +++ b/libc/src/math/generic/CMakeLists.txt @@ -1894,6 +1894,19 @@ add_entrypoint_object( -O2 ) +add_entrypoint_object( + fmaximumf16 + SRCS + fmaximumf16.cpp + HDRS + ../fmaximumf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.basic_operations + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( fmaximumf128 SRCS @@ -1943,6 +1956,19 @@ add_entrypoint_object( -O2 ) +add_entrypoint_object( + fmaximum_numf16 + SRCS + fmaximum_numf16.cpp + HDRS + ../fmaximum_numf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.basic_operations + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( fmaximum_numf128 SRCS @@ -1992,6 +2018,19 @@ add_entrypoint_object( -O2 ) +add_entrypoint_object( + fmaximum_magf16 + SRCS + fmaximum_magf16.cpp + HDRS + ../fmaximum_magf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.basic_operations + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( fmaximum_magf128 SRCS @@ -2005,7 +2044,6 @@ add_entrypoint_object( -O3 ) - add_entrypoint_object( fmaximum_mag_num SRCS @@ -2042,6 +2080,19 @@ add_entrypoint_object( -O2 ) +add_entrypoint_object( + fmaximum_mag_numf16 + SRCS + fmaximum_mag_numf16.cpp + HDRS + ../fmaximum_mag_numf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.basic_operations + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( fmaximum_mag_numf128 SRCS @@ -2091,6 +2142,19 @@ add_entrypoint_object( -O2 ) +add_entrypoint_object( + fminimumf16 + SRCS + fminimumf16.cpp + HDRS + ../fminimumf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.basic_operations + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( fminimumf128 SRCS @@ -2140,6 +2204,19 @@ add_entrypoint_object( -O2 ) +add_entrypoint_object( + fminimum_numf16 + SRCS + fminimum_numf16.cpp + HDRS + ../fminimum_numf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.basic_operations + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( fminimum_numf128 SRCS @@ -2189,6 +2266,19 @@ add_entrypoint_object( -O2 ) +add_entrypoint_object( + fminimum_magf16 + SRCS + fminimum_magf16.cpp + HDRS + ../fminimum_magf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.basic_operations + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( fminimum_magf128 SRCS @@ -2202,7 +2292,6 @@ add_entrypoint_object( -O3 ) - add_entrypoint_object( fminimum_mag_num SRCS @@ -2239,6 +2328,19 @@ add_entrypoint_object( -O2 ) +add_entrypoint_object( + fminimum_mag_numf16 + SRCS + fminimum_mag_numf16.cpp + HDRS + ../fminimum_mag_numf16.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.basic_operations + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( fminimum_mag_numf128 SRCS diff --git a/libc/src/math/generic/fmaximum_mag_numf16.cpp b/libc/src/math/generic/fmaximum_mag_numf16.cpp new file mode 100644 index 000000000000..5055802c4cf8 --- /dev/null +++ b/libc/src/math/generic/fmaximum_mag_numf16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of fmaximum_mag_numf16 function --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmaximum_mag_numf16.h" +#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, fmaximum_mag_numf16, (float16 x, float16 y)) { + return fputil::fmaximum_mag_num(x, y); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/fmaximum_magf16.cpp b/libc/src/math/generic/fmaximum_magf16.cpp new file mode 100644 index 000000000000..fbd5eaccf309 --- /dev/null +++ b/libc/src/math/generic/fmaximum_magf16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of fmaximum_magf16 function ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmaximum_magf16.h" +#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, fmaximum_magf16, (float16 x, float16 y)) { + return fputil::fmaximum_mag(x, y); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/fmaximum_numf16.cpp b/libc/src/math/generic/fmaximum_numf16.cpp new file mode 100644 index 000000000000..187cfbeee6e2 --- /dev/null +++ b/libc/src/math/generic/fmaximum_numf16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of fmaximum_numf16 function ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmaximum_numf16.h" +#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, fmaximum_numf16, (float16 x, float16 y)) { + return fputil::fmaximum_num(x, y); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/fmaximumf16.cpp b/libc/src/math/generic/fmaximumf16.cpp new file mode 100644 index 000000000000..9e194d2ecef6 --- /dev/null +++ b/libc/src/math/generic/fmaximumf16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of fmaximumf16 function ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmaximumf16.h" +#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, fmaximumf16, (float16 x, float16 y)) { + return fputil::fmaximum(x, y); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/fminimum_mag_numf16.cpp b/libc/src/math/generic/fminimum_mag_numf16.cpp new file mode 100644 index 000000000000..1a893c6c4bbc --- /dev/null +++ b/libc/src/math/generic/fminimum_mag_numf16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of fminimum_mag_numf16 function --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fminimum_mag_numf16.h" +#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, fminimum_mag_numf16, (float16 x, float16 y)) { + return fputil::fminimum_mag_num(x, y); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/fminimum_magf16.cpp b/libc/src/math/generic/fminimum_magf16.cpp new file mode 100644 index 000000000000..45183a963e2d --- /dev/null +++ b/libc/src/math/generic/fminimum_magf16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of fminimum_magf16 function ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fminimum_magf16.h" +#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, fminimum_magf16, (float16 x, float16 y)) { + return fputil::fminimum_mag(x, y); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/fminimum_numf16.cpp b/libc/src/math/generic/fminimum_numf16.cpp new file mode 100644 index 000000000000..825ad3e7b63a --- /dev/null +++ b/libc/src/math/generic/fminimum_numf16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of fminimum_numf16 function ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fminimum_numf16.h" +#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, fminimum_numf16, (float16 x, float16 y)) { + return fputil::fminimum_num(x, y); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/fminimumf16.cpp b/libc/src/math/generic/fminimumf16.cpp new file mode 100644 index 000000000000..16f738be7e58 --- /dev/null +++ b/libc/src/math/generic/fminimumf16.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of fminimumf16 function ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fminimumf16.h" +#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float16, fminimumf16, (float16 x, float16 y)) { + return fputil::fminimum(x, y); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt index 16c6537df2ab..110fa1de97d6 100644 --- a/libc/test/src/math/smoke/CMakeLists.txt +++ b/libc/test/src/math/smoke/CMakeLists.txt @@ -1845,6 +1845,21 @@ add_fp_unittest( FMaximumTest.h DEPENDS libc.src.math.fmaximuml + libc.src.__support.CPP.algorithm + libc.src.__support.FPUtil.fp_bits +) + +add_fp_unittest( + fmaximumf16_test + SUITE + libc-math-smoke-tests + SRCS + fmaximumf16_test.cpp + HDRS + FMaximumTest.h + DEPENDS + libc.src.math.fmaximumf16 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1858,6 +1873,7 @@ add_fp_unittest( FMaximumTest.h DEPENDS libc.src.math.fmaximumf128 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1871,6 +1887,7 @@ add_fp_unittest( FMaximumTest.h DEPENDS libc.src.math.fmaximum + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1884,6 +1901,7 @@ add_fp_unittest( FMaximumTest.h DEPENDS libc.src.math.fmaximumf + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1897,6 +1915,7 @@ add_fp_unittest( FMaximumNumTest.h DEPENDS libc.src.math.fmaximum_numf + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1910,6 +1929,7 @@ add_fp_unittest( FMaximumNumTest.h DEPENDS libc.src.math.fmaximum_num + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1923,6 +1943,21 @@ add_fp_unittest( FMaximumNumTest.h DEPENDS libc.src.math.fmaximum_numl + libc.src.__support.CPP.algorithm + libc.src.__support.FPUtil.fp_bits +) + +add_fp_unittest( + fmaximum_numf16_test + SUITE + libc-math-smoke-tests + SRCS + fmaximum_numf16_test.cpp + HDRS + FMaximumNumTest.h + DEPENDS + libc.src.math.fmaximum_numf16 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1936,6 +1971,7 @@ add_fp_unittest( FMaximumNumTest.h DEPENDS libc.src.math.fmaximum_numf128 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -1949,6 +1985,8 @@ add_fp_unittest( FMaximumMagTest.h DEPENDS libc.src.math.fmaximum_magf + libc.src.__support.CPP.algorithm + libc.src.__support.FPUtil.basic_operations libc.src.__support.FPUtil.fp_bits ) @@ -1962,6 +2000,8 @@ add_fp_unittest( FMaximumMagTest.h DEPENDS libc.src.math.fmaximum_mag + libc.src.__support.CPP.algorithm + libc.src.__support.FPUtil.basic_operations libc.src.__support.FPUtil.fp_bits ) @@ -1975,6 +2015,23 @@ add_fp_unittest( FMaximumMagTest.h DEPENDS libc.src.math.fmaximum_magl + libc.src.__support.CPP.algorithm + libc.src.__support.FPUtil.basic_operations + libc.src.__support.FPUtil.fp_bits +) + +add_fp_unittest( + fmaximum_magf16_test + SUITE + libc-math-smoke-tests + SRCS + fmaximum_magf16_test.cpp + HDRS + FMaximumMagTest.h + DEPENDS + libc.src.math.fmaximum_magf16 + libc.src.__support.CPP.algorithm + libc.src.__support.FPUtil.basic_operations libc.src.__support.FPUtil.fp_bits ) @@ -1988,10 +2045,11 @@ add_fp_unittest( FMaximumMagTest.h DEPENDS libc.src.math.fmaximum_magf128 + libc.src.__support.CPP.algorithm + libc.src.__support.FPUtil.basic_operations libc.src.__support.FPUtil.fp_bits ) - add_fp_unittest( fmaximum_mag_numf_test SUITE @@ -2002,6 +2060,7 @@ add_fp_unittest( FMaximumMagNumTest.h DEPENDS libc.src.math.fmaximum_mag_numf + libc.src.__support.FPUtil.basic_operations libc.src.__support.FPUtil.fp_bits ) @@ -2015,6 +2074,7 @@ add_fp_unittest( FMaximumMagNumTest.h DEPENDS libc.src.math.fmaximum_mag_num + libc.src.__support.FPUtil.basic_operations libc.src.__support.FPUtil.fp_bits ) @@ -2028,6 +2088,21 @@ add_fp_unittest( FMaximumMagNumTest.h DEPENDS libc.src.math.fmaximum_mag_numl + libc.src.__support.FPUtil.basic_operations + libc.src.__support.FPUtil.fp_bits +) + +add_fp_unittest( + fmaximum_mag_numf16_test + SUITE + libc-math-smoke-tests + SRCS + fmaximum_mag_numf16_test.cpp + HDRS + FMaximumMagNumTest.h + DEPENDS + libc.src.math.fmaximum_mag_numf16 + libc.src.__support.FPUtil.basic_operations libc.src.__support.FPUtil.fp_bits ) @@ -2041,6 +2116,7 @@ add_fp_unittest( FMaximumMagNumTest.h DEPENDS libc.src.math.fmaximum_mag_numf128 + libc.src.__support.FPUtil.basic_operations libc.src.__support.FPUtil.fp_bits ) @@ -2054,6 +2130,21 @@ add_fp_unittest( FMinimumTest.h DEPENDS libc.src.math.fminimuml + libc.src.__support.CPP.algorithm + libc.src.__support.FPUtil.fp_bits +) + +add_fp_unittest( + fminimumf16_test + SUITE + libc-math-smoke-tests + SRCS + fminimumf16_test.cpp + HDRS + FMinimumTest.h + DEPENDS + libc.src.math.fminimumf16 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2067,6 +2158,7 @@ add_fp_unittest( FMinimumTest.h DEPENDS libc.src.math.fminimumf128 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2080,6 +2172,7 @@ add_fp_unittest( FMinimumTest.h DEPENDS libc.src.math.fminimum + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2093,6 +2186,7 @@ add_fp_unittest( FMinimumTest.h DEPENDS libc.src.math.fminimumf + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2106,6 +2200,7 @@ add_fp_unittest( FMinimumNumTest.h DEPENDS libc.src.math.fminimum_numf + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2119,6 +2214,7 @@ add_fp_unittest( FMinimumNumTest.h DEPENDS libc.src.math.fminimum_num + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2132,6 +2228,21 @@ add_fp_unittest( FMinimumNumTest.h DEPENDS libc.src.math.fminimum_numl + libc.src.__support.CPP.algorithm + libc.src.__support.FPUtil.fp_bits +) + +add_fp_unittest( + fminimum_numf16_test + SUITE + libc-math-smoke-tests + SRCS + fminimum_numf16_test.cpp + HDRS + FMinimumNumTest.h + DEPENDS + libc.src.math.fminimum_numf16 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2145,6 +2256,7 @@ add_fp_unittest( FMinimumNumTest.h DEPENDS libc.src.math.fminimum_numf128 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2158,6 +2270,7 @@ add_fp_unittest( FMinimumMagTest.h DEPENDS libc.src.math.fminimum_magf + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2171,6 +2284,7 @@ add_fp_unittest( FMinimumMagTest.h DEPENDS libc.src.math.fminimum_mag + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2184,6 +2298,21 @@ add_fp_unittest( FMinimumMagTest.h DEPENDS libc.src.math.fminimum_magl + libc.src.__support.CPP.algorithm + libc.src.__support.FPUtil.fp_bits +) + +add_fp_unittest( + fminimum_magf16_test + SUITE + libc-math-smoke-tests + SRCS + fminimum_magf16_test.cpp + HDRS + FMinimumMagTest.h + DEPENDS + libc.src.math.fminimum_magf16 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2197,10 +2326,10 @@ add_fp_unittest( FMinimumMagTest.h DEPENDS libc.src.math.fminimum_magf128 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) - add_fp_unittest( fminimum_mag_numf_test SUITE @@ -2211,6 +2340,7 @@ add_fp_unittest( FMinimumMagNumTest.h DEPENDS libc.src.math.fminimum_mag_numf + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2224,6 +2354,7 @@ add_fp_unittest( FMinimumMagNumTest.h DEPENDS libc.src.math.fminimum_mag_num + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2237,6 +2368,21 @@ add_fp_unittest( FMinimumMagNumTest.h DEPENDS libc.src.math.fminimum_mag_numl + libc.src.__support.CPP.algorithm + libc.src.__support.FPUtil.fp_bits +) + +add_fp_unittest( + fminimum_mag_numf16_test + SUITE + libc-math-smoke-tests + SRCS + fminimum_mag_numf16_test.cpp + HDRS + FMinimumMagNumTest.h + DEPENDS + libc.src.math.fminimum_mag_numf16 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) @@ -2250,6 +2396,7 @@ add_fp_unittest( FMinimumMagNumTest.h DEPENDS libc.src.math.fminimum_mag_numf128 + libc.src.__support.CPP.algorithm libc.src.__support.FPUtil.fp_bits ) diff --git a/libc/test/src/math/smoke/FMaximumMagNumTest.h b/libc/test/src/math/smoke/FMaximumMagNumTest.h index aafb6d2b0d5e..726f87059fc6 100644 --- a/libc/test/src/math/smoke/FMaximumMagNumTest.h +++ b/libc/test/src/math/smoke/FMaximumMagNumTest.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMMAG_NUMTEST_H #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMMAG_NUMTEST_H +#include "src/__support/CPP/algorithm.h" #include "src/__support/FPUtil/BasicOperations.h" #include "src/__support/FPUtil/FPBits.h" #include "test/UnitTest/FEnvSafeTest.h" @@ -68,10 +69,11 @@ public: } void testRange(FMaximumMagNumFunc func) { - constexpr StorageType COUNT = 100'001; - constexpr StorageType STEP = STORAGE_MAX / COUNT; - for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT; - ++i, v += STEP, w -= STEP) { + constexpr int COUNT = 100'001; + constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max( + static_cast(STORAGE_MAX / COUNT), StorageType(1)); + StorageType v = 0, w = STORAGE_MAX; + for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) { FPBits xbits(v), ybits(w); if (xbits.is_inf_or_nan()) continue; @@ -82,11 +84,10 @@ public: if ((x == 0) && (y == 0)) continue; - if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y)) { + if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y)) EXPECT_FP_EQ(x, func(x, y)); - } else { + else EXPECT_FP_EQ(y, func(x, y)); - } } } }; diff --git a/libc/test/src/math/smoke/FMaximumMagTest.h b/libc/test/src/math/smoke/FMaximumMagTest.h index 7bb79a69be58..b5b2c1ca79ab 100644 --- a/libc/test/src/math/smoke/FMaximumMagTest.h +++ b/libc/test/src/math/smoke/FMaximumMagTest.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUM_MAGTEST_H #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUM_MAGTEST_H +#include "src/__support/CPP/algorithm.h" #include "src/__support/FPUtil/BasicOperations.h" #include "test/UnitTest/FEnvSafeTest.h" #include "test/UnitTest/FPMatcher.h" @@ -56,10 +57,11 @@ public: } void testRange(FMaximumMagFunc func) { - constexpr StorageType COUNT = 100'001; - constexpr StorageType STEP = STORAGE_MAX / COUNT; - for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT; - ++i, v += STEP, w -= STEP) { + constexpr int COUNT = 100'001; + constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max( + static_cast(STORAGE_MAX / COUNT), StorageType(1)); + StorageType v = 0, w = STORAGE_MAX; + for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) { FPBits xbits(v), ybits(w); if (xbits.is_inf_or_nan()) continue; @@ -70,11 +72,10 @@ public: if ((x == 0) && (y == 0)) continue; - if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y)) { + if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y)) EXPECT_FP_EQ(x, func(x, y)); - } else { + else EXPECT_FP_EQ(y, func(x, y)); - } } } }; diff --git a/libc/test/src/math/smoke/FMaximumNumTest.h b/libc/test/src/math/smoke/FMaximumNumTest.h index da0ea2c247a9..ec7913509d39 100644 --- a/libc/test/src/math/smoke/FMaximumNumTest.h +++ b/libc/test/src/math/smoke/FMaximumNumTest.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMNUMTEST_H #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMNUMTEST_H +#include "src/__support/CPP/algorithm.h" #include "src/__support/FPUtil/FPBits.h" #include "test/UnitTest/FEnvSafeTest.h" #include "test/UnitTest/FPMatcher.h" @@ -67,10 +68,11 @@ public: } void testRange(FMaximumNumFunc func) { - constexpr StorageType COUNT = 100'001; - constexpr StorageType STEP = STORAGE_MAX / COUNT; - for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT; - ++i, v += STEP, w -= STEP) { + constexpr int COUNT = 100'001; + constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max( + static_cast(STORAGE_MAX / COUNT), StorageType(1)); + StorageType v = 0, w = STORAGE_MAX; + for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) { FPBits xbits(v), ybits(w); if (xbits.is_inf_or_nan()) continue; @@ -81,11 +83,10 @@ public: if ((x == 0) && (y == 0)) continue; - if (x > y) { + if (x > y) EXPECT_FP_EQ(x, func(x, y)); - } else { + else EXPECT_FP_EQ(y, func(x, y)); - } } } }; diff --git a/libc/test/src/math/smoke/FMaximumTest.h b/libc/test/src/math/smoke/FMaximumTest.h index 1bd15163ed75..94e4a343190a 100644 --- a/libc/test/src/math/smoke/FMaximumTest.h +++ b/libc/test/src/math/smoke/FMaximumTest.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMTEST_H #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMTEST_H +#include "src/__support/CPP/algorithm.h" #include "test/UnitTest/FEnvSafeTest.h" #include "test/UnitTest/FPMatcher.h" #include "test/UnitTest/Test.h" @@ -55,10 +56,11 @@ public: } void testRange(FMaximumFunc func) { - constexpr StorageType COUNT = 100'001; - constexpr StorageType STEP = STORAGE_MAX / COUNT; - for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT; - ++i, v += STEP, w -= STEP) { + constexpr int COUNT = 100'001; + constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max( + static_cast(STORAGE_MAX / COUNT), StorageType(1)); + StorageType v = 0, w = STORAGE_MAX; + for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) { FPBits xbits(v), ybits(w); if (xbits.is_inf_or_nan()) continue; @@ -69,11 +71,10 @@ public: if ((x == 0) && (y == 0)) continue; - if (x > y) { + if (x > y) EXPECT_FP_EQ(x, func(x, y)); - } else { + else EXPECT_FP_EQ(y, func(x, y)); - } } } }; diff --git a/libc/test/src/math/smoke/FMinimumMagNumTest.h b/libc/test/src/math/smoke/FMinimumMagNumTest.h index e4b8fd9e3353..2ceca6ff95ba 100644 --- a/libc/test/src/math/smoke/FMinimumMagNumTest.h +++ b/libc/test/src/math/smoke/FMinimumMagNumTest.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMMAG_NUMTEST_H #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMMAG_NUMTEST_H +#include "src/__support/CPP/algorithm.h" #include "src/__support/FPUtil/BasicOperations.h" #include "src/__support/FPUtil/FPBits.h" #include "test/UnitTest/FEnvSafeTest.h" @@ -68,10 +69,11 @@ public: } void testRange(FMinimumMagNumFunc func) { - constexpr StorageType COUNT = 100'001; - constexpr StorageType STEP = STORAGE_MAX / COUNT; - for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT; - ++i, v += STEP, w -= STEP) { + constexpr int COUNT = 100'001; + constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max( + static_cast(STORAGE_MAX / COUNT), StorageType(1)); + StorageType v = 0, w = STORAGE_MAX; + for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) { FPBits xbits(v), ybits(w); if (xbits.is_inf_or_nan()) continue; @@ -82,11 +84,10 @@ public: if ((x == 0) && (y == 0)) continue; - if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y)) { + if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y)) EXPECT_FP_EQ(y, func(x, y)); - } else { + else EXPECT_FP_EQ(x, func(x, y)); - } } } }; diff --git a/libc/test/src/math/smoke/FMinimumMagTest.h b/libc/test/src/math/smoke/FMinimumMagTest.h index 3e16622fe3fa..9c49446795ce 100644 --- a/libc/test/src/math/smoke/FMinimumMagTest.h +++ b/libc/test/src/math/smoke/FMinimumMagTest.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUM_MAGTEST_H #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUM_MAGTEST_H +#include "src/__support/CPP/algorithm.h" #include "src/__support/FPUtil/BasicOperations.h" #include "test/UnitTest/FEnvSafeTest.h" #include "test/UnitTest/FPMatcher.h" @@ -56,10 +57,11 @@ public: } void testRange(FMinimumMagFunc func) { - constexpr StorageType COUNT = 100'001; - constexpr StorageType STEP = STORAGE_MAX / COUNT; - for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT; - ++i, v += STEP, w -= STEP) { + constexpr int COUNT = 100'001; + constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max( + static_cast(STORAGE_MAX / COUNT), StorageType(1)); + StorageType v = 0, w = STORAGE_MAX; + for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) { FPBits xbits(v), ybits(w); if (xbits.is_inf_or_nan()) continue; @@ -70,11 +72,10 @@ public: if ((x == 0) && (y == 0)) continue; - if (LIBC_NAMESPACE::fputil::abs(x) < LIBC_NAMESPACE::fputil::abs(y)) { + if (LIBC_NAMESPACE::fputil::abs(x) < LIBC_NAMESPACE::fputil::abs(y)) EXPECT_FP_EQ(x, func(x, y)); - } else { + else EXPECT_FP_EQ(y, func(x, y)); - } } } }; diff --git a/libc/test/src/math/smoke/FMinimumNumTest.h b/libc/test/src/math/smoke/FMinimumNumTest.h index 6186ea0df17c..8004ee987454 100644 --- a/libc/test/src/math/smoke/FMinimumNumTest.h +++ b/libc/test/src/math/smoke/FMinimumNumTest.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMNUMTEST_H #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMNUMTEST_H +#include "src/__support/CPP/algorithm.h" #include "src/__support/FPUtil/FPBits.h" #include "test/UnitTest/FEnvSafeTest.h" #include "test/UnitTest/FPMatcher.h" @@ -67,10 +68,11 @@ public: } void testRange(FMinimumNumFunc func) { - constexpr StorageType COUNT = 100'001; - constexpr StorageType STEP = STORAGE_MAX / COUNT; - for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT; - ++i, v += STEP, w -= STEP) { + constexpr int COUNT = 100'001; + constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max( + static_cast(STORAGE_MAX / COUNT), StorageType(1)); + StorageType v = 0, w = STORAGE_MAX; + for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) { FPBits xbits(v), ybits(w); if (xbits.is_inf_or_nan()) continue; @@ -81,11 +83,10 @@ public: if ((x == 0) && (y == 0)) continue; - if (x > y) { + if (x > y) EXPECT_FP_EQ(y, func(x, y)); - } else { + else EXPECT_FP_EQ(x, func(x, y)); - } } } }; diff --git a/libc/test/src/math/smoke/FMinimumTest.h b/libc/test/src/math/smoke/FMinimumTest.h index a267f6c78321..242c857fbb99 100644 --- a/libc/test/src/math/smoke/FMinimumTest.h +++ b/libc/test/src/math/smoke/FMinimumTest.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMTEST_H #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMTEST_H +#include "src/__support/CPP/algorithm.h" #include "test/UnitTest/FEnvSafeTest.h" #include "test/UnitTest/FPMatcher.h" #include "test/UnitTest/Test.h" @@ -55,10 +56,11 @@ public: } void testRange(FMinimumFunc func) { - constexpr StorageType COUNT = 100'001; - constexpr StorageType STEP = STORAGE_MAX / COUNT; - for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT; - ++i, v += STEP, w -= STEP) { + constexpr int COUNT = 100'001; + constexpr StorageType STEP = LIBC_NAMESPACE::cpp::max( + static_cast(STORAGE_MAX / COUNT), StorageType(1)); + StorageType v = 0, w = STORAGE_MAX; + for (int i = 0; i <= COUNT; ++i, v += STEP, w -= STEP) { FPBits xbits(v), ybits(w); if (xbits.is_inf_or_nan()) continue; @@ -69,11 +71,10 @@ public: if ((x == 0) && (y == 0)) continue; - if (x > y) { + if (x > y) EXPECT_FP_EQ(y, func(x, y)); - } else { + else EXPECT_FP_EQ(x, func(x, y)); - } } } }; diff --git a/libc/test/src/math/smoke/fmaximum_mag_numf16_test.cpp b/libc/test/src/math/smoke/fmaximum_mag_numf16_test.cpp new file mode 100644 index 000000000000..b11653eb395e --- /dev/null +++ b/libc/test/src/math/smoke/fmaximum_mag_numf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for fmaximum_mag_numf16 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "FMaximumMagNumTest.h" + +#include "src/math/fmaximum_mag_numf16.h" + +LIST_FMAXIMUM_MAG_NUM_TESTS(float16, LIBC_NAMESPACE::fmaximum_mag_numf16) diff --git a/libc/test/src/math/smoke/fmaximum_magf16_test.cpp b/libc/test/src/math/smoke/fmaximum_magf16_test.cpp new file mode 100644 index 000000000000..6df1e4aaf6a9 --- /dev/null +++ b/libc/test/src/math/smoke/fmaximum_magf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for fmaximum_magf16 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "FMaximumMagTest.h" + +#include "src/math/fmaximum_magf16.h" + +LIST_FMAXIMUM_MAG_TESTS(float16, LIBC_NAMESPACE::fmaximum_magf16) diff --git a/libc/test/src/math/smoke/fmaximum_numf16_test.cpp b/libc/test/src/math/smoke/fmaximum_numf16_test.cpp new file mode 100644 index 000000000000..7cb9cb068cf7 --- /dev/null +++ b/libc/test/src/math/smoke/fmaximum_numf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for fmaximum_numf16 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "FMaximumNumTest.h" + +#include "src/math/fmaximum_numf16.h" + +LIST_FMAXIMUM_NUM_TESTS(float16, LIBC_NAMESPACE::fmaximum_numf16) diff --git a/libc/test/src/math/smoke/fmaximumf16_test.cpp b/libc/test/src/math/smoke/fmaximumf16_test.cpp new file mode 100644 index 000000000000..4cbf846a1610 --- /dev/null +++ b/libc/test/src/math/smoke/fmaximumf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for fmaximumf16 -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "FMaximumTest.h" + +#include "src/math/fmaximumf16.h" + +LIST_FMAXIMUM_TESTS(float16, LIBC_NAMESPACE::fmaximumf16) diff --git a/libc/test/src/math/smoke/fminimum_mag_numf16_test.cpp b/libc/test/src/math/smoke/fminimum_mag_numf16_test.cpp new file mode 100644 index 000000000000..2c6aede529cd --- /dev/null +++ b/libc/test/src/math/smoke/fminimum_mag_numf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for fminimum_mag_numf16 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "FMinimumMagNumTest.h" + +#include "src/math/fminimum_mag_numf16.h" + +LIST_FMINIMUM_MAG_NUM_TESTS(float16, LIBC_NAMESPACE::fminimum_mag_numf16) diff --git a/libc/test/src/math/smoke/fminimum_magf16_test.cpp b/libc/test/src/math/smoke/fminimum_magf16_test.cpp new file mode 100644 index 000000000000..3687aecedfd9 --- /dev/null +++ b/libc/test/src/math/smoke/fminimum_magf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for fminimum_magf16 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "FMinimumMagTest.h" + +#include "src/math/fminimum_magf16.h" + +LIST_FMINIMUM_MAG_TESTS(float16, LIBC_NAMESPACE::fminimum_magf16) diff --git a/libc/test/src/math/smoke/fminimum_numf16_test.cpp b/libc/test/src/math/smoke/fminimum_numf16_test.cpp new file mode 100644 index 000000000000..67750816d229 --- /dev/null +++ b/libc/test/src/math/smoke/fminimum_numf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for fminimum_numf16 -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "FMinimumNumTest.h" + +#include "src/math/fminimum_numf16.h" + +LIST_FMINIMUM_NUM_TESTS(float16, LIBC_NAMESPACE::fminimum_numf16) diff --git a/libc/test/src/math/smoke/fminimumf16_test.cpp b/libc/test/src/math/smoke/fminimumf16_test.cpp new file mode 100644 index 000000000000..f8b0577490b9 --- /dev/null +++ b/libc/test/src/math/smoke/fminimumf16_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for fminimumf16 -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "FMinimumTest.h" + +#include "src/math/fminimumf16.h" + +LIST_FMINIMUM_TESTS(float16, LIBC_NAMESPACE::fminimumf16) -- GitLab From a71aecb05fb8021edb9a12d865ae07a20532347e Mon Sep 17 00:00:00 2001 From: Jon Roelofs Date: Thu, 6 Jun 2024 08:26:40 -0700 Subject: [PATCH 119/462] [llvm][ScheduleDAG] Set a fixed size for Sched::Preference (#94523) This trims off 8 bytes from llvm::SUnit: ``` --- before 2024-06-05 12:13:00 +++ after 2024-06-05 12:12:58 @@ -1,65 +1,65 @@ *** Dumping AST Record Layout 0 | class llvm::SUnit 0 | SDNode * Node 8 | MachineInstr * Instr 16 | SUnit * OrigNode 24 | const MCSchedClassDesc * SchedClass 32 | class llvm::SmallVector Preds 32 | class llvm::SmallVectorImpl (base) 32 | class llvm::SmallVectorTemplateBase (base) 32 | class llvm::SmallVectorTemplateCommon (base) 32 | class llvm::SmallVectorBase (base) 32 | void * BeginX 40 | unsigned int Size 44 | unsigned int Capacity 48 | struct llvm::SmallVectorStorage (base) 48 | char[64] InlineElts 112 | class llvm::SmallVector Succs 112 | class llvm::SmallVectorImpl (base) 112 | class llvm::SmallVectorTemplateBase (base) 112 | class llvm::SmallVectorTemplateCommon (base) 112 | class llvm::SmallVectorBase (base) 112 | void * BeginX 120 | unsigned int Size 124 | unsigned int Capacity 128 | struct llvm::SmallVectorStorage (base) 128 | char[64] InlineElts 192 | unsigned int NodeNum 196 | unsigned int NodeQueueId 200 | unsigned int NumPreds 204 | unsigned int NumSuccs 208 | unsigned int NumPredsLeft 212 | unsigned int NumSuccsLeft 216 | unsigned int WeakPredsLeft 220 | unsigned int WeakSuccsLeft 224 | unsigned short NumRegDefsLeft 226 | unsigned short Latency 228:0-0 | _Bool isVRegCycle 228:1-1 | _Bool isCall 228:2-2 | _Bool isCallOp 228:3-3 | _Bool isTwoAddress 228:4-4 | _Bool isCommutable 228:5-5 | _Bool hasPhysRegUses 228:6-6 | _Bool hasPhysRegDefs 228:7-7 | _Bool hasPhysRegClobbers 229:0-0 | _Bool isPending 229:1-1 | _Bool isAvailable 229:2-2 | _Bool isScheduled 229:3-3 | _Bool isScheduleHigh 229:4-4 | _Bool isScheduleLow 229:5-5 | _Bool isCloned 229:6-6 | _Bool isUnbuffered 229:7-7 | _Bool hasReservedResource - 232 | Sched::Preference SchedulingPref - 236:0-0 | _Bool isDepthCurrent - 236:1-1 | _Bool isHeightCurrent - 240 | unsigned int Depth - 244 | unsigned int Height - 248 | unsigned int TopReadyCycle - 252 | unsigned int BotReadyCycle - 256 | const TargetRegisterClass * CopyDstRC - 264 | const TargetRegisterClass * CopySrcRC - | [sizeof=272, dsize=272, align=8, - | nvsize=272, nvalign=8] + 230 | Sched::Preference SchedulingPref + 231:0-0 | _Bool isDepthCurrent + 231:1-1 | _Bool isHeightCurrent + 232 | unsigned int Depth + 236 | unsigned int Height + 240 | unsigned int TopReadyCycle + 244 | unsigned int BotReadyCycle + 248 | const TargetRegisterClass * CopyDstRC + 256 | const TargetRegisterClass * CopySrcRC + | [sizeof=264, dsize=264, align=8, + | nvsize=264, nvalign=8] ``` --- llvm/include/llvm/CodeGen/TargetLowering.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index d1912b1c4c0f..aa7a32e86ad8 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -96,7 +96,7 @@ class Value; namespace Sched { -enum Preference { +enum Preference : uint8_t { None, // No preference Source, // Follow source order. RegPressure, // Scheduling for lowest register pressure. -- GitLab From df168427b314f057c739eaccb21f361d3628f03b Mon Sep 17 00:00:00 2001 From: David Tenty Date: Thu, 6 Jun 2024 11:29:15 -0400 Subject: [PATCH 120/462] [NFC][libc++][test][AIX] fix SIMD test XFAIL for clang before 19 (#94509) 058e4454 added an XFAIL for this test on AIX because of a backend limitation. That backend limitation has been resolved by 0295c2ad and will be available for clang 19, so we should update the test to limit the XFAIL to clang versions before that. --- .../test/std/experimental/simd/simd.class/simd_copy.pass.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libcxx/test/std/experimental/simd/simd.class/simd_copy.pass.cpp b/libcxx/test/std/experimental/simd/simd.class/simd_copy.pass.cpp index 8fcc811f6df3..7d91ca0eada1 100644 --- a/libcxx/test/std/experimental/simd/simd.class/simd_copy.pass.cpp +++ b/libcxx/test/std/experimental/simd/simd.class/simd_copy.pass.cpp @@ -8,9 +8,9 @@ // UNSUPPORTED: c++03, c++11, c++14 -// FIXME: Fatal error with following targets (remove XFAIL when fixed): +// Older versions of clang may encounter a backend error (see 0295c2ad): // Pass-by-value arguments with alignment greater than register width are not supported. -// XFAIL: target=powerpc{{.*}}-ibm-aix7.2.5.7 +// XFAIL: target=powerpc{{.*}}-ibm-{{.*}} && (clang-17 || clang-18) // // -- GitLab From 09f19c7396ecf26623d08c4288b35a60e950fcd8 Mon Sep 17 00:00:00 2001 From: jensmassberg <87519353+jensmassberg@users.noreply.github.com> Date: Thu, 6 Jun 2024 17:32:50 +0200 Subject: [PATCH 121/462] [clang] Fix handling of adding a file with the same name as an existing dir to VFS (#94461) When trying to add a file to clang's VFS via `addFile` and a directory of the same name already exists, we run into a [out-of-bound access](https://github.com/llvm/llvm-project/blob/145815c180fc82c5a55bf568d01d98d250490a55/llvm/lib/Support/Path.cpp#L244). The problem is that the file name is [recognised as existing path]( https://github.com/llvm/llvm-project/blob/145815c180fc82c5a55bf568d01d98d250490a55/llvm/lib/Support/VirtualFileSystem.cpp#L896) and thus continues to process the next part of the path which doesn't exist. This patch adds a check if we have reached the last part of the filename and return false in that case. This we reject to add a file if a directory of the same name already exists. This is in sync with [this check](https://github.com/llvm/llvm-project/blob/145815c180fc82c5a55bf568d01d98d250490a55/llvm/lib/Support/VirtualFileSystem.cpp#L903) that rejects adding a path if a file of the same name already exists. --- llvm/lib/Support/VirtualFileSystem.cpp | 61 ++++++++++--------- .../Support/VirtualFileSystemTest.cpp | 5 ++ 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp index fcefdef992be..7360901f2962 100644 --- a/llvm/lib/Support/VirtualFileSystem.cpp +++ b/llvm/lib/Support/VirtualFileSystem.cpp @@ -867,21 +867,16 @@ bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime, // Any intermediate directories we create should be accessible by // the owner, even if Perms says otherwise for the final path. const auto NewDirectoryPerms = ResolvedPerms | sys::fs::owner_all; + + StringRef Name = *I; while (true) { - StringRef Name = *I; - detail::InMemoryNode *Node = Dir->getChild(Name); + Name = *I; ++I; + if (I == E) + break; + detail::InMemoryNode *Node = Dir->getChild(Name); if (!Node) { - if (I == E) { - // End of the path. - Dir->addChild( - Name, MakeNode({Dir->getUniqueID(), Path, Name, ModificationTime, - std::move(Buffer), ResolvedUser, ResolvedGroup, - ResolvedType, ResolvedPerms})); - return true; - } - - // Create a new directory. Use the path up to here. + // This isn't the last element, so we create a new directory. Status Stat( StringRef(Path.str().begin(), Name.end() - Path.str().begin()), getDirectoryID(Dir->getUniqueID(), Name), @@ -891,27 +886,33 @@ bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime, Name, std::make_unique(std::move(Stat)))); continue; } + // Creating file under another file. + if (!isa(Node)) + return false; + Dir = cast(Node); + } + detail::InMemoryNode *Node = Dir->getChild(Name); + if (!Node) { + Dir->addChild(Name, + MakeNode({Dir->getUniqueID(), Path, Name, ModificationTime, + std::move(Buffer), ResolvedUser, ResolvedGroup, + ResolvedType, ResolvedPerms})); + return true; + } + if (isa(Node)) + return ResolvedType == sys::fs::file_type::directory_file; - if (auto *NewDir = dyn_cast(Node)) { - Dir = NewDir; - } else { - assert((isa(Node) || - isa(Node)) && - "Must be either file, hardlink or directory!"); - - // Trying to insert a directory in place of a file. - if (I != E) - return false; + assert((isa(Node) || + isa(Node)) && + "Must be either file, hardlink or directory!"); - // Return false only if the new file is different from the existing one. - if (auto Link = dyn_cast(Node)) { - return Link->getResolvedFile().getBuffer()->getBuffer() == - Buffer->getBuffer(); - } - return cast(Node)->getBuffer()->getBuffer() == - Buffer->getBuffer(); - } + // Return false only if the new file is different from the existing one. + if (auto *Link = dyn_cast(Node)) { + return Link->getResolvedFile().getBuffer()->getBuffer() == + Buffer->getBuffer(); } + return cast(Node)->getBuffer()->getBuffer() == + Buffer->getBuffer(); } bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime, diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp index e9fd9671ea6a..9e9b4fbcdedd 100644 --- a/llvm/unittests/Support/VirtualFileSystemTest.cpp +++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp @@ -1138,6 +1138,11 @@ TEST_F(InMemoryFileSystemTest, DuplicatedFile) { ASSERT_FALSE(FS.addFile("/a/b", 0, MemoryBuffer::getMemBuffer("a"))); ASSERT_TRUE(FS.addFile("/a", 0, MemoryBuffer::getMemBuffer("a"))); ASSERT_FALSE(FS.addFile("/a", 0, MemoryBuffer::getMemBuffer("b"))); + ASSERT_TRUE(FS.addFile("/b/c/d", 0, MemoryBuffer::getMemBuffer("a"))); + ASSERT_FALSE(FS.addFile("/b/c", 0, MemoryBuffer::getMemBuffer("a"))); + ASSERT_TRUE(FS.addFile( + "/b/c", 0, MemoryBuffer::getMemBuffer(""), /*User=*/std::nullopt, + /*Group=*/std::nullopt, sys::fs::file_type::directory_file)); } TEST_F(InMemoryFileSystemTest, DirectoryIteration) { -- GitLab From e9174ba789531b26709764b4f404ec368b77db44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Thu, 6 Jun 2024 17:41:20 +0200 Subject: [PATCH 122/462] [clang][Interp] Always decay root array pointers to the first element This is similar to what the current interpreter does. --- clang/lib/AST/Interp/Interp.h | 2 +- clang/test/AST/Interp/arrays.cpp | 8 ++++++++ clang/test/AST/Interp/literals.cpp | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h index 1248eeb79cbf..98caea5c7014 100644 --- a/clang/lib/AST/Interp/Interp.h +++ b/clang/lib/AST/Interp/Interp.h @@ -2194,7 +2194,7 @@ inline bool ArrayDecay(InterpState &S, CodePtr OpPC) { if (!CheckRange(S, OpPC, Ptr, CSK_ArrayToPointer)) return false; - if (!Ptr.isUnknownSizeArray() || Ptr.isDummy()) { + if (Ptr.isRoot() || !Ptr.isUnknownSizeArray() || Ptr.isDummy()) { S.Stk.push(Ptr.atIndex(0)); return true; } diff --git a/clang/test/AST/Interp/arrays.cpp b/clang/test/AST/Interp/arrays.cpp index 6146d41c5ff5..6f6fca8c1cfd 100644 --- a/clang/test/AST/Interp/arrays.cpp +++ b/clang/test/AST/Interp/arrays.cpp @@ -615,3 +615,11 @@ namespace OnePastEndSub { constexpr A a[3][3]; constexpr int diff2 = &a[1][3] - &a[1][0]; /// Used to crash. } + +static int same_entity_2[3]; +constexpr int *get2() { + // This is a redeclaration of the same entity, even though it doesn't + // inherit the type of the prior declaration. + extern int same_entity_2[]; + return same_entity_2; +} diff --git a/clang/test/AST/Interp/literals.cpp b/clang/test/AST/Interp/literals.cpp index 27b2af7db0da..5a29013a053a 100644 --- a/clang/test/AST/Interp/literals.cpp +++ b/clang/test/AST/Interp/literals.cpp @@ -1209,7 +1209,7 @@ namespace incdecbool { constexpr int externvar1() { // both-error {{never produces a constant expression}} extern char arr[]; // ref-note {{declared here}} return arr[0]; // ref-note {{read of non-constexpr variable 'arr'}} \ - // expected-note {{array-to-pointer decay of array member without known bound is not supported}} + // expected-note {{indexing of array without known bound}} } #endif -- GitLab From f882f8c293d2064619f7eb3dc716dcaf3e2da875 Mon Sep 17 00:00:00 2001 From: Chen Zheng Date: Thu, 6 Jun 2024 23:49:58 +0800 Subject: [PATCH 123/462] [AIX] use LIBPATH on AIX instead of LD_LIBRARY_PATH (#94602) LD_LIBRARY_PATH will become invalid when LIBPATH is also set on AIX. See below example on AIX: ``` $ldd a.out a.out needs: /usr/lib/libc.a(shr.o) Cannot find libtest.a /unix /usr/lib/libcrypt.a(shr.o) $./a.out Could not load program ./a.out: Dependent module libtest.a could not be loaded. Could not load module libtest.a. System error: No such file or directory $export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp $./a.out ; echo $? 10 $export LIBPATH=./ $./a.out ; echo $? >>>>>> Now LD_LIBRARY_PATH is not used by system loader Could not load program ./a.out: Dependent module libtest.a could not be loaded. Could not load module libtest.a. System error: No such file or directory ``` This breaks many AIX LIT cases on our downstream buildbots which sets LIBPATH. --------- Co-authored-by: Anh Tuyen Tran <34661776+anhtuyenibm@users.noreply.github.com> Co-authored-by: David Tenty --- llvm/utils/lit/lit/llvm/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py index 1d4babc99984..afb7f078072f 100644 --- a/llvm/utils/lit/lit/llvm/config.py +++ b/llvm/utils/lit/lit/llvm/config.py @@ -588,7 +588,10 @@ class LLVMConfig(object): if getattr(self.config, pp, None) ] - self.with_environment("LD_LIBRARY_PATH", lib_paths, append_path=True) + if platform.system() == "AIX": + self.with_environment("LIBPATH", lib_paths, append_path=True) + else: + self.with_environment("LD_LIBRARY_PATH", lib_paths, append_path=True) shl = getattr(self.config, "llvm_shlib_dir", None) pext = getattr(self.config, "llvm_plugin_ext", None) -- GitLab From 7573d5e4b10cc7befc54d29edd7ec94d9bf11b93 Mon Sep 17 00:00:00 2001 From: Chaitanya Date: Thu, 6 Jun 2024 21:20:29 +0530 Subject: [PATCH 124/462] [AMDGPU] Update removeFnAttrFromReachable to accept array of Fn Attrs. (#94188) This PR updates removeFnAttrFromReachable in AMDGPUMemoryUtils to accept array of function attributes as argument. Helps to remove multiple attributes in one CallGraph walk. --- .../lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp | 2 +- llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp | 14 +++++++++----- llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h | 3 ++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp index 625ac0230f16..2bdbf4151dd9 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp @@ -1017,7 +1017,7 @@ public: // // TODO: We could filter out subgraphs that do not access LDS globals. for (Function *F : KernelsThatAllocateTableLDS) - removeFnAttrFromReachable(CG, F, "amdgpu-no-lds-kernel-id"); + removeFnAttrFromReachable(CG, F, {"amdgpu-no-lds-kernel-id"}); } DenseMap KernelToCreatedDynamicLDS = diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp index 239e0ee70572..04c6e940e6ed 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp @@ -235,8 +235,9 @@ LDSUsesInfoTy getTransitiveUsesOfLDS(const CallGraph &CG, Module &M) { } void removeFnAttrFromReachable(CallGraph &CG, Function *KernelRoot, - StringRef FnAttr) { - KernelRoot->removeFnAttr(FnAttr); + ArrayRef FnAttrs) { + for (StringRef Attr : FnAttrs) + KernelRoot->removeFnAttr(Attr); SmallVector WorkList = {CG[KernelRoot]->getFunction()}; SmallPtrSet Visited; @@ -261,12 +262,15 @@ void removeFnAttrFromReachable(CallGraph &CG, Function *KernelRoot, Function *PotentialCallee = ExternalCallRecord.second->getFunction(); assert(PotentialCallee); - if (!isKernelLDS(PotentialCallee)) - PotentialCallee->removeFnAttr(FnAttr); + if (!isKernelLDS(PotentialCallee)) { + for (StringRef Attr : FnAttrs) + PotentialCallee->removeFnAttr(Attr); + } } } } else { - Callee->removeFnAttr(FnAttr); + for (StringRef Attr : FnAttrs) + Callee->removeFnAttr(Attr); if (Visited.insert(Callee).second) WorkList.push_back(Callee); } diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h index 4d3ad328e131..e1cd4d03052b 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUMEMORYUTILS_H #define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUMEMORYUTILS_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" @@ -54,7 +55,7 @@ LDSUsesInfoTy getTransitiveUsesOfLDS(const CallGraph &CG, Module &M); /// Strip FnAttr attribute from any functions where we may have /// introduced its use. void removeFnAttrFromReachable(CallGraph &CG, Function *KernelRoot, - StringRef FnAttr); + ArrayRef FnAttrs); /// Given a \p Def clobbering a load from \p Ptr according to the MSSA check /// if this is actually a memory update or an artificial clobber to facilitate -- GitLab From a76290d6acedb4dcdd431cdd21f057255117f8d3 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Thu, 6 Jun 2024 08:54:20 -0700 Subject: [PATCH 125/462] [lldb] Refactor ReadRegisterValueAsScalar to return an llvm::Error (NFC) (#94556) --- lldb/source/Expression/DWARFExpression.cpp | 130 +++++++++------------ 1 file changed, 56 insertions(+), 74 deletions(-) diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp index 4681dbafb6f9..622b36b855e3 100644 --- a/lldb/source/Expression/DWARFExpression.cpp +++ b/lldb/source/Expression/DWARFExpression.cpp @@ -94,51 +94,38 @@ void DWARFExpression::SetRegisterKind(RegisterKind reg_kind) { m_reg_kind = reg_kind; } - -static bool ReadRegisterValueAsScalar(RegisterContext *reg_ctx, - lldb::RegisterKind reg_kind, - uint32_t reg_num, Status *error_ptr, - Value &value) { - if (reg_ctx == nullptr) { - if (error_ptr) - error_ptr->SetErrorString("No register context in frame.\n"); - } else { - uint32_t native_reg = - reg_ctx->ConvertRegisterKindToRegisterNumber(reg_kind, reg_num); - if (native_reg == LLDB_INVALID_REGNUM) { - if (error_ptr) - error_ptr->SetErrorStringWithFormat("Unable to convert register " - "kind=%u reg_num=%u to a native " - "register number.\n", - reg_kind, reg_num); - } else { - const RegisterInfo *reg_info = - reg_ctx->GetRegisterInfoAtIndex(native_reg); - RegisterValue reg_value; - if (reg_ctx->ReadRegister(reg_info, reg_value)) { - if (reg_value.GetScalarValue(value.GetScalar())) { - value.SetValueType(Value::ValueType::Scalar); - value.SetContext(Value::ContextType::RegisterInfo, - const_cast(reg_info)); - if (error_ptr) - error_ptr->Clear(); - return true; - } else { - // If we get this error, then we need to implement a value buffer in - // the dwarf expression evaluation function... - if (error_ptr) - error_ptr->SetErrorStringWithFormat( - "register %s can't be converted to a scalar value", - reg_info->name); - } - } else { - if (error_ptr) - error_ptr->SetErrorStringWithFormat("register %s is not available", - reg_info->name); - } +static llvm::Error ReadRegisterValueAsScalar(RegisterContext *reg_ctx, + lldb::RegisterKind reg_kind, + uint32_t reg_num, Value &value) { + if (reg_ctx == nullptr) + return llvm::createStringError("no register context in frame"); + + const uint32_t native_reg = + reg_ctx->ConvertRegisterKindToRegisterNumber(reg_kind, reg_num); + if (native_reg == LLDB_INVALID_REGNUM) + return llvm::createStringError( + "unable to convert register kind=%u reg_num=%u to a native " + "register number", + reg_kind, reg_num); + + const RegisterInfo *reg_info = reg_ctx->GetRegisterInfoAtIndex(native_reg); + RegisterValue reg_value; + if (reg_ctx->ReadRegister(reg_info, reg_value)) { + if (reg_value.GetScalarValue(value.GetScalar())) { + value.SetValueType(Value::ValueType::Scalar); + value.SetContext(Value::ContextType::RegisterInfo, + const_cast(reg_info)); + return llvm::Error::success(); } + + // If we get this error, then we need to implement a value buffer in + // the dwarf expression evaluation function... + return llvm::createStringError( + "register %s can't be converted to a scalar value", reg_info->name); } - return false; + + return llvm::createStringError("register %s is not available", + reg_info->name); } /// Return the length in bytes of the set of operands for \p op. No guarantees @@ -1832,11 +1819,10 @@ llvm::Expected DWARFExpression::Evaluate( dwarf4_location_description_kind = Register; reg_num = op - DW_OP_reg0; - Status read_err; - if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, &read_err, tmp)) - stack.push_back(tmp); - else - return read_err.ToError(); + if (llvm::Error err = + ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, tmp)) + return err; + stack.push_back(tmp); } break; // OPCODE: DW_OP_regx // OPERANDS: @@ -1846,10 +1832,10 @@ llvm::Expected DWARFExpression::Evaluate( dwarf4_location_description_kind = Register; reg_num = opcodes.GetULEB128(&offset); Status read_err; - if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, &read_err, tmp)) - stack.push_back(tmp); - else - return read_err.ToError(); + if (llvm::Error err = + ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, tmp)) + return err; + stack.push_back(tmp); } break; // OPCODE: DW_OP_bregN @@ -1890,17 +1876,15 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_breg30: case DW_OP_breg31: { reg_num = op - DW_OP_breg0; - - Status read_err; - if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, &read_err, - tmp)) { - int64_t breg_offset = opcodes.GetSLEB128(&offset); - tmp.ResolveValue(exe_ctx) += (uint64_t)breg_offset; - tmp.ClearContext(); - stack.push_back(tmp); - stack.back().SetValueType(Value::ValueType::LoadAddress); - } else - return read_err.ToError(); + if (llvm::Error err = + ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, tmp)) + return err; + + int64_t breg_offset = opcodes.GetSLEB128(&offset); + tmp.ResolveValue(exe_ctx) += (uint64_t)breg_offset; + tmp.ClearContext(); + stack.push_back(tmp); + stack.back().SetValueType(Value::ValueType::LoadAddress); } break; // OPCODE: DW_OP_bregx // OPERANDS: 2 @@ -1910,17 +1894,15 @@ llvm::Expected DWARFExpression::Evaluate( // N plus an offset. case DW_OP_bregx: { reg_num = opcodes.GetULEB128(&offset); - - Status read_err; - if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, &read_err, - tmp)) { - int64_t breg_offset = opcodes.GetSLEB128(&offset); - tmp.ResolveValue(exe_ctx) += (uint64_t)breg_offset; - tmp.ClearContext(); - stack.push_back(tmp); - stack.back().SetValueType(Value::ValueType::LoadAddress); - } else - return read_err.ToError(); + if (llvm::Error err = + ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, tmp)) + return err; + + int64_t breg_offset = opcodes.GetSLEB128(&offset); + tmp.ResolveValue(exe_ctx) += (uint64_t)breg_offset; + tmp.ClearContext(); + stack.push_back(tmp); + stack.back().SetValueType(Value::ValueType::LoadAddress); } break; case DW_OP_fbreg: -- GitLab From 67aaa9f9974993f360cc0dabffd73b51c030d775 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Thu, 6 Jun 2024 08:54:37 -0700 Subject: [PATCH 126/462] [lldb] Refactor ResolveLoadAddress to return an llvm::Expected (NFC) (#94558) --- lldb/source/Expression/DWARFExpression.cpp | 48 ++++++++-------------- 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp index 622b36b855e3..05767a8d02ff 100644 --- a/lldb/source/Expression/DWARFExpression.cpp +++ b/lldb/source/Expression/DWARFExpression.cpp @@ -769,7 +769,6 @@ void UpdateValueTypeFromLocationDescription(Log *log, const DWARFUnit *dwarf_cu, /// /// \param exe_ctx Pointer to the execution context /// \param module_sp shared_ptr contains the module if we have one -/// \param error_ptr pointer to Status object if we have one /// \param dw_op_type C-style string used to vary the error output /// \param file_addr the file address we are trying to resolve and turn into a /// load address @@ -780,32 +779,22 @@ void UpdateValueTypeFromLocationDescription(Log *log, const DWARFUnit *dwarf_cu, /// the load address succeed or an empty Optinal otherwise. If /// check_sectionoffset is true we consider LLDB_INVALID_ADDRESS a /// success if so_addr.IsSectionOffset() is true. -static std::optional +static llvm::Expected ResolveLoadAddress(ExecutionContext *exe_ctx, lldb::ModuleSP &module_sp, - Status *error_ptr, const char *dw_op_type, - lldb::addr_t file_addr, Address &so_addr, - bool check_sectionoffset = false) { - if (!module_sp) { - if (error_ptr) - error_ptr->SetErrorStringWithFormat( - "need module to resolve file address for %s", dw_op_type); - return {}; - } + const char *dw_op_type, lldb::addr_t file_addr, + Address &so_addr, bool check_sectionoffset = false) { + if (!module_sp) + return llvm::createStringError("need module to resolve file address for %s", + dw_op_type); - if (!module_sp->ResolveFileAddress(file_addr, so_addr)) { - if (error_ptr) - error_ptr->SetErrorString("failed to resolve file address in module"); - return {}; - } + if (!module_sp->ResolveFileAddress(file_addr, so_addr)) + return llvm::createStringError("failed to resolve file address in module"); - addr_t load_addr = so_addr.GetLoadAddress(exe_ctx->GetTargetPtr()); + const addr_t load_addr = so_addr.GetLoadAddress(exe_ctx->GetTargetPtr()); if (load_addr == LLDB_INVALID_ADDRESS && - (check_sectionoffset && !so_addr.IsSectionOffset())) { - if (error_ptr) - error_ptr->SetErrorString("failed to resolve load address"); - return {}; - } + (check_sectionoffset && !so_addr.IsSectionOffset())) + return llvm::createStringError("failed to resolve load address"); return load_addr; } @@ -975,12 +964,11 @@ llvm::Expected DWARFExpression::Evaluate( LLDB_INVALID_ADDRESS); Address so_addr; - Status load_err; auto maybe_load_addr = ResolveLoadAddress( - exe_ctx, module_sp, &load_err, "DW_OP_deref", file_addr, so_addr); + exe_ctx, module_sp, "DW_OP_deref", file_addr, so_addr); if (!maybe_load_addr) - return load_err.ToError(); + return maybe_load_addr.takeError(); stack.back().GetScalar() = *maybe_load_addr; // Fall through to load address promotion code below. @@ -1092,14 +1080,12 @@ llvm::Expected DWARFExpression::Evaluate( auto file_addr = stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS); Address so_addr; - Status resolve_err; - auto maybe_load_addr = - ResolveLoadAddress(exe_ctx, module_sp, &resolve_err, - "DW_OP_deref_size", file_addr, so_addr, - /*check_sectionoffset=*/true); + auto maybe_load_addr = ResolveLoadAddress( + exe_ctx, module_sp, "DW_OP_deref_size", file_addr, so_addr, + /*check_sectionoffset=*/true); if (!maybe_load_addr) - return resolve_err.ToError(); + return maybe_load_addr.takeError(); addr_t load_addr = *maybe_load_addr; -- GitLab From 7fdbc30b445286f03203e16d0be067c25c6f0df0 Mon Sep 17 00:00:00 2001 From: Pavel Labath Date: Thu, 6 Jun 2024 15:06:46 +0000 Subject: [PATCH 127/462] Revert "[lldb][DebugNames] Only skip processing of DW_AT_declarations for class/union types" and two follow-up commits. The reason is the crash we've discovered when processing -gsimple-template-names binaries. I'm committing a minimal reproducer as a separate patch. This reverts the following commits: - 51dd4eaaa29683c16151f5168e7f8645acbd6e6c (#92328) - 3d9d48523977af3590f7dd0edfd258454cb9e9cf (#93839) - afe6ab7586f7078cc410f6162bd9851e48e2a286 (#94400) --- .../Plugins/SymbolFile/DWARF/DWARFASTParser.h | 2 - .../SymbolFile/DWARF/DWARFASTParserClang.cpp | 401 ++++++++---------- .../SymbolFile/DWARF/DWARFASTParserClang.h | 197 +++++---- .../SymbolFile/DWARF/DebugNamesDWARFIndex.cpp | 5 - .../SymbolFile/DWARF/SymbolFileDWARF.cpp | 51 +-- .../SymbolFile/DWARF/SymbolFileDWARF.h | 15 +- .../DWARF/SymbolFileDWARFDebugMap.h | 9 - .../SymbolFile/DWARF/SymbolFileDWARFDwo.cpp | 2 +- .../SymbolFile/DWARF/SymbolFileDWARFDwo.h | 3 +- .../SymbolFile/DWARF/UniqueDWARFASTType.cpp | 117 +++-- .../SymbolFile/DWARF/UniqueDWARFASTType.h | 36 +- .../delayed-definition-die-searching.test | 36 -- 12 files changed, 393 insertions(+), 481 deletions(-) delete mode 100644 lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h index e144cf0f9bd9..66db396279e0 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h @@ -60,8 +60,6 @@ public: virtual ConstString GetDIEClassTemplateParams(const DWARFDIE &die) = 0; - virtual lldb_private::Type *FindDefinitionTypeForDIE(const DWARFDIE &die) = 0; - static std::optional ParseChildArrayInfo(const DWARFDIE &parent_die, const ExecutionContext *exe_ctx = nullptr); diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp index 7d7e835c3d73..579a538af363 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp @@ -154,26 +154,6 @@ static bool TagIsRecordType(dw_tag_t tag) { } } -static bool IsForwardDeclaration(const DWARFDIE &die, - const ParsedDWARFTypeAttributes &attrs, - LanguageType cu_language) { - if (attrs.is_forward_declaration) - return true; - - // Work around an issue with clang at the moment where forward - // declarations for objective C classes are emitted as: - // DW_TAG_structure_type [2] - // DW_AT_name( "ForwardObjcClass" ) - // DW_AT_byte_size( 0x00 ) - // DW_AT_decl_file( "..." ) - // DW_AT_decl_line( 1 ) - // - // Note that there is no DW_AT_declaration and there are no children, - // and the byte size is zero. - return attrs.byte_size && *attrs.byte_size == 0 && attrs.name && - !die.HasChildren() && cu_language == eLanguageTypeObjC; -} - TypeSP DWARFASTParserClang::ParseTypeFromClangModule(const SymbolContext &sc, const DWARFDIE &die, Log *log) { @@ -269,9 +249,11 @@ static void ForcefullyCompleteType(CompilerType type) { /// This function serves a similar purpose as RequireCompleteType above, but it /// avoids completing the type if it is not immediately necessary. It only /// ensures we _can_ complete the type later. -void DWARFASTParserClang::PrepareContextToReceiveMembers( - clang::DeclContext *decl_ctx, const DWARFDIE &decl_ctx_die, - const DWARFDIE &die, const char *type_name_cstr) { +static void PrepareContextToReceiveMembers(TypeSystemClang &ast, + ClangASTImporter &ast_importer, + clang::DeclContext *decl_ctx, + DWARFDIE die, + const char *type_name_cstr) { auto *tag_decl_ctx = clang::dyn_cast(decl_ctx); if (!tag_decl_ctx) return; // Non-tag context are always ready. @@ -286,8 +268,7 @@ void DWARFASTParserClang::PrepareContextToReceiveMembers( // gmodules case), we can complete the type by doing a full import. // If this type was not imported from an external AST, there's nothing to do. - CompilerType type = m_ast.GetTypeForDecl(tag_decl_ctx); - ClangASTImporter &ast_importer = GetClangASTImporter(); + CompilerType type = ast.GetTypeForDecl(tag_decl_ctx); if (type && ast_importer.CanImport(type)) { auto qual_type = ClangUtil::GetQualType(type); if (ast_importer.RequireCompleteType(qual_type)) @@ -298,13 +279,6 @@ void DWARFASTParserClang::PrepareContextToReceiveMembers( type_name_cstr ? type_name_cstr : "", die.GetOffset()); } - // By searching for the definition DIE of the decl_ctx type, we will either: - // 1. Found the the definition DIE and start its definition with - // TypeSystemClang::StartTagDeclarationDefinition. - // 2. Unable to find it, then need to forcefully complete it. - FindDefinitionTypeForDIE(decl_ctx_die); - if (tag_decl_ctx->isCompleteDefinition() || tag_decl_ctx->isBeingDefined()) - return; // We don't have a type definition and/or the import failed. We must // forcefully complete the type to avoid crashes. ForcefullyCompleteType(type); @@ -650,11 +624,10 @@ DWARFASTParserClang::ParseTypeModifier(const SymbolContext &sc, if (tag == DW_TAG_typedef) { // DeclContext will be populated when the clang type is materialized in // Type::ResolveCompilerType. - DWARFDIE decl_ctx_die; - clang::DeclContext *decl_ctx = - GetClangDeclContextContainingDIE(die, &decl_ctx_die); - PrepareContextToReceiveMembers(decl_ctx, decl_ctx_die, die, - attrs.name.GetCString()); + PrepareContextToReceiveMembers( + m_ast, GetClangASTImporter(), + GetClangDeclContextContainingDIE(die, nullptr), die, + attrs.name.GetCString()); if (attrs.type.IsValid()) { // Try to parse a typedef from the (DWARF embedded in the) Clang @@ -1134,6 +1107,32 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die, // struct and see if this is actually a C++ method Type *class_type = dwarf->ResolveType(decl_ctx_die); if (class_type) { + if (class_type->GetID() != decl_ctx_die.GetID() || + IsClangModuleFwdDecl(decl_ctx_die)) { + + // We uniqued the parent class of this function to another + // class so we now need to associate all dies under + // "decl_ctx_die" to DIEs in the DIE for "class_type"... + DWARFDIE class_type_die = dwarf->GetDIE(class_type->GetID()); + + if (class_type_die) { + std::vector failures; + + CopyUniqueClassMethodTypes(decl_ctx_die, class_type_die, + class_type, failures); + + // FIXME do something with these failures that's + // smarter than just dropping them on the ground. + // Unfortunately classes don't like having stuff added + // to them after their definitions are complete... + + Type *type_ptr = dwarf->GetDIEToType()[die.GetDIE()]; + if (type_ptr && type_ptr != DIE_IS_BEING_PARSED) { + return type_ptr->shared_from_this(); + } + } + } + if (attrs.specification.IsValid()) { // We have a specification which we are going to base our // function prototype off of, so we need this type to be @@ -1268,39 +1267,6 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die, } } } - // By here, we should have already completed the c++ class_type - // because if either specification or abstract_origin is present, we - // call GetClangDeclContextForDIE to resolve the DW_TAG_subprogram - // refered by this one until we reached the DW_TAG_subprogram without - // specification or abstract_origin (the else branch above). Then the - // above GetFullCompilerType() will complete the class_type if it's - // not completed yet. After that, we will have the mapping from DIEs - // in class_type_die to DeclContexts in m_die_to_decl_ctx. - if (class_type->GetID() != decl_ctx_die.GetID() || - IsClangModuleFwdDecl(decl_ctx_die)) { - - // We uniqued the parent class of this function to another - // class so we now need to associate all dies under - // "decl_ctx_die" to DIEs in the DIE for "class_type"... - DWARFDIE class_type_die = dwarf->GetDIE(class_type->GetID()); - - if (class_type_die) { - std::vector failures; - - CopyUniqueClassMethodTypes(decl_ctx_die, class_type_die, - class_type, failures); - - // FIXME do something with these failures that's - // smarter than just dropping them on the ground. - // Unfortunately classes don't like having stuff added - // to them after their definitions are complete... - - Type *type_ptr = dwarf->GetDIEToType()[die.GetDIE()]; - if (type_ptr && type_ptr != DIE_IS_BEING_PARSED) { - return type_ptr->shared_from_this(); - } - } - } } } } @@ -1673,93 +1639,6 @@ DWARFASTParserClang::GetCPlusPlusQualifiedName(const DWARFDIE &die) { return qualified_name; } -lldb_private::Type * -DWARFASTParserClang::FindDefinitionTypeForDIE(const DWARFDIE &die) { - SymbolFileDWARF *dwarf = die.GetDWARF(); - ParsedDWARFTypeAttributes attrs(die); - bool is_forward_declaration = IsForwardDeclaration( - die, attrs, SymbolFileDWARF::GetLanguage(*die.GetCU())); - if (!is_forward_declaration) - return dwarf->GetDIEToType()[die.GetDIE()]; - - const dw_tag_t tag = die.Tag(); - TypeSP type_sp; - Log *log = GetLog(DWARFLog::TypeCompletion | DWARFLog::Lookups); - if (log) { - dwarf->GetObjectFile()->GetModule()->LogMessage( - log, - "SymbolFileDWARF({0:p}) - {1:x16}: {2} type \"{3}\" is a " - "forward declaration DIE, trying to find definition DIE", - static_cast(this), die.GetOffset(), DW_TAG_value_to_name(tag), - attrs.name.GetCString()); - } - // We haven't parse definition die for this type, starting to search for it. - // After we found the definition die, the GetDeclarationDIEToDefinitionDIE() - // map will have the new mapping from this declaration die to definition die. - if (attrs.class_language == eLanguageTypeObjC || - attrs.class_language == eLanguageTypeObjC_plus_plus) { - if (!attrs.is_complete_objc_class && - die.Supports_DW_AT_APPLE_objc_complete_type()) { - // We have a valid eSymbolTypeObjCClass class symbol whose name - // matches the current objective C class that we are trying to find - // and this DIE isn't the complete definition (we checked - // is_complete_objc_class above and know it is false), so the real - // definition is in here somewhere - type_sp = - dwarf->FindCompleteObjCDefinitionTypeForDIE(die, attrs.name, true); - - if (!type_sp) { - SymbolFileDWARFDebugMap *debug_map_symfile = - dwarf->GetDebugMapSymfile(); - if (debug_map_symfile) { - // We weren't able to find a full declaration in this DWARF, - // see if we have a declaration anywhere else... - type_sp = debug_map_symfile->FindCompleteObjCDefinitionTypeForDIE( - die, attrs.name, true); - } - } - - if (type_sp && log) { - dwarf->GetObjectFile()->GetModule()->LogMessage( - log, - "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is an " - "incomplete objc type, complete type is {5:x8}", - static_cast(this), die.GetOffset(), - DW_TAG_value_to_name(tag), tag, attrs.name.GetCString(), - type_sp->GetID()); - } - } - } - - type_sp = dwarf->FindDefinitionTypeForDWARFDeclContext(die); - if (!type_sp) { - SymbolFileDWARFDebugMap *debug_map_symfile = dwarf->GetDebugMapSymfile(); - if (debug_map_symfile) { - // We weren't able to find a full declaration in this DWARF, see - // if we have a declaration anywhere else... - type_sp = debug_map_symfile->FindDefinitionTypeForDWARFDeclContext(die); - } - if (type_sp && log) { - dwarf->GetObjectFile()->GetModule()->LogMessage( - log, - "SymbolFileDWARF({0:p}) - {1:x16}: {2} type \"{3}\" is a " - "forward declaration, complete type is {4:x8}", - static_cast(this), die.GetOffset(), DW_TAG_value_to_name(tag), - attrs.name.GetCString(), type_sp->GetID()); - } - } - - if (!type_sp && log) { - dwarf->GetObjectFile()->GetModule()->LogMessage( - log, - "SymbolFileDWARF({0:p}) - {1:x16}: {2} type \"{3}\" is a " - "forward declaration, unable to find definition DIE for it", - static_cast(this), die.GetOffset(), DW_TAG_value_to_name(tag), - attrs.name.GetCString()); - } - return type_sp.get(); -} - TypeSP DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc, const DWARFDIE &die, @@ -1771,10 +1650,14 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc, LanguageType cu_language = SymbolFileDWARF::GetLanguage(*die.GetCU()); Log *log = GetLog(DWARFLog::TypeCompletion | DWARFLog::Lookups); + // UniqueDWARFASTType is large, so don't create a local variables on the + // stack, put it on the heap. This function is often called recursively and + // clang isn't good at sharing the stack space for variables in different + // blocks. + auto unique_ast_entry_up = std::make_unique(); + ConstString unique_typename(attrs.name); Declaration unique_decl(attrs.decl); - uint64_t byte_size = attrs.byte_size.value_or(0); - attrs.is_forward_declaration = IsForwardDeclaration(die, attrs, cu_language); if (attrs.name) { if (Language::LanguageIsCPlusPlus(cu_language)) { @@ -1787,42 +1670,14 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc, unique_decl.Clear(); } - if (UniqueDWARFASTType *unique_ast_entry_type = - dwarf->GetUniqueDWARFASTTypeMap().Find( - unique_typename, die, unique_decl, byte_size, - attrs.is_forward_declaration)) { - type_sp = unique_ast_entry_type->m_type_sp; + if (dwarf->GetUniqueDWARFASTTypeMap().Find( + unique_typename, die, unique_decl, attrs.byte_size.value_or(-1), + *unique_ast_entry_up)) { + type_sp = unique_ast_entry_up->m_type_sp; if (type_sp) { dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get(); LinkDeclContextToDIE( - GetCachedClangDeclContextForDIE(unique_ast_entry_type->m_die), die); - if (!attrs.is_forward_declaration) { - // If the DIE being parsed in this function is a definition and the - // entry in the map is a declaration, then we need to update the entry - // to point to the definition DIE. - if (unique_ast_entry_type->m_is_forward_declaration) { - unique_ast_entry_type->m_die = die; - unique_ast_entry_type->m_byte_size = byte_size; - unique_ast_entry_type->m_declaration = unique_decl; - unique_ast_entry_type->m_is_forward_declaration = false; - // Need to update Type ID to refer to the definition DIE. because - // it's used in ParseSubroutine to determine if we need to copy cxx - // method types from a declaration DIE to this definition DIE. - type_sp->SetID(die.GetID()); - clang_type = type_sp->GetForwardCompilerType(); - if (attrs.class_language != eLanguageTypeObjC && - attrs.class_language != eLanguageTypeObjC_plus_plus) - TypeSystemClang::StartTagDeclarationDefinition(clang_type); - - CompilerType compiler_type_no_qualifiers = - ClangUtil::RemoveFastQualifiers(clang_type); - auto result = dwarf->GetForwardDeclCompilerTypeToDIE().try_emplace( - compiler_type_no_qualifiers.GetOpaqueQualType(), - *die.GetDIERef()); - if (!result.second) - result.first->second = *die.GetDIERef(); - } - } + GetCachedClangDeclContextForDIE(unique_ast_entry_up->m_die), die); return type_sp; } } @@ -1844,21 +1699,125 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc, default_accessibility = eAccessPrivate; } + if (attrs.byte_size && *attrs.byte_size == 0 && attrs.name && + !die.HasChildren() && cu_language == eLanguageTypeObjC) { + // Work around an issue with clang at the moment where forward + // declarations for objective C classes are emitted as: + // DW_TAG_structure_type [2] + // DW_AT_name( "ForwardObjcClass" ) + // DW_AT_byte_size( 0x00 ) + // DW_AT_decl_file( "..." ) + // DW_AT_decl_line( 1 ) + // + // Note that there is no DW_AT_declaration and there are no children, + // and the byte size is zero. + attrs.is_forward_declaration = true; + } + + if (attrs.class_language == eLanguageTypeObjC || + attrs.class_language == eLanguageTypeObjC_plus_plus) { + if (!attrs.is_complete_objc_class && + die.Supports_DW_AT_APPLE_objc_complete_type()) { + // We have a valid eSymbolTypeObjCClass class symbol whose name + // matches the current objective C class that we are trying to find + // and this DIE isn't the complete definition (we checked + // is_complete_objc_class above and know it is false), so the real + // definition is in here somewhere + type_sp = + dwarf->FindCompleteObjCDefinitionTypeForDIE(die, attrs.name, true); + + if (!type_sp) { + SymbolFileDWARFDebugMap *debug_map_symfile = + dwarf->GetDebugMapSymfile(); + if (debug_map_symfile) { + // We weren't able to find a full declaration in this DWARF, + // see if we have a declaration anywhere else... + type_sp = debug_map_symfile->FindCompleteObjCDefinitionTypeForDIE( + die, attrs.name, true); + } + } + + if (type_sp) { + if (log) { + dwarf->GetObjectFile()->GetModule()->LogMessage( + log, + "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is an " + "incomplete objc type, complete type is {5:x8}", + static_cast(this), die.GetOffset(), + DW_TAG_value_to_name(tag), tag, attrs.name.GetCString(), + type_sp->GetID()); + } + + // We found a real definition for this type elsewhere so lets use + // it and cache the fact that we found a complete type for this + // die + dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get(); + return type_sp; + } + } + } + if (attrs.is_forward_declaration) { + // We have a forward declaration to a type and we need to try and + // find a full declaration. We look in the current type index just in + // case we have a forward declaration followed by an actual + // declarations in the DWARF. If this fails, we need to look + // elsewhere... + if (log) { + dwarf->GetObjectFile()->GetModule()->LogMessage( + log, + "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is a " + "forward declaration, trying to find complete type", + static_cast(this), die.GetOffset(), DW_TAG_value_to_name(tag), + tag, attrs.name.GetCString()); + } + // See if the type comes from a Clang module and if so, track down // that type. type_sp = ParseTypeFromClangModule(sc, die, log); if (type_sp) return type_sp; - } + // type_sp = FindDefinitionTypeForDIE (dwarf_cu, die, + // type_name_const_str); + type_sp = dwarf->FindDefinitionTypeForDWARFDeclContext(die); + + if (!type_sp) { + SymbolFileDWARFDebugMap *debug_map_symfile = dwarf->GetDebugMapSymfile(); + if (debug_map_symfile) { + // We weren't able to find a full declaration in this DWARF, see + // if we have a declaration anywhere else... + type_sp = debug_map_symfile->FindDefinitionTypeForDWARFDeclContext(die); + } + } + + if (type_sp) { + if (log) { + dwarf->GetObjectFile()->GetModule()->LogMessage( + log, + "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is a " + "forward declaration, complete type is {5:x8}", + static_cast(this), die.GetOffset(), + DW_TAG_value_to_name(tag), tag, attrs.name.GetCString(), + type_sp->GetID()); + } + + // We found a real definition for this type elsewhere so lets use + // it and cache the fact that we found a complete type for this die + dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get(); + clang::DeclContext *defn_decl_ctx = + GetCachedClangDeclContextForDIE(dwarf->GetDIE(type_sp->GetID())); + if (defn_decl_ctx) + LinkDeclContextToDIE(defn_decl_ctx, die); + return type_sp; + } + } assert(tag_decl_kind != -1); UNUSED_IF_ASSERT_DISABLED(tag_decl_kind); - DWARFDIE decl_ctx_die; - clang::DeclContext *decl_ctx = - GetClangDeclContextContainingDIE(die, &decl_ctx_die); + bool clang_type_was_created = false; + clang::DeclContext *decl_ctx = GetClangDeclContextContainingDIE(die, nullptr); - PrepareContextToReceiveMembers(decl_ctx, decl_ctx_die, die, + PrepareContextToReceiveMembers(m_ast, GetClangASTImporter(), decl_ctx, die, attrs.name.GetCString()); if (attrs.accessibility == eAccessNone && decl_ctx) { @@ -1897,17 +1856,20 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc, tag_decl_kind, template_param_infos); clang_type = m_ast.CreateClassTemplateSpecializationType(class_specialization_decl); + clang_type_was_created = true; m_ast.SetMetadata(class_template_decl, metadata); m_ast.SetMetadata(class_specialization_decl, metadata); } - if (!clang_type) { + if (!clang_type_was_created) { + clang_type_was_created = true; clang_type = m_ast.CreateRecordType( decl_ctx, GetOwningClangModule(die), attrs.accessibility, attrs.name.GetCString(), tag_decl_kind, attrs.class_language, &metadata, attrs.exports_symbols); } + // Store a forward declaration to this class type in case any // parameters in any class methods need it for the clang types for // function prototypes. @@ -1918,19 +1880,13 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc, Type::ResolveState::Forward, TypePayloadClang(OptionalClangModuleID(), attrs.is_complete_objc_class)); - // UniqueDWARFASTType is large, so don't create a local variables on the - // stack, put it on the heap. This function is often called recursively and - // clang isn't good at sharing the stack space for variables in different - // blocks. - auto unique_ast_entry_up = std::make_unique(); // Add our type to the unique type map so we don't end up creating many // copies of the same type over and over in the ASTContext for our // module unique_ast_entry_up->m_type_sp = type_sp; unique_ast_entry_up->m_die = die; unique_ast_entry_up->m_declaration = unique_decl; - unique_ast_entry_up->m_byte_size = byte_size; - unique_ast_entry_up->m_is_forward_declaration = attrs.is_forward_declaration; + unique_ast_entry_up->m_byte_size = attrs.byte_size.value_or(0); dwarf->GetUniqueDWARFASTTypeMap().Insert(unique_typename, *unique_ast_entry_up); @@ -1971,7 +1927,7 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc, GetClangASTImporter().SetRecordLayout(record_decl, layout); } } - } else { + } else if (clang_type_was_created) { // Start the definition if the class is not objective C since the // underlying decls respond to isCompleteDefinition(). Objective // C decls don't respond to isCompleteDefinition() so we can't @@ -1983,21 +1939,26 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc, if (attrs.class_language != eLanguageTypeObjC && attrs.class_language != eLanguageTypeObjC_plus_plus) TypeSystemClang::StartTagDeclarationDefinition(clang_type); + + // Leave this as a forward declaration until we need to know the + // details of the type. lldb_private::Type will automatically call + // the SymbolFile virtual function + // "SymbolFileDWARF::CompleteType(Type *)" When the definition + // needs to be defined. + assert(!dwarf->GetForwardDeclCompilerTypeToDIE().count( + ClangUtil::RemoveFastQualifiers(clang_type) + .GetOpaqueQualType()) && + "Type already in the forward declaration map!"); + // Can't assume m_ast.GetSymbolFile() is actually a + // SymbolFileDWARF, it can be a SymbolFileDWARFDebugMap for Apple + // binaries. + dwarf->GetForwardDeclCompilerTypeToDIE().try_emplace( + ClangUtil::RemoveFastQualifiers(clang_type).GetOpaqueQualType(), + *die.GetDIERef()); + m_ast.SetHasExternalStorage(clang_type.GetOpaqueQualType(), true); } } - // If this is a declaration DIE, leave this as a forward declaration until we - // need to know the details of the type. lldb_private::Type will automatically - // call the SymbolFile virtual function "SymbolFileDWARF::CompleteType(Type - // *)" When the definition needs to be defined. - assert(!dwarf->GetForwardDeclCompilerTypeToDIE().count( - ClangUtil::RemoveFastQualifiers(clang_type).GetOpaqueQualType()) && - "Type already in the forward declaration map!"); - dwarf->GetForwardDeclCompilerTypeToDIE().try_emplace( - ClangUtil::RemoveFastQualifiers(clang_type).GetOpaqueQualType(), - *die.GetDIERef()); - m_ast.SetHasExternalStorage(clang_type.GetOpaqueQualType(), true); - // If we made a clang type, set the trivial abi if applicable: We only // do this for pass by value - which implies the Trivial ABI. There // isn't a way to assert that something that would normally be pass by @@ -2236,10 +2197,6 @@ bool DWARFASTParserClang::CompleteRecordType(const DWARFDIE &die, // For objective C we don't start the definition when the class is // created. TypeSystemClang::StartTagDeclarationDefinition(clang_type); - } else { - assert(clang_type.IsBeingDefined() && - "Trying to complete a definition without a prior call to " - "StartTagDeclarationDefinition."); } AccessType default_accessibility = eAccessNone; diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h index 853b8ccc3036..8d4af203bb28 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h @@ -42,40 +42,40 @@ struct ParsedDWARFTypeAttributes; class DWARFASTParserClang : public lldb_private::plugin::dwarf::DWARFASTParser { public: - typedef lldb_private::plugin::dwarf::DWARFDIE DWARFDIE; - DWARFASTParserClang(lldb_private::TypeSystemClang &ast); ~DWARFASTParserClang() override; // DWARFASTParser interface. - lldb::TypeSP ParseTypeFromDWARF(const lldb_private::SymbolContext &sc, - const DWARFDIE &die, - bool *type_is_new_ptr) override; + lldb::TypeSP + ParseTypeFromDWARF(const lldb_private::SymbolContext &sc, + const lldb_private::plugin::dwarf::DWARFDIE &die, + bool *type_is_new_ptr) override; - lldb_private::ConstString - ConstructDemangledNameFromDWARF(const DWARFDIE &die) override; + lldb_private::ConstString ConstructDemangledNameFromDWARF( + const lldb_private::plugin::dwarf::DWARFDIE &die) override; lldb_private::Function * ParseFunctionFromDWARF(lldb_private::CompileUnit &comp_unit, - const DWARFDIE &die, + const lldb_private::plugin::dwarf::DWARFDIE &die, const lldb_private::AddressRange &func_range) override; bool - CompleteTypeFromDWARF(const DWARFDIE &die, lldb_private::Type *type, + CompleteTypeFromDWARF(const lldb_private::plugin::dwarf::DWARFDIE &die, + lldb_private::Type *type, lldb_private::CompilerType &compiler_type) override; - lldb_private::CompilerDecl - GetDeclForUIDFromDWARF(const DWARFDIE &die) override; + lldb_private::CompilerDecl GetDeclForUIDFromDWARF( + const lldb_private::plugin::dwarf::DWARFDIE &die) override; void EnsureAllDIEsInDeclContextHaveBeenParsed( lldb_private::CompilerDeclContext decl_context) override; - lldb_private::CompilerDeclContext - GetDeclContextForUIDFromDWARF(const DWARFDIE &die) override; + lldb_private::CompilerDeclContext GetDeclContextForUIDFromDWARF( + const lldb_private::plugin::dwarf::DWARFDIE &die) override; - lldb_private::CompilerDeclContext - GetDeclContextContainingUIDFromDWARF(const DWARFDIE &die) override; + lldb_private::CompilerDeclContext GetDeclContextContainingUIDFromDWARF( + const lldb_private::plugin::dwarf::DWARFDIE &die) override; lldb_private::ClangASTImporter &GetClangASTImporter(); @@ -105,13 +105,8 @@ public: /// \return A string, including surrounding '<>', of the template parameters. /// If the DIE's name already has '<>', returns an empty ConstString because /// it's assumed that the caller is using the DIE name anyway. - lldb_private::ConstString - GetDIEClassTemplateParams(const DWARFDIE &die) override; - - // Searching for definition DIE for the given DIE and return the type - // associated with the definition DIE, or nullptr if definition DIE is not - // found. - lldb_private::Type *FindDefinitionTypeForDIE(const DWARFDIE &die) override; + lldb_private::ConstString GetDIEClassTemplateParams( + const lldb_private::plugin::dwarf::DWARFDIE &die) override; protected: /// Protected typedefs and members. @@ -123,7 +118,8 @@ protected: const lldb_private::plugin::dwarf::DWARFDebugInfoEntry *, clang::DeclContext *> DIEToDeclContextMap; - typedef std::multimap + typedef std::multimap DeclContextToDIEMap; typedef llvm::DenseMap< const lldb_private::plugin::dwarf::DWARFDebugInfoEntry *, @@ -141,11 +137,14 @@ protected: std::unique_ptr m_clang_ast_importer_up; /// @} - clang::DeclContext *GetDeclContextForBlock(const DWARFDIE &die); + clang::DeclContext * + GetDeclContextForBlock(const lldb_private::plugin::dwarf::DWARFDIE &die); - clang::BlockDecl *ResolveBlockDIE(const DWARFDIE &die); + clang::BlockDecl * + ResolveBlockDIE(const lldb_private::plugin::dwarf::DWARFDIE &die); - clang::NamespaceDecl *ResolveNamespaceDIE(const DWARFDIE &die); + clang::NamespaceDecl * + ResolveNamespaceDIE(const lldb_private::plugin::dwarf::DWARFDIE &die); /// Returns the namespace decl that a DW_TAG_imported_declaration imports. /// @@ -156,86 +155,96 @@ protected: /// 'die' imports. If the imported entity is not a namespace /// or another import declaration, returns nullptr. If an error /// occurs, returns nullptr. - clang::NamespaceDecl *ResolveImportedDeclarationDIE(const DWARFDIE &die); + clang::NamespaceDecl *ResolveImportedDeclarationDIE( + const lldb_private::plugin::dwarf::DWARFDIE &die); - bool ParseTemplateDIE(const DWARFDIE &die, + bool ParseTemplateDIE(const lldb_private::plugin::dwarf::DWARFDIE &die, lldb_private::TypeSystemClang::TemplateParameterInfos &template_param_infos); bool ParseTemplateParameterInfos( - const DWARFDIE &parent_die, + const lldb_private::plugin::dwarf::DWARFDIE &parent_die, lldb_private::TypeSystemClang::TemplateParameterInfos &template_param_infos); - std::string GetCPlusPlusQualifiedName(const DWARFDIE &die); + std::string + GetCPlusPlusQualifiedName(const lldb_private::plugin::dwarf::DWARFDIE &die); bool ParseChildMembers( - const DWARFDIE &die, lldb_private::CompilerType &class_compiler_type, + const lldb_private::plugin::dwarf::DWARFDIE &die, + lldb_private::CompilerType &class_compiler_type, std::vector> &base_classes, - std::vector &member_function_dies, - std::vector &contained_type_dies, + std::vector &member_function_dies, + std::vector &contained_type_dies, DelayedPropertyList &delayed_properties, const lldb::AccessType default_accessibility, lldb_private::ClangASTImporter::LayoutInfo &layout_info); size_t ParseChildParameters(clang::DeclContext *containing_decl_ctx, - const DWARFDIE &parent_die, bool skip_artificial, - bool &is_static, bool &is_variadic, + const lldb_private::plugin::dwarf::DWARFDIE &parent_die, + bool skip_artificial, bool &is_static, bool &is_variadic, bool &has_template_params, std::vector &function_args, std::vector &function_param_decls, unsigned &type_quals); - size_t ParseChildEnumerators(lldb_private::CompilerType &compiler_type, - bool is_signed, uint32_t enumerator_byte_size, - const DWARFDIE &parent_die); + size_t ParseChildEnumerators( + lldb_private::CompilerType &compiler_type, bool is_signed, + uint32_t enumerator_byte_size, + const lldb_private::plugin::dwarf::DWARFDIE &parent_die); /// Parse a structure, class, or union type DIE. - lldb::TypeSP ParseStructureLikeDIE(const lldb_private::SymbolContext &sc, - const DWARFDIE &die, - ParsedDWARFTypeAttributes &attrs); + lldb::TypeSP + ParseStructureLikeDIE(const lldb_private::SymbolContext &sc, + const lldb_private::plugin::dwarf::DWARFDIE &die, + ParsedDWARFTypeAttributes &attrs); - clang::Decl *GetClangDeclForDIE(const DWARFDIE &die); + clang::Decl * + GetClangDeclForDIE(const lldb_private::plugin::dwarf::DWARFDIE &die); - clang::DeclContext *GetClangDeclContextForDIE(const DWARFDIE &die); + clang::DeclContext * + GetClangDeclContextForDIE(const lldb_private::plugin::dwarf::DWARFDIE &die); - clang::DeclContext *GetClangDeclContextContainingDIE(const DWARFDIE &die, - DWARFDIE *decl_ctx_die); - lldb_private::OptionalClangModuleID GetOwningClangModule(const DWARFDIE &die); + clang::DeclContext *GetClangDeclContextContainingDIE( + const lldb_private::plugin::dwarf::DWARFDIE &die, + lldb_private::plugin::dwarf::DWARFDIE *decl_ctx_die); + lldb_private::OptionalClangModuleID + GetOwningClangModule(const lldb_private::plugin::dwarf::DWARFDIE &die); - bool CopyUniqueClassMethodTypes(const DWARFDIE &src_class_die, - const DWARFDIE &dst_class_die, - lldb_private::Type *class_type, - std::vector &failures); + bool CopyUniqueClassMethodTypes( + const lldb_private::plugin::dwarf::DWARFDIE &src_class_die, + const lldb_private::plugin::dwarf::DWARFDIE &dst_class_die, + lldb_private::Type *class_type, + std::vector &failures); - clang::DeclContext *GetCachedClangDeclContextForDIE(const DWARFDIE &die); + clang::DeclContext *GetCachedClangDeclContextForDIE( + const lldb_private::plugin::dwarf::DWARFDIE &die); - void LinkDeclContextToDIE(clang::DeclContext *decl_ctx, const DWARFDIE &die); + void LinkDeclContextToDIE(clang::DeclContext *decl_ctx, + const lldb_private::plugin::dwarf::DWARFDIE &die); - void LinkDeclToDIE(clang::Decl *decl, const DWARFDIE &die); + void LinkDeclToDIE(clang::Decl *decl, + const lldb_private::plugin::dwarf::DWARFDIE &die); /// If \p type_sp is valid, calculate and set its symbol context scope, and /// update the type list for its backing symbol file. /// /// Returns \p type_sp. - lldb::TypeSP - UpdateSymbolContextScopeForType(const lldb_private::SymbolContext &sc, - const DWARFDIE &die, lldb::TypeSP type_sp); + lldb::TypeSP UpdateSymbolContextScopeForType( + const lldb_private::SymbolContext &sc, + const lldb_private::plugin::dwarf::DWARFDIE &die, lldb::TypeSP type_sp); /// Follow Clang Module Skeleton CU references to find a type definition. - lldb::TypeSP ParseTypeFromClangModule(const lldb_private::SymbolContext &sc, - const DWARFDIE &die, - lldb_private::Log *log); + lldb::TypeSP + ParseTypeFromClangModule(const lldb_private::SymbolContext &sc, + const lldb_private::plugin::dwarf::DWARFDIE &die, + lldb_private::Log *log); // Return true if this type is a declaration to a type in an external // module. - lldb::ModuleSP GetModuleForType(const DWARFDIE &die); - - void PrepareContextToReceiveMembers(clang::DeclContext *decl_ctx, - const DWARFDIE &decl_ctx_die, - const DWARFDIE &die, - const char *type_name_cstr); + lldb::ModuleSP + GetModuleForType(const lldb_private::plugin::dwarf::DWARFDIE &die); static bool classof(const DWARFASTParser *Parser) { return Parser->GetKind() == Kind::DWARFASTParserClang; @@ -265,8 +274,10 @@ private: /// Parsed form of all attributes that are relevant for parsing type members. struct MemberAttributes { - explicit MemberAttributes(const DWARFDIE &die, const DWARFDIE &parent_die, - lldb::ModuleSP module_sp); + explicit MemberAttributes( + const lldb_private::plugin::dwarf::DWARFDIE &die, + const lldb_private::plugin::dwarf::DWARFDIE &parent_die, + lldb::ModuleSP module_sp); const char *name = nullptr; /// Indicates how many bits into the word (according to the host endianness) /// the low-order bit of the field starts. Can be negative. @@ -313,12 +324,15 @@ private: /// created property. /// \param delayed_properties The list of delayed properties that the result /// will be appended to. - void ParseObjCProperty(const DWARFDIE &die, const DWARFDIE &parent_die, - const lldb_private::CompilerType &class_clang_type, - DelayedPropertyList &delayed_properties); + void + ParseObjCProperty(const lldb_private::plugin::dwarf::DWARFDIE &die, + const lldb_private::plugin::dwarf::DWARFDIE &parent_die, + const lldb_private::CompilerType &class_clang_type, + DelayedPropertyList &delayed_properties); void - ParseSingleMember(const DWARFDIE &die, const DWARFDIE &parent_die, + ParseSingleMember(const lldb_private::plugin::dwarf::DWARFDIE &die, + const lldb_private::plugin::dwarf::DWARFDIE &parent_die, const lldb_private::CompilerType &class_clang_type, lldb::AccessType default_accessibility, lldb_private::ClangASTImporter::LayoutInfo &layout_info, @@ -336,25 +350,31 @@ private: /// \param[in] class_clang_type The parent RecordType of the static /// member this function will create. void CreateStaticMemberVariable( - const DWARFDIE &die, const MemberAttributes &attrs, + const lldb_private::plugin::dwarf::DWARFDIE &die, + const MemberAttributes &attrs, const lldb_private::CompilerType &class_clang_type); - bool CompleteRecordType(const DWARFDIE &die, lldb_private::Type *type, + bool CompleteRecordType(const lldb_private::plugin::dwarf::DWARFDIE &die, + lldb_private::Type *type, lldb_private::CompilerType &clang_type); - bool CompleteEnumType(const DWARFDIE &die, lldb_private::Type *type, + bool CompleteEnumType(const lldb_private::plugin::dwarf::DWARFDIE &die, + lldb_private::Type *type, lldb_private::CompilerType &clang_type); - lldb::TypeSP ParseTypeModifier(const lldb_private::SymbolContext &sc, - const DWARFDIE &die, - ParsedDWARFTypeAttributes &attrs); + lldb::TypeSP + ParseTypeModifier(const lldb_private::SymbolContext &sc, + const lldb_private::plugin::dwarf::DWARFDIE &die, + ParsedDWARFTypeAttributes &attrs); lldb::TypeSP ParseEnum(const lldb_private::SymbolContext &sc, - const DWARFDIE &die, ParsedDWARFTypeAttributes &attrs); - lldb::TypeSP ParseSubroutine(const DWARFDIE &die, + const lldb_private::plugin::dwarf::DWARFDIE &die, + ParsedDWARFTypeAttributes &attrs); + lldb::TypeSP ParseSubroutine(const lldb_private::plugin::dwarf::DWARFDIE &die, const ParsedDWARFTypeAttributes &attrs); - lldb::TypeSP ParseArrayType(const DWARFDIE &die, + lldb::TypeSP ParseArrayType(const lldb_private::plugin::dwarf::DWARFDIE &die, const ParsedDWARFTypeAttributes &attrs); - lldb::TypeSP ParsePointerToMemberType(const DWARFDIE &die, - const ParsedDWARFTypeAttributes &attrs); + lldb::TypeSP + ParsePointerToMemberType(const lldb_private::plugin::dwarf::DWARFDIE &die, + const ParsedDWARFTypeAttributes &attrs); /// Parses a DW_TAG_inheritance DIE into a base/super class. /// @@ -371,7 +391,8 @@ private: /// \param layout_info The layout information that will be updated for C++ /// base classes with the base offset. void ParseInheritance( - const DWARFDIE &die, const DWARFDIE &parent_die, + const lldb_private::plugin::dwarf::DWARFDIE &die, + const lldb_private::plugin::dwarf::DWARFDIE &parent_die, const lldb_private::CompilerType class_clang_type, const lldb::AccessType default_accessibility, const lldb::ModuleSP &module_sp, @@ -388,7 +409,8 @@ private: /// \param layout_info The layout information that will be updated for // base classes with the base offset void - ParseRustVariantPart(DWARFDIE &die, const DWARFDIE &parent_die, + ParseRustVariantPart(lldb_private::plugin::dwarf::DWARFDIE &die, + const lldb_private::plugin::dwarf::DWARFDIE &parent_die, lldb_private::CompilerType &class_clang_type, const lldb::AccessType default_accesibility, lldb_private::ClangASTImporter::LayoutInfo &layout_info); @@ -398,9 +420,8 @@ private: /// Some attributes are relevant for all kinds of types (declaration), while /// others are only meaningful to a specific type (is_virtual) struct ParsedDWARFTypeAttributes { - typedef lldb_private::plugin::dwarf::DWARFDIE DWARFDIE; - - explicit ParsedDWARFTypeAttributes(const DWARFDIE &die); + explicit ParsedDWARFTypeAttributes( + const lldb_private::plugin::dwarf::DWARFDIE &die); lldb::AccessType accessibility = lldb::eAccessNone; bool is_artificial = false; @@ -417,7 +438,7 @@ struct ParsedDWARFTypeAttributes { const char *mangled_name = nullptr; lldb_private::ConstString name; lldb_private::Declaration decl; - DWARFDIE object_pointer; + lldb_private::plugin::dwarf::DWARFDIE object_pointer; lldb_private::plugin::dwarf::DWARFFormValue abstract_origin; lldb_private::plugin::dwarf::DWARFFormValue containing_type; lldb_private::plugin::dwarf::DWARFFormValue signature; diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp index 6330470b970e..90e42be7202d 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp @@ -85,11 +85,6 @@ bool DebugNamesDWARFIndex::ProcessEntry( DWARFDIE die = GetDIE(entry); if (!die) return true; - // Clang erroneously emits index entries for declaration DIEs in case when the - // definition is in a type unit (llvm.org/pr77696). Weed those out. - if (die.IsStructUnionOrClass() && - die.GetAttributeValueAsUnsigned(DW_AT_declaration, 0)) - return true; return callback(die); } diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index af3ba2cd5b39..a52a7d676737 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -481,13 +481,6 @@ static ConstString GetDWARFMachOSegmentName() { return g_dwarf_section_name; } -llvm::DenseMap & -SymbolFileDWARF::GetForwardDeclCompilerTypeToDIE() { - if (SymbolFileDWARFDebugMap *debug_map_symfile = GetDebugMapSymfile()) - return debug_map_symfile->GetForwardDeclCompilerTypeToDIE(); - return m_forward_decl_compiler_type_to_die; -} - UniqueDWARFASTTypeMap &SymbolFileDWARF::GetUniqueDWARFASTTypeMap() { SymbolFileDWARFDebugMap *debug_map_symfile = GetDebugMapSymfile(); if (debug_map_symfile) @@ -1639,33 +1632,27 @@ bool SymbolFileDWARF::CompleteType(CompilerType &compiler_type) { return true; } - // Once we start resolving this type, remove it from the forward - // declaration map in case anyone's child members or other types require this - // type to get resolved. - DWARFDIE dwarf_die = GetDIE(die_it->second); - GetForwardDeclCompilerTypeToDIE().erase(die_it); - Type *type = nullptr; - if (DWARFASTParser *dwarf_ast = GetDWARFParser(*dwarf_die.GetCU())) - type = dwarf_ast->FindDefinitionTypeForDIE(dwarf_die); - if (!type) - return false; - - die_it = GetForwardDeclCompilerTypeToDIE().find( - compiler_type_no_qualifiers.GetOpaqueQualType()); - if (die_it != GetForwardDeclCompilerTypeToDIE().end()) { - dwarf_die = GetDIE(die_it->getSecond()); + DWARFDIE dwarf_die = GetDIE(die_it->getSecond()); + if (dwarf_die) { + // Once we start resolving this type, remove it from the forward + // declaration map in case anyone child members or other types require this + // type to get resolved. The type will get resolved when all of the calls + // to SymbolFileDWARF::ResolveClangOpaqueTypeDefinition are done. GetForwardDeclCompilerTypeToDIE().erase(die_it); - } - if (Log *log = GetLog(DWARFLog::DebugInfo | DWARFLog::TypeCompletion)) - GetObjectFile()->GetModule()->LogMessageVerboseBacktrace( - log, "{0:x8}: {1} ({2}) '{3}' resolving forward declaration...", - dwarf_die.GetID(), DW_TAG_value_to_name(dwarf_die.Tag()), - dwarf_die.Tag(), type->GetName().AsCString()); - assert(compiler_type); - if (DWARFASTParser *dwarf_ast = GetDWARFParser(*dwarf_die.GetCU())) - return dwarf_ast->CompleteTypeFromDWARF(dwarf_die, type, compiler_type); - return true; + Type *type = GetDIEToType().lookup(dwarf_die.GetDIE()); + + Log *log = GetLog(DWARFLog::DebugInfo | DWARFLog::TypeCompletion); + if (log) + GetObjectFile()->GetModule()->LogMessageVerboseBacktrace( + log, "{0:x8}: {1} ({2}) '{3}' resolving forward declaration...", + dwarf_die.GetID(), DW_TAG_value_to_name(dwarf_die.Tag()), + dwarf_die.Tag(), type->GetName().AsCString()); + assert(compiler_type); + if (DWARFASTParser *dwarf_ast = GetDWARFParser(*dwarf_die.GetCU())) + return dwarf_ast->CompleteTypeFromDWARF(dwarf_die, type, compiler_type); + } + return false; } Type *SymbolFileDWARF::ResolveType(const DWARFDIE &die, diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h index 35893f2072dd..7282c08c6857 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h @@ -335,8 +335,12 @@ public: virtual DIEToTypePtr &GetDIEToType() { return m_die_to_type; } - virtual llvm::DenseMap & - GetForwardDeclCompilerTypeToDIE(); + typedef llvm::DenseMap + CompilerTypeToDIE; + + virtual CompilerTypeToDIE &GetForwardDeclCompilerTypeToDIE() { + return m_forward_decl_compiler_type_to_die; + } typedef llvm::DenseMap DIEToVariableSP; @@ -529,14 +533,9 @@ protected: NameToOffsetMap m_function_scope_qualified_name_map; std::unique_ptr m_ranges; UniqueDWARFASTTypeMap m_unique_ast_type_map; - // A map from DIE to lldb_private::Type. For record type, the key might be - // either declaration DIE or definition DIE. DIEToTypePtr m_die_to_type; DIEToVariableSP m_die_to_variable_sp; - // A map from CompilerType to the struct/class/union/enum DIE (might be a - // declaration or a definition) that is used to construct it. - llvm::DenseMap - m_forward_decl_compiler_type_to_die; + CompilerTypeToDIE m_forward_decl_compiler_type_to_die; llvm::DenseMap> m_type_unit_support_files; std::vector m_lldb_cu_to_dwarf_unit; diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h index d7d571919bc7..de22dd676eef 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h @@ -284,11 +284,6 @@ protected: lldb::TypeSP FindCompleteObjCDefinitionTypeForDIE( const DWARFDIE &die, ConstString type_name, bool must_be_implementation); - llvm::DenseMap & - GetForwardDeclCompilerTypeToDIE() { - return m_forward_decl_compiler_type_to_die; - } - UniqueDWARFASTTypeMap &GetUniqueDWARFASTTypeMap() { return m_unique_ast_type_map; } @@ -326,10 +321,6 @@ protected: std::vector m_func_indexes; // Sorted by address std::vector m_glob_indexes; std::map>, OSOInfoSP> m_oso_map; - // A map from CompilerType to the struct/class/union/enum DIE (might be a - // declaration or a definition) that is used to construct it. - llvm::DenseMap - m_forward_decl_compiler_type_to_die; UniqueDWARFASTTypeMap m_unique_ast_type_map; LazyBool m_supports_DW_AT_APPLE_objc_complete_type; DebugMap m_debug_map; diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp index e4db39cabf6f..71c9997e4c82 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp @@ -110,7 +110,7 @@ SymbolFileDWARF::DIEToVariableSP &SymbolFileDWARFDwo::GetDIEToVariable() { return GetBaseSymbolFile().GetDIEToVariable(); } -llvm::DenseMap & +SymbolFileDWARF::CompilerTypeToDIE & SymbolFileDWARFDwo::GetForwardDeclCompilerTypeToDIE() { return GetBaseSymbolFile().GetForwardDeclCompilerTypeToDIE(); } diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h index 2f0ac415e90d..1500540424b5 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h @@ -72,8 +72,7 @@ protected: DIEToVariableSP &GetDIEToVariable() override; - llvm::DenseMap & - GetForwardDeclCompilerTypeToDIE() override; + CompilerTypeToDIE &GetForwardDeclCompilerTypeToDIE() override; UniqueDWARFASTTypeMap &GetUniqueDWARFASTTypeMap() override; diff --git a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp index 3d201e96f92c..223518f0ae82 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp @@ -13,75 +13,66 @@ using namespace lldb_private::dwarf; using namespace lldb_private::plugin::dwarf; -static bool IsStructOrClassTag(llvm::dwarf::Tag Tag) { - return Tag == llvm::dwarf::Tag::DW_TAG_class_type || - Tag == llvm::dwarf::Tag::DW_TAG_structure_type; -} - -UniqueDWARFASTType *UniqueDWARFASTTypeList::Find( - const DWARFDIE &die, const lldb_private::Declaration &decl, - const int32_t byte_size, bool is_forward_declaration) { - for (UniqueDWARFASTType &udt : m_collection) { +bool UniqueDWARFASTTypeList::Find(const DWARFDIE &die, + const lldb_private::Declaration &decl, + const int32_t byte_size, + UniqueDWARFASTType &entry) const { + for (const UniqueDWARFASTType &udt : m_collection) { // Make sure the tags match - if (udt.m_die.Tag() == die.Tag() || (IsStructOrClassTag(udt.m_die.Tag()) && - IsStructOrClassTag(die.Tag()))) { - // If they are not both definition DIEs or both declaration DIEs, then - // don't check for byte size and declaration location, because declaration - // DIEs usually don't have those info. - bool matching_size_declaration = - udt.m_is_forward_declaration != is_forward_declaration - ? true - : (udt.m_byte_size < 0 || byte_size < 0 || - udt.m_byte_size == byte_size) && - udt.m_declaration == decl; - if (!matching_size_declaration) - continue; - // The type has the same name, and was defined on the same file and - // line. Now verify all of the parent DIEs match. - DWARFDIE parent_arg_die = die.GetParent(); - DWARFDIE parent_pos_die = udt.m_die.GetParent(); - bool match = true; - bool done = false; - while (!done && match && parent_arg_die && parent_pos_die) { - const dw_tag_t parent_arg_tag = parent_arg_die.Tag(); - const dw_tag_t parent_pos_tag = parent_pos_die.Tag(); - if (parent_arg_tag == parent_pos_tag || - (IsStructOrClassTag(parent_arg_tag) && - IsStructOrClassTag(parent_pos_tag))) { - switch (parent_arg_tag) { - case DW_TAG_class_type: - case DW_TAG_structure_type: - case DW_TAG_union_type: - case DW_TAG_namespace: { - const char *parent_arg_die_name = parent_arg_die.GetName(); - if (parent_arg_die_name == nullptr) { - // Anonymous (i.e. no-name) struct - match = false; - } else { - const char *parent_pos_die_name = parent_pos_die.GetName(); - if (parent_pos_die_name == nullptr || - ((parent_arg_die_name != parent_pos_die_name) && - strcmp(parent_arg_die_name, parent_pos_die_name))) - match = false; + if (udt.m_die.Tag() == die.Tag()) { + // Validate byte sizes of both types only if both are valid. + if (udt.m_byte_size < 0 || byte_size < 0 || + udt.m_byte_size == byte_size) { + // Make sure the file and line match + if (udt.m_declaration == decl) { + // The type has the same name, and was defined on the same file and + // line. Now verify all of the parent DIEs match. + DWARFDIE parent_arg_die = die.GetParent(); + DWARFDIE parent_pos_die = udt.m_die.GetParent(); + bool match = true; + bool done = false; + while (!done && match && parent_arg_die && parent_pos_die) { + const dw_tag_t parent_arg_tag = parent_arg_die.Tag(); + const dw_tag_t parent_pos_tag = parent_pos_die.Tag(); + if (parent_arg_tag == parent_pos_tag) { + switch (parent_arg_tag) { + case DW_TAG_class_type: + case DW_TAG_structure_type: + case DW_TAG_union_type: + case DW_TAG_namespace: { + const char *parent_arg_die_name = parent_arg_die.GetName(); + if (parent_arg_die_name == + nullptr) // Anonymous (i.e. no-name) struct + { + match = false; + } else { + const char *parent_pos_die_name = parent_pos_die.GetName(); + if (parent_pos_die_name == nullptr || + ((parent_arg_die_name != parent_pos_die_name) && + strcmp(parent_arg_die_name, parent_pos_die_name))) + match = false; + } + } break; + + case DW_TAG_compile_unit: + case DW_TAG_partial_unit: + done = true; + break; + default: + break; + } } - } break; + parent_arg_die = parent_arg_die.GetParent(); + parent_pos_die = parent_pos_die.GetParent(); + } - case DW_TAG_compile_unit: - case DW_TAG_partial_unit: - done = true; - break; - default: - break; + if (match) { + entry = udt; + return true; } } - parent_arg_die = parent_arg_die.GetParent(); - parent_pos_die = parent_pos_die.GetParent(); - } - - if (match) { - return &udt; } } } - return nullptr; + return false; } diff --git a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h index 29e5c02dcbe1..bf3cbae55e5c 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h @@ -23,19 +23,31 @@ public: // Constructors and Destructors UniqueDWARFASTType() : m_type_sp(), m_die(), m_declaration() {} + UniqueDWARFASTType(lldb::TypeSP &type_sp, const DWARFDIE &die, + const Declaration &decl, int32_t byte_size) + : m_type_sp(type_sp), m_die(die), m_declaration(decl), + m_byte_size(byte_size) {} + UniqueDWARFASTType(const UniqueDWARFASTType &rhs) : m_type_sp(rhs.m_type_sp), m_die(rhs.m_die), - m_declaration(rhs.m_declaration), m_byte_size(rhs.m_byte_size), - m_is_forward_declaration(rhs.m_is_forward_declaration) {} + m_declaration(rhs.m_declaration), m_byte_size(rhs.m_byte_size) {} ~UniqueDWARFASTType() = default; + UniqueDWARFASTType &operator=(const UniqueDWARFASTType &rhs) { + if (this != &rhs) { + m_type_sp = rhs.m_type_sp; + m_die = rhs.m_die; + m_declaration = rhs.m_declaration; + m_byte_size = rhs.m_byte_size; + } + return *this; + } + lldb::TypeSP m_type_sp; DWARFDIE m_die; Declaration m_declaration; int32_t m_byte_size = -1; - // True if the m_die is a forward declaration DIE. - bool m_is_forward_declaration = true; }; class UniqueDWARFASTTypeList { @@ -50,9 +62,8 @@ public: m_collection.push_back(entry); } - UniqueDWARFASTType *Find(const DWARFDIE &die, const Declaration &decl, - const int32_t byte_size, - bool is_forward_declaration); + bool Find(const DWARFDIE &die, const Declaration &decl, + const int32_t byte_size, UniqueDWARFASTType &entry) const; protected: typedef std::vector collection; @@ -69,15 +80,14 @@ public: m_collection[name.GetCString()].Append(entry); } - UniqueDWARFASTType *Find(ConstString name, const DWARFDIE &die, - const Declaration &decl, const int32_t byte_size, - bool is_forward_declaration) { + bool Find(ConstString name, const DWARFDIE &die, const Declaration &decl, + const int32_t byte_size, UniqueDWARFASTType &entry) const { const char *unique_name_cstr = name.GetCString(); - collection::iterator pos = m_collection.find(unique_name_cstr); + collection::const_iterator pos = m_collection.find(unique_name_cstr); if (pos != m_collection.end()) { - return pos->second.Find(die, decl, byte_size, is_forward_declaration); + return pos->second.Find(die, decl, byte_size, entry); } - return nullptr; + return false; } protected: diff --git a/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test b/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test deleted file mode 100644 index d253981b498c..000000000000 --- a/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test +++ /dev/null @@ -1,36 +0,0 @@ -# Test definition DIE searching is delayed until complete type is required. - -# UNSUPPORTED: system-windows - -# RUN: split-file %s %t -# RUN: %clangxx_host %t/main.cpp %t/t1_def.cpp -gdwarf -o %t.out -# RUN: %lldb -b %t.out -s %t/lldb.cmd | FileCheck %s - -# CHECK: (lldb) p v1 -# CHECK: DWARFASTParserClang::ParseTypeFromDWARF{{.*}}DW_TAG_structure_type (DW_TAG_structure_type) name = 't2' -# CHECK: DWARFASTParserClang::ParseTypeFromDWARF{{.*}}DW_TAG_structure_type (DW_TAG_structure_type) name = 't1' -# CHECK: DW_TAG_structure_type (DW_TAG_structure_type) 't2' resolving forward declaration... -# CHECK: (t2) {} -# CHECK: (lldb) p v2 -# CHECK: DWARFASTParserClang::ParseTypeFromDWARF{{.*}}DW_TAG_structure_type (DW_TAG_structure_type) name = 't1' -# CHECK: DW_TAG_structure_type (DW_TAG_structure_type) 't1' resolving forward declaration... - -#--- lldb.cmd -log enable dwarf comp -p v1 -p v2 - -#--- main.cpp -template -struct t2 { -}; -struct t1; -t2 v1; // this CU doesn't have definition DIE for t1, but only declaration DIE for it. -int main() { -} - -#--- t1_def.cpp -struct t1 { // this CU contains definition DIE for t1. - int x; -}; -t1 v2; -- GitLab From de3f1b6d68ab8a0e827db84b328803857a4f60df Mon Sep 17 00:00:00 2001 From: Pavel Labath Date: Thu, 6 Jun 2024 15:39:30 +0000 Subject: [PATCH 128/462] [lldb] Test case for the bug in #92328 --- .../x86/simple-template-names-context.cpp | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 lldb/test/Shell/SymbolFile/DWARF/x86/simple-template-names-context.cpp diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/simple-template-names-context.cpp b/lldb/test/Shell/SymbolFile/DWARF/x86/simple-template-names-context.cpp new file mode 100644 index 000000000000..a8a4d3b8fbd5 --- /dev/null +++ b/lldb/test/Shell/SymbolFile/DWARF/x86/simple-template-names-context.cpp @@ -0,0 +1,44 @@ +// Test that we can correctly resolve forward declared types when they only +// differ in the template arguments of the surrounding context. The reproducer +// is sensitive to the order of declarations, so we test in both directions. + +// REQUIRES: lld + +// RUN: %clang --target=x86_64-pc-linux -c %s -o %t-a.o -g -gsimple-template-names -DFILE_A +// RUN: %clang --target=x86_64-pc-linux -c %s -o %t-b.o -g -gsimple-template-names -DFILE_B +// RUN: ld.lld %t-a.o %t-b.o -o %t +// RUN: %lldb %t -o "target variable --ptr-depth 1 --show-types both_a both_b" -o exit | FileCheck %s + +// CHECK: (lldb) target variable +// CHECK-NEXT: (ReferencesBoth<'A'>) both_a = { +// CHECK-NEXT: (Outer<'A'>::Inner *) a = 0x{{[0-9A-Fa-f]*}} {} +// CHECK-NEXT: (Outer<'A'>::Inner *) b = 0x{{[0-9A-Fa-f]*}} {} +// CHECK-NEXT: } +// CHECK-NEXT: (ReferencesBoth<'B'>) both_b = { +// CHECK-NEXT: (Outer<'A'>::Inner *) a = 0x{{[0-9A-Fa-f]*}} {} +// CHECK-NEXT: (Outer<'B'>::Inner *) b = 0x{{[0-9A-Fa-f]*}} {} +// CHECK-NEXT: } + +template +struct Outer { + struct Inner {}; +}; + +template +struct ReferencesBoth { + Outer<'A'>::Inner *a; + Outer<'B'>::Inner *b; +}; + +#ifdef FILE_A +Outer<'A'>::Inner inner_a; +extern Outer<'B'>::Inner inner_b; + +ReferencesBoth<'A'> both_a{&inner_a, &inner_b}; + +#else +extern Outer<'A'>::Inner inner_a; +Outer<'B'>::Inner inner_b; + +ReferencesBoth<'B'> both_b{&inner_a, &inner_b}; +#endif -- GitLab From 1ca0055f45abe7e4d484a0af457fc20be318c8e2 Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Thu, 6 Jun 2024 12:16:11 -0400 Subject: [PATCH 129/462] [AMDGPU] Add a new target gfx1152 (#94534) --- clang/include/clang/Basic/Cuda.h | 1 + clang/lib/Basic/Cuda.cpp | 1 + clang/lib/Basic/Targets/NVPTX.cpp | 1 + clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp | 1 + clang/test/CodeGenOpenCL/amdgpu-features.cl | 2 ++ clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl | 1 + clang/test/Driver/amdgpu-macros.cl | 1 + clang/test/Driver/amdgpu-mcpu.cl | 2 ++ clang/test/Misc/target-invalid-cpu-note.c | 4 ++-- flang/cmake/modules/AddFlangOffloadRuntime.cmake | 1 + libclc/CMakeLists.txt | 2 +- llvm/docs/AMDGPUUsage.rst | 15 +++++++++++++-- llvm/include/llvm/BinaryFormat/ELF.h | 1 + llvm/include/llvm/TargetParser/TargetParser.h | 1 + llvm/lib/Object/ELFObjectFile.cpp | 2 ++ llvm/lib/ObjectYAML/ELFYAML.cpp | 1 + llvm/lib/Target/AMDGPU/AMDGPU.td | 6 ++++++ llvm/lib/Target/AMDGPU/GCNProcessors.td | 6 +++++- .../AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp | 2 ++ llvm/lib/TargetParser/TargetParser.cpp | 4 ++++ .../CodeGen/AMDGPU/directive-amdgcn-target.ll | 2 ++ llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll | 2 ++ llvm/test/CodeGen/AMDGPU/occupancy-levels.ll | 2 ++ llvm/test/MC/AMDGPU/gfx1150_asm_features.s | 1 + .../test/Object/AMDGPU/elf-header-flags-mach.yaml | 7 +++++++ .../tools/llvm-objdump/ELF/AMDGPU/subtarget.ll | 5 +++++ .../llvm-readobj/ELF/AMDGPU/elf-headers.test | 9 +++++++++ llvm/tools/llvm-readobj/ELFDumper.cpp | 1 + offload/DeviceRTL/CMakeLists.txt | 2 +- 29 files changed, 79 insertions(+), 7 deletions(-) diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h index d15171d959c4..0d5e38e825aa 100644 --- a/clang/include/clang/Basic/Cuda.h +++ b/clang/include/clang/Basic/Cuda.h @@ -124,6 +124,7 @@ enum class CudaArch { GFX1103, GFX1150, GFX1151, + GFX1152, GFX12_GENERIC, GFX1200, GFX1201, diff --git a/clang/lib/Basic/Cuda.cpp b/clang/lib/Basic/Cuda.cpp index e2609b9573cc..1d96a929f95d 100644 --- a/clang/lib/Basic/Cuda.cpp +++ b/clang/lib/Basic/Cuda.cpp @@ -144,6 +144,7 @@ static const CudaArchToStringMap arch_names[] = { GFX(1103), // gfx1103 GFX(1150), // gfx1150 GFX(1151), // gfx1151 + GFX(1152), // gfx1152 {CudaArch::GFX12_GENERIC, "gfx12-generic", "compute_amdgcn"}, GFX(1200), // gfx1200 GFX(1201), // gfx1201 diff --git a/clang/lib/Basic/Targets/NVPTX.cpp b/clang/lib/Basic/Targets/NVPTX.cpp index fc6ef1119e9c..ff7d2f1f92aa 100644 --- a/clang/lib/Basic/Targets/NVPTX.cpp +++ b/clang/lib/Basic/Targets/NVPTX.cpp @@ -228,6 +228,7 @@ void NVPTXTargetInfo::getTargetDefines(const LangOptions &Opts, case CudaArch::GFX1103: case CudaArch::GFX1150: case CudaArch::GFX1151: + case CudaArch::GFX1152: case CudaArch::GFX12_GENERIC: case CudaArch::GFX1200: case CudaArch::GFX1201: diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp index 191bd757831f..6e9a1bacd9bf 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp @@ -3537,6 +3537,7 @@ void CGOpenMPRuntimeGPU::processRequiresDirective( case CudaArch::GFX1103: case CudaArch::GFX1150: case CudaArch::GFX1151: + case CudaArch::GFX1152: case CudaArch::GFX12_GENERIC: case CudaArch::GFX1200: case CudaArch::GFX1201: diff --git a/clang/test/CodeGenOpenCL/amdgpu-features.cl b/clang/test/CodeGenOpenCL/amdgpu-features.cl index 2fda52dcd2dc..854ab39791f1 100644 --- a/clang/test/CodeGenOpenCL/amdgpu-features.cl +++ b/clang/test/CodeGenOpenCL/amdgpu-features.cl @@ -49,6 +49,7 @@ // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1103 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1103 %s // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1150 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1150 %s // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1151 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1151 %s +// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1152 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1152 %s // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1200 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1200 %s // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1201 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1201 %s @@ -100,6 +101,7 @@ // GFX1103: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" // GFX1150: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" // GFX1151: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" +// GFX1152: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" // GFX1200: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" // GFX1201: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl index d17ff81e5d43..66061786cca6 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl @@ -5,6 +5,7 @@ // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1103 -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1150 -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1151 -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1152 -emit-llvm -o - %s | FileCheck %s typedef unsigned int uint; typedef unsigned long ulong; diff --git a/clang/test/Driver/amdgpu-macros.cl b/clang/test/Driver/amdgpu-macros.cl index a878a7decee9..3e4a570671ba 100644 --- a/clang/test/Driver/amdgpu-macros.cl +++ b/clang/test/Driver/amdgpu-macros.cl @@ -127,6 +127,7 @@ // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1103 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1103 -DFAMILY=GFX11 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1150 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1150 -DFAMILY=GFX11 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1151 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1151 -DFAMILY=GFX11 +// RUN: %clang -E -dM -target amdgcn -mcpu=gfx1152 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1152 -DFAMILY=GFX11 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1200 -DFAMILY=GFX12 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1201 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1201 -DFAMILY=GFX12 diff --git a/clang/test/Driver/amdgpu-mcpu.cl b/clang/test/Driver/amdgpu-mcpu.cl index 5b6a22016f04..4b0ef92b682a 100644 --- a/clang/test/Driver/amdgpu-mcpu.cl +++ b/clang/test/Driver/amdgpu-mcpu.cl @@ -112,6 +112,7 @@ // RUN: %clang -### -target amdgcn -mcpu=gfx1103 %s 2>&1 | FileCheck --check-prefix=GFX1103 %s // RUN: %clang -### -target amdgcn -mcpu=gfx1150 %s 2>&1 | FileCheck --check-prefix=GFX1150 %s // RUN: %clang -### -target amdgcn -mcpu=gfx1151 %s 2>&1 | FileCheck --check-prefix=GFX1151 %s +// RUN: %clang -### -target amdgcn -mcpu=gfx1152 %s 2>&1 | FileCheck --check-prefix=GFX1152 %s // RUN: %clang -### -target amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefix=GFX1200 %s // RUN: %clang -### -target amdgcn -mcpu=gfx1201 %s 2>&1 | FileCheck --check-prefix=GFX1201 %s @@ -164,6 +165,7 @@ // GFX1103: "-target-cpu" "gfx1103" // GFX1150: "-target-cpu" "gfx1150" // GFX1151: "-target-cpu" "gfx1151" +// GFX1152: "-target-cpu" "gfx1152" // GFX1200: "-target-cpu" "gfx1200" // GFX1201: "-target-cpu" "gfx1201" diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c index 5ec2743c8d41..cb5b6752850c 100644 --- a/clang/test/Misc/target-invalid-cpu-note.c +++ b/clang/test/Misc/target-invalid-cpu-note.c @@ -29,7 +29,7 @@ // RUN: not %clang_cc1 -triple nvptx--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix NVPTX // NVPTX: error: unknown target CPU 'not-a-cpu' -// NVPTX-NEXT: note: valid target CPU values are: sm_20, sm_21, sm_30, sm_32, sm_35, sm_37, sm_50, sm_52, sm_53, sm_60, sm_61, sm_62, sm_70, sm_72, sm_75, sm_80, sm_86, sm_87, sm_89, sm_90, sm_90a, gfx600, gfx601, gfx602, gfx700, gfx701, gfx702, gfx703, gfx704, gfx705, gfx801, gfx802, gfx803, gfx805, gfx810, gfx9-generic, gfx900, gfx902, gfx904, gfx906, gfx908, gfx909, gfx90a, gfx90c, gfx940, gfx941, gfx942, gfx10-1-generic, gfx1010, gfx1011, gfx1012, gfx1013, gfx10-3-generic, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034, gfx1035, gfx1036, gfx11-generic, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx12-generic, gfx1200, gfx1201{{$}} +// NVPTX-NEXT: note: valid target CPU values are: sm_20, sm_21, sm_30, sm_32, sm_35, sm_37, sm_50, sm_52, sm_53, sm_60, sm_61, sm_62, sm_70, sm_72, sm_75, sm_80, sm_86, sm_87, sm_89, sm_90, sm_90a, gfx600, gfx601, gfx602, gfx700, gfx701, gfx702, gfx703, gfx704, gfx705, gfx801, gfx802, gfx803, gfx805, gfx810, gfx9-generic, gfx900, gfx902, gfx904, gfx906, gfx908, gfx909, gfx90a, gfx90c, gfx940, gfx941, gfx942, gfx10-1-generic, gfx1010, gfx1011, gfx1012, gfx1013, gfx10-3-generic, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034, gfx1035, gfx1036, gfx11-generic, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1152, gfx12-generic, gfx1200, gfx1201{{$}} // RUN: not %clang_cc1 -triple r600--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix R600 // R600: error: unknown target CPU 'not-a-cpu' @@ -37,7 +37,7 @@ // RUN: not %clang_cc1 -triple amdgcn--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix AMDGCN // AMDGCN: error: unknown target CPU 'not-a-cpu' -// AMDGCN-NEXT: note: valid target CPU values are: gfx600, tahiti, gfx601, pitcairn, verde, gfx602, hainan, oland, gfx700, kaveri, gfx701, hawaii, gfx702, gfx703, kabini, mullins, gfx704, bonaire, gfx705, gfx801, carrizo, gfx802, iceland, tonga, gfx803, fiji, polaris10, polaris11, gfx805, tongapro, gfx810, stoney, gfx900, gfx902, gfx904, gfx906, gfx908, gfx909, gfx90a, gfx90c, gfx940, gfx941, gfx942, gfx1010, gfx1011, gfx1012, gfx1013, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034, gfx1035, gfx1036, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1200, gfx1201, gfx9-generic, gfx10-1-generic, gfx10-3-generic, gfx11-generic, gfx12-generic{{$}} +// AMDGCN-NEXT: note: valid target CPU values are: gfx600, tahiti, gfx601, pitcairn, verde, gfx602, hainan, oland, gfx700, kaveri, gfx701, hawaii, gfx702, gfx703, kabini, mullins, gfx704, bonaire, gfx705, gfx801, carrizo, gfx802, iceland, tonga, gfx803, fiji, polaris10, polaris11, gfx805, tongapro, gfx810, stoney, gfx900, gfx902, gfx904, gfx906, gfx908, gfx909, gfx90a, gfx90c, gfx940, gfx941, gfx942, gfx1010, gfx1011, gfx1012, gfx1013, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034, gfx1035, gfx1036, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1152, gfx1200, gfx1201, gfx9-generic, gfx10-1-generic, gfx10-3-generic, gfx11-generic, gfx12-generic{{$}} // RUN: not %clang_cc1 -triple wasm64--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix WEBASM // WEBASM: error: unknown target CPU 'not-a-cpu' diff --git a/flang/cmake/modules/AddFlangOffloadRuntime.cmake b/flang/cmake/modules/AddFlangOffloadRuntime.cmake index 0af12c8cfd54..6407be5d038b 100644 --- a/flang/cmake/modules/AddFlangOffloadRuntime.cmake +++ b/flang/cmake/modules/AddFlangOffloadRuntime.cmake @@ -101,6 +101,7 @@ macro(enable_omp_offload_compilation files) "gfx908;gfx90a;gfx90c;gfx940;gfx1010;gfx1030" "gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036" "gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151" + "gfx1152" ) set(all_nvptx_architectures "sm_35;sm_37;sm_50;sm_52;sm_53;sm_60;sm_61;sm_62" diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt index 5ce179524308..9858ae905983 100644 --- a/libclc/CMakeLists.txt +++ b/libclc/CMakeLists.txt @@ -198,7 +198,7 @@ set( tahiti_aliases pitcairn verde oland hainan bonaire kabini kaveri hawaii gfx1010 gfx1011 gfx1012 gfx1013 gfx1030 gfx1031 gfx1032 gfx1033 gfx1034 gfx1035 gfx1036 gfx1100 gfx1101 gfx1102 gfx1103 - gfx1150 gfx1151 + gfx1150 gfx1151 gfx1152 gfx1200 gfx1201 ) diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index bb6751038fc9..aa50ce329d1d 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -504,6 +504,13 @@ Every processor supports every OS ABI (see :ref:`amdgpu-os`) with the following work-item Add product IDs names. + ``gfx1152`` ``amdgcn`` APU - cumode - Architected *TBA* + - wavefrontsize64 flat + scratch .. TODO:: + - Packed + work-item Add product + IDs names. + ``gfx1200`` ``amdgcn`` dGPU - cumode - Architected *TBA* - wavefrontsize64 flat scratch .. TODO:: @@ -591,11 +598,13 @@ Generic processor code objects are versioned. See :ref:`amdgpu-generic-processor - ``gfx1102`` - Packed hazards specific to some targets - ``gfx1103`` work-item within this family. - ``gfx1150`` IDs - - ``gfx1151`` Not all VGPRs can be used on: + - ``gfx1151`` + - ``gfx1152`` Not all VGPRs can be used on: - ``gfx1100`` - ``gfx1101`` - ``gfx1151`` + - ``gfx1152`` SALU floating point instructions and single-use VGPR hint @@ -604,12 +613,14 @@ Generic processor code objects are versioned. See :ref:`amdgpu-generic-processor - ``gfx1150`` - ``gfx1151`` + - ``gfx1152`` SGPRs are not supported for src1 in dpp instructions for: - ``gfx1150`` - ``gfx1151`` + - ``gfx1152`` ``gfx12-generic`` ``amdgcn`` - ``gfx1200`` - wavefrontsize64 - Architected No restrictions. @@ -1979,7 +1990,7 @@ The AMDGPU backend uses the following ELF header: ``EF_AMDGPU_MACH_AMDGCN_GFX10_1_GENERIC`` 0x052 ``gfx10-1-generic`` ``EF_AMDGPU_MACH_AMDGCN_GFX10_3_GENERIC`` 0x053 ``gfx10-3-generic`` ``EF_AMDGPU_MACH_AMDGCN_GFX11_GENERIC`` 0x054 ``gfx11-generic`` - *reserved* 0x055 Reserved. + ``EF_AMDGPU_MACH_AMDGCN_GFX1152`` 0x055 ``gfx1152``. *reserved* 0x056 Reserved. *reserved* 0x057 Reserved. *reserved* 0x058 Reserved. diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index 7364d619bc1b..fbfba515fa5f 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -785,6 +785,7 @@ enum : unsigned { EF_AMDGPU_MACH_AMDGCN_GFX1200 = 0x048, EF_AMDGPU_MACH_AMDGCN_RESERVED_0X49 = 0x049, EF_AMDGPU_MACH_AMDGCN_GFX1151 = 0x04a, + EF_AMDGPU_MACH_AMDGCN_GFX1152 = 0x055, EF_AMDGPU_MACH_AMDGCN_GFX941 = 0x04b, EF_AMDGPU_MACH_AMDGCN_GFX942 = 0x04c, EF_AMDGPU_MACH_AMDGCN_RESERVED_0X4D = 0x04d, diff --git a/llvm/include/llvm/TargetParser/TargetParser.h b/llvm/include/llvm/TargetParser/TargetParser.h index 8fc6fa37cb9a..e03d8f6eebfc 100644 --- a/llvm/include/llvm/TargetParser/TargetParser.h +++ b/llvm/include/llvm/TargetParser/TargetParser.h @@ -105,6 +105,7 @@ enum GPUKind : uint32_t { GK_GFX1103 = 93, GK_GFX1150 = 94, GK_GFX1151 = 95, + GK_GFX1152 = 96, GK_GFX1200 = 100, GK_GFX1201 = 101, diff --git a/llvm/lib/Object/ELFObjectFile.cpp b/llvm/lib/Object/ELFObjectFile.cpp index 2b6bdbf24afa..cbc55a145e0e 100644 --- a/llvm/lib/Object/ELFObjectFile.cpp +++ b/llvm/lib/Object/ELFObjectFile.cpp @@ -586,6 +586,8 @@ StringRef ELFObjectFileBase::getAMDGPUCPUName() const { return "gfx1150"; case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1151: return "gfx1151"; + case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1152: + return "gfx1152"; // AMDGCN GFX12. case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200: diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp index 8e2a9481c922..0fee299994bc 100644 --- a/llvm/lib/ObjectYAML/ELFYAML.cpp +++ b/llvm/lib/ObjectYAML/ELFYAML.cpp @@ -611,6 +611,7 @@ void ScalarBitSetTraits::bitset(IO &IO, BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1103, EF_AMDGPU_MACH); BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1150, EF_AMDGPU_MACH); BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1151, EF_AMDGPU_MACH); + BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1152, EF_AMDGPU_MACH); BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1200, EF_AMDGPU_MACH); BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1201, EF_AMDGPU_MACH); BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC, EF_AMDGPU_MACH); diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index 8e302786c746..d0d7a9dc1724 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -1534,6 +1534,12 @@ def FeatureISAVersion11_5_1 : FeatureSet< FeatureVGPRSingleUseHintInsts, Feature1_5xVGPRs])>; +def FeatureISAVersion11_5_2 : FeatureSet< + !listconcat(FeatureISAVersion11_Common.Features, + [FeatureSALUFloatInsts, + FeatureDPPSrc1SGPR, + FeatureVGPRSingleUseHintInsts])>; + def FeatureISAVersion12 : FeatureSet< [FeatureGFX12, FeatureLDSBankCount32, diff --git a/llvm/lib/Target/AMDGPU/GCNProcessors.td b/llvm/lib/Target/AMDGPU/GCNProcessors.td index 2ada981a77cd..d218ffeb1fec 100644 --- a/llvm/lib/Target/AMDGPU/GCNProcessors.td +++ b/llvm/lib/Target/AMDGPU/GCNProcessors.td @@ -295,7 +295,11 @@ def : ProcessorModel<"gfx1151", GFX11SpeedModel, FeatureISAVersion11_5_1.Features >; -// [gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151] +def : ProcessorModel<"gfx1152", GFX11SpeedModel, + FeatureISAVersion11_5_2.Features +>; + +// [gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1152] def : ProcessorModel<"gfx11-generic", GFX11SpeedModel, FeatureISAVersion11_Generic.Features >; diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp index d7d6e00d2389..e805e964ffe4 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp @@ -113,6 +113,7 @@ StringRef AMDGPUTargetStreamer::getArchNameFromElfMach(unsigned ElfMach) { case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1103: AK = GK_GFX1103; break; case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1150: AK = GK_GFX1150; break; case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1151: AK = GK_GFX1151; break; + case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1152: AK = GK_GFX1152; break; case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200: AK = GK_GFX1200; break; case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201: AK = GK_GFX1201; break; case ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC: AK = GK_GFX9_GENERIC; break; @@ -196,6 +197,7 @@ unsigned AMDGPUTargetStreamer::getElfMach(StringRef GPU) { case GK_GFX1103: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1103; case GK_GFX1150: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1150; case GK_GFX1151: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1151; + case GK_GFX1152: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1152; case GK_GFX1200: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200; case GK_GFX1201: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201; case GK_GFX9_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC; diff --git a/llvm/lib/TargetParser/TargetParser.cpp b/llvm/lib/TargetParser/TargetParser.cpp index 7464237d26d4..60a784ef002f 100644 --- a/llvm/lib/TargetParser/TargetParser.cpp +++ b/llvm/lib/TargetParser/TargetParser.cpp @@ -124,6 +124,7 @@ constexpr GPUInfo AMDGCNGPUs[] = { {{"gfx1103"}, {"gfx1103"}, GK_GFX1103, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP}, {{"gfx1150"}, {"gfx1150"}, GK_GFX1150, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP}, {{"gfx1151"}, {"gfx1151"}, GK_GFX1151, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP}, + {{"gfx1152"}, {"gfx1152"}, GK_GFX1152, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP}, {{"gfx1200"}, {"gfx1200"}, GK_GFX1200, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP}, {{"gfx1201"}, {"gfx1201"}, GK_GFX1201, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP}, @@ -275,6 +276,7 @@ AMDGPU::IsaVersion AMDGPU::getIsaVersion(StringRef GPU) { case GK_GFX1103: return {11, 0, 3}; case GK_GFX1150: return {11, 5, 0}; case GK_GFX1151: return {11, 5, 1}; + case GK_GFX1152: return {11, 5, 2}; case GK_GFX1200: return {12, 0, 0}; case GK_GFX1201: return {12, 0, 1}; @@ -341,6 +343,7 @@ void AMDGPU::fillAMDGPUFeatureMap(StringRef GPU, const Triple &T, Features["image-insts"] = true; Features["fp8-conversion-insts"] = true; break; + case GK_GFX1152: case GK_GFX1151: case GK_GFX1150: case GK_GFX1103: @@ -542,6 +545,7 @@ static bool isWave32Capable(StringRef GPU, const Triple &T) { switch (parseArchAMDGCN(GPU)) { case GK_GFX1201: case GK_GFX1200: + case GK_GFX1152: case GK_GFX1151: case GK_GFX1150: case GK_GFX1103: diff --git a/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll b/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll index c347437c3082..40d77a7f51e9 100644 --- a/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll +++ b/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll @@ -105,6 +105,7 @@ ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1103 < %s | FileCheck --check-prefixes=GFX1103 %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1150 < %s | FileCheck --check-prefixes=GFX1150 %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1151 < %s | FileCheck --check-prefixes=GFX1151 %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1152 < %s | FileCheck --check-prefixes=GFX1152 %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX1200 %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1201 < %s | FileCheck --check-prefixes=GFX1201 %s @@ -201,6 +202,7 @@ ; GFX1103: .amdgcn_target "amdgcn-amd-amdhsa--gfx1103" ; GFX1150: .amdgcn_target "amdgcn-amd-amdhsa--gfx1150" ; GFX1151: .amdgcn_target "amdgcn-amd-amdhsa--gfx1151" +; GFX1152: .amdgcn_target "amdgcn-amd-amdhsa--gfx1152" ; GFX1200: .amdgcn_target "amdgcn-amd-amdhsa--gfx1200" ; GFX1201: .amdgcn_target "amdgcn-amd-amdhsa--gfx1201" diff --git a/llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll b/llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll index edc20153ffd7..560a05abd5e7 100644 --- a/llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll +++ b/llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll @@ -74,6 +74,7 @@ ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1103 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1103 %s ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1150 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1150 %s ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1151 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1151 %s +; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1152 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1152 %s ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1200 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1200 %s ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1201 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1201 %s @@ -153,6 +154,7 @@ ; GFX1103: EF_AMDGPU_MACH_AMDGCN_GFX1103 (0x44) ; GFX1150: EF_AMDGPU_MACH_AMDGCN_GFX1150 (0x43) ; GFX1151: EF_AMDGPU_MACH_AMDGCN_GFX1151 (0x4A) +; GFX1152: EF_AMDGPU_MACH_AMDGCN_GFX1152 (0x55) ; GFX1200: EF_AMDGPU_MACH_AMDGCN_GFX1200 (0x48) ; GFX1201: EF_AMDGPU_MACH_AMDGCN_GFX1201 (0x4E) diff --git a/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll b/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll index d87eb9711488..e0ccda1cbb9e 100644 --- a/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll +++ b/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll @@ -13,6 +13,8 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx1150 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10W64,GFX1030,GFX1030W64 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1151 < %s | FileCheck --check-prefixes=GCN,GFX1100,GFX1100W32 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1151 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GCN,GFX1100,GFX1100W64 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1152 < %s | FileCheck --check-prefixes=GCN,GFX1030,GFX1030W32 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1152 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GCN,GFX1030,GFX1030W64 %s ; GCN-LABEL: {{^}}max_occupancy: ; GFX9: ; Occupancy: 10 diff --git a/llvm/test/MC/AMDGPU/gfx1150_asm_features.s b/llvm/test/MC/AMDGPU/gfx1150_asm_features.s index 58b784779ac7..d3f82a217a17 100644 --- a/llvm/test/MC/AMDGPU/gfx1150_asm_features.s +++ b/llvm/test/MC/AMDGPU/gfx1150_asm_features.s @@ -1,5 +1,6 @@ // RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1150 %s | FileCheck --check-prefix=GFX1150 %s // RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1151 %s | FileCheck --check-prefix=GFX1150 %s +// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1152 %s | FileCheck --check-prefix=GFX1150 %s // // Subtargets allow src1 of VOP3 DPP instructions to be SGPR or inlinable diff --git a/llvm/test/Object/AMDGPU/elf-header-flags-mach.yaml b/llvm/test/Object/AMDGPU/elf-header-flags-mach.yaml index 7f36795aa6a9..7512edd27491 100644 --- a/llvm/test/Object/AMDGPU/elf-header-flags-mach.yaml +++ b/llvm/test/Object/AMDGPU/elf-header-flags-mach.yaml @@ -230,6 +230,10 @@ # RUN: llvm-readobj -S --file-headers %t.o.AMDGCN_GFX1151 | FileCheck --check-prefixes=ELF-AMDGCN-ALL,ELF-AMDGCN-GFX1151 %s # RUN: obj2yaml %t.o.AMDGCN_GFX1151 | FileCheck --check-prefixes=YAML-AMDGCN-ALL,YAML-AMDGCN-GFX1151 %s +# RUN: sed -e 's//64/' -e 's//AMDGCN_GFX1152/' %s | yaml2obj -o %t.o.AMDGCN_GFX1152 +# RUN: llvm-readobj -S --file-headers %t.o.AMDGCN_GFX1152 | FileCheck --check-prefixes=ELF-AMDGCN-ALL,ELF-AMDGCN-GFX1152 %s +# RUN: obj2yaml %t.o.AMDGCN_GFX1152 | FileCheck --check-prefixes=YAML-AMDGCN-ALL,YAML-AMDGCN-GFX1152 %s + # RUN: sed -e 's//64/' -e 's//AMDGCN_GFX1200/' %s | yaml2obj -o %t.o.AMDGCN_GFX1200 # RUN: llvm-readobj -S --file-headers %t.o.AMDGCN_GFX1200 | FileCheck --check-prefixes=ELF-AMDGCN-ALL,ELF-AMDGCN-GFX1200 %s # RUN: obj2yaml %t.o.AMDGCN_GFX1200 | FileCheck --check-prefixes=YAML-AMDGCN-ALL,YAML-AMDGCN-GFX1200 %s @@ -450,6 +454,9 @@ # ELF-AMDGCN-GFX1151: EF_AMDGPU_MACH_AMDGCN_GFX1151 (0x4A) # YAML-AMDGCN-GFX1151: Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX1151 ] +# ELF-AMDGCN-GFX1152: EF_AMDGPU_MACH_AMDGCN_GFX1152 (0x55) +# YAML-AMDGCN-GFX1152: Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX1152 ] + # ELF-AMDGCN-GFX1200: EF_AMDGPU_MACH_AMDGCN_GFX1200 (0x48) # YAML-AMDGCN-GFX1200: Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX1200 ] diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/subtarget.ll b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/subtarget.ll index f1262c5a2fc2..f79f358e83a7 100644 --- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/subtarget.ll +++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/subtarget.ll @@ -28,6 +28,11 @@ define amdgpu_kernel void @test_kernel() { ; RUN: llvm-objdump -D -mllvm --amdhsa-code-object-version=6 %t.o > %t-detect.txt ; RUN: diff %t-specify.txt %t-detect.txt +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1152 -filetype=obj -O0 -o %t.o %s +; RUN: llvm-objdump -D --arch-name=amdgcn --mcpu=gfx1152 %t.o > %t-specify.txt +; RUN: llvm-objdump -D %t.o > %t-detect.txt +; RUN: diff %t-specify.txt %t-detect.txt + ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1151 -filetype=obj -O0 -o %t.o %s ; RUN: llvm-objdump -D --arch-name=amdgcn --mcpu=gfx1151 %t.o > %t-specify.txt ; RUN: llvm-objdump -D %t.o > %t-detect.txt diff --git a/llvm/test/tools/llvm-readobj/ELF/AMDGPU/elf-headers.test b/llvm/test/tools/llvm-readobj/ELF/AMDGPU/elf-headers.test index f9e90e21400e..50d437b50b69 100644 --- a/llvm/test/tools/llvm-readobj/ELF/AMDGPU/elf-headers.test +++ b/llvm/test/tools/llvm-readobj/ELF/AMDGPU/elf-headers.test @@ -421,6 +421,15 @@ # RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1151 # RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1151 -DFLAG_VALUE=0x4A +# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1152 +# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1152 -DFLAG_VALUE=0x55 + +# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1152 +# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1152 -DFLAG_VALUE=0x55 + +# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1152 +# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1152 -DFLAG_VALUE=0x55 + # RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1200 # RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1200 -DFLAG_VALUE=0x48 diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp index 40494da8b21b..c696934a959b 100644 --- a/llvm/tools/llvm-readobj/ELFDumper.cpp +++ b/llvm/tools/llvm-readobj/ELFDumper.cpp @@ -1631,6 +1631,7 @@ const EnumEntry ElfHeaderMipsFlags[] = { ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1103, "gfx1103"), \ ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1150, "gfx1150"), \ ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1151, "gfx1151"), \ + ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1152, "gfx1152"), \ ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1200, "gfx1200"), \ ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1201, "gfx1201"), \ ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC, "gfx9-generic"), \ diff --git a/offload/DeviceRTL/CMakeLists.txt b/offload/DeviceRTL/CMakeLists.txt index b35600367276..d88430a52b8b 100644 --- a/offload/DeviceRTL/CMakeLists.txt +++ b/offload/DeviceRTL/CMakeLists.txt @@ -46,7 +46,7 @@ set(all_amdgpu_architectures "gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906" "gfx908;gfx90a;gfx90c;gfx940;gfx941;gfx942;gfx1010" "gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035" "gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150" - "gfx1151") + "gfx1151;gfx1152") set(all_nvptx_architectures "sm_35;sm_37;sm_50;sm_52;sm_53;sm_60;sm_61;sm_62" "sm_70;sm_72;sm_75;sm_80;sm_86;sm_87;sm_89;sm_90") set(all_gpu_architectures -- GitLab From ab33fa59a2a5ae688426be8307584f78363326d0 Mon Sep 17 00:00:00 2001 From: Jordan Rupprecht Date: Thu, 6 Jun 2024 11:46:05 -0500 Subject: [PATCH 130/462] [bazel] Fix smoke/BUILD.bazel layering in nextafter_test_template (#94641) After 63cda2d19c310826722e8724649ceae7307389d7. See also a97871e07dd62510dea1bb71b0f74261f52e4479. --- .../llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel index eccea8faeebc..2ad2209925ce 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel @@ -136,7 +136,9 @@ libc_support_library( "//libc:__support_cpp_bit", "//libc:__support_cpp_type_traits", "//libc:__support_fputil_basic_operations", + "//libc:__support_fputil_fenv_impl", "//libc:__support_fputil_fp_bits", + "//libc:hdr_fenv_macros", "//libc:hdr_math_macros", "//libc/test/UnitTest:LibcUnitTest", "//libc/test/UnitTest:fp_test_helpers", -- GitLab From 212b78aad41b35df3af33bfffac678b460d467e9 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Thu, 6 Jun 2024 19:01:39 +0200 Subject: [PATCH 131/462] DAG: Improve fminimum/fmaximum vector expansion logic (#93579) First, expandFMINIMUM_FMAXIMUM should be a never-fail API. The client wanted it expanded, and it can always be expanded. This logic was tied up with what the VectorLegalizer wanted. Prefer using the min/max opcodes, and unrolling if we don't have a vselect. This seems to produce better code in all the changed tests. --- .../SelectionDAG/LegalizeVectorOps.cpp | 7 +- .../CodeGen/SelectionDAG/TargetLowering.cpp | 7 +- llvm/test/CodeGen/AMDGPU/fmaximum3.ll | 600 +- llvm/test/CodeGen/AMDGPU/fminimum3.ll | 600 +- llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll | 88 +- llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll | 3449 ++------- llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll | 6331 ++++------------- llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll | 88 +- llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll | 3449 ++------- llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll | 6331 ++++------------- .../X86/avx512fp16-fminimum-fmaximum.ll | 79 +- 11 files changed, 4758 insertions(+), 16271 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index 042684a434fd..8cdb4ba0ade6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -992,11 +992,8 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl &Results) { break; case ISD::FMINIMUM: case ISD::FMAXIMUM: - if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(Node, DAG)) { - Results.push_back(Expanded); - return; - } - break; + Results.push_back(TLI.expandFMINIMUM_FMAXIMUM(Node, DAG)); + return; case ISD::SMIN: case ISD::SMAX: case ISD::UMIN: diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index ee0bce344a01..e1c1a6b09b11 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -8427,10 +8427,6 @@ SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N, bool IsMax = Opc == ISD::FMAXIMUM; SDNodeFlags Flags = N->getFlags(); - if (VT.isVector() && - isOperationLegalOrCustomOrPromote(Opc, VT.getScalarType())) - return SDValue(); - // First, implement comparison not propagating NaN. If no native fmin or fmax // available, use plain select with setcc instead. SDValue MinMax; @@ -8447,6 +8443,9 @@ SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N, } else if (isOperationLegalOrCustom(CompOpc, VT)) { MinMax = DAG.getNode(CompOpc, DL, VT, LHS, RHS, Flags); } else { + if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) + return DAG.UnrollVectorOp(N); + // NaN (if exists) will be propagated later, so orderness doesn't matter. SDValue Compare = DAG.getSetCC(DL, CCVT, LHS, RHS, IsMax ? ISD::SETGT : ISD::SETLT); diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum3.ll b/llvm/test/CodeGen/AMDGPU/fmaximum3.ll index 3ec36f03a48a..9ce1ba3316dd 100644 --- a/llvm/test/CodeGen/AMDGPU/fmaximum3.ll +++ b/llvm/test/CodeGen/AMDGPU/fmaximum3.ll @@ -497,47 +497,19 @@ define <2 x float> @v_fmaximum3_v2f32(<2 x float> %a, <2 x float> %b, <2 x float ; GFX9-LABEL: v_fmaximum3_v2f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v1, vcc +; GFX9-NEXT: v_max_f32_e32 v6, v1, v3 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v0, v2 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v4, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v0, v4, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v3, vcc +; GFX9-NEXT: v_max_f32_e32 v2, v4, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v4, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v5, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v1, v5, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc +; GFX9-NEXT: v_max_f32_e32 v2, v5, v1 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v5, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v5, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %max0 = call <2 x float> @llvm.maximum.v2f32(<2 x float> %a, <2 x float> %b) %max1 = call <2 x float> @llvm.maximum.v2f32(<2 x float> %c, <2 x float> %max0) @@ -559,47 +531,19 @@ define <2 x float> @v_fmaximum3_v2f32_commute(<2 x float> %a, <2 x float> %b, <2 ; GFX9-LABEL: v_fmaximum3_v2f32_commute: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v1, vcc +; GFX9-NEXT: v_max_f32_e32 v6, v1, v3 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v0, v2 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v3, vcc +; GFX9-NEXT: v_max_f32_e32 v2, v0, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc +; GFX9-NEXT: v_max_f32_e32 v2, v1, v5 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %max0 = call <2 x float> @llvm.maximum.v2f32(<2 x float> %a, <2 x float> %b) %max1 = call <2 x float> @llvm.maximum.v2f32(<2 x float> %max0, <2 x float> %c) @@ -621,47 +565,19 @@ define <2 x float> @v_fmaximum3_v2f32__fabs_all(<2 x float> %a, <2 x float> %b, ; GFX9-LABEL: v_fmaximum3_v2f32__fabs_all: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e64 vcc, |v1|, |v3| -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v1, vcc +; GFX9-NEXT: v_max_f32_e64 v6, |v1|, |v3| ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], |v1|, |v3| -; GFX9-NEXT: v_cndmask_b32_e64 v6, v7, |v6|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v1|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v6, |v1|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v3|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, |v3|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e64 vcc, |v0|, |v2| -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v0, vcc -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], |v0|, |v2| -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, |v3|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v0|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v3, |v0|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v2|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, |v2|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e64 s[4:5], v0, |v4| -; GFX9-NEXT: v_cndmask_b32_e64 v2, |v4|, v0, s[4:5] +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, |v1|, |v3| +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc +; GFX9-NEXT: v_max_f32_e64 v3, |v0|, |v2| +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, |v0|, |v2| +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v3, vcc +; GFX9-NEXT: v_max_f32_e64 v2, v0, |v4| ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v0, |v4| -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v4|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, |v4|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cmp_gt_f32_e64 s[4:5], v1, |v5| -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, |v5|, v1, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc +; GFX9-NEXT: v_max_f32_e64 v2, v1, |v5| ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v1, |v5| -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v5|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, |v5|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %a.fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %a) %b.fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %b) @@ -686,47 +602,19 @@ define <2 x float> @v_fmaximum3_v2f32__fneg_all(<2 x float> %a, <2 x float> %b, ; GFX9-LABEL: v_fmaximum3_v2f32__fneg_all: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e64 vcc, -v1, -v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v1, vcc +; GFX9-NEXT: v_max_f32_e64 v6, -v1, -v3 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], -v1, -v3 -; GFX9-NEXT: v_cndmask_b32_e64 v6, v7, -v6, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v1, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v6, -v1, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v3, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, -v3, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e64 vcc, -v0, -v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v0, vcc -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], -v0, -v2 -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, -v3, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v0, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v3, -v0, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v2, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, -v2, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e64 s[4:5], v0, -v4 -; GFX9-NEXT: v_cndmask_b32_e64 v2, -v4, v0, s[4:5] +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, -v1, -v3 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc +; GFX9-NEXT: v_max_f32_e64 v3, -v0, -v2 +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, -v0, -v2 +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v3, vcc +; GFX9-NEXT: v_max_f32_e64 v2, v0, -v4 ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v0, -v4 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v4, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, -v4, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cmp_gt_f32_e64 s[4:5], v1, -v5 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, -v5, v1, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc +; GFX9-NEXT: v_max_f32_e64 v2, v1, -v5 ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v1, -v5 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v5, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, -v5, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %a.fneg = fneg <2 x float> %a %b.fneg = fneg <2 x float> %b @@ -751,35 +639,19 @@ define <2 x float> @v_fmaximum3_v2f32__inlineimm1(<2 x float> %a, <2 x float> %c ; GFX9-LABEL: v_fmaximum3_v2f32__inlineimm1: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, 2.0, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v4, 2.0, v1, vcc +; GFX9-NEXT: v_max_f32_e32 v4, 2.0, v1 ; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, 2.0, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v4, 2.0, v0, vcc +; GFX9-NEXT: v_max_f32_e32 v4, 2.0, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc +; GFX9-NEXT: v_max_f32_e32 v4, v0, v2 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc +; GFX9-NEXT: v_max_f32_e32 v2, v1, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %max0 = call <2 x float> @llvm.maximum.v2f32(<2 x float> %a, <2 x float> ) %max1 = call <2 x float> @llvm.maximum.v2f32(<2 x float> %max0, <2 x float> %c) @@ -801,33 +673,17 @@ define <2 x float> @v_fmaximum3_v2f32__inlineimm2(<2 x float> %a, <2 x float> %b ; GFX9-LABEL: v_fmaximum3_v2f32__inlineimm2: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v1, vcc +; GFX9-NEXT: v_max_f32_e32 v4, v1, v3 ; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v0, v2 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, 4.0, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v2, 4.0, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v3, vcc +; GFX9-NEXT: v_max_f32_e32 v2, 4.0, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, 4.0, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v2, 4.0, v1, vcc +; GFX9-NEXT: v_max_f32_e32 v2, 4.0, v1 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] @@ -852,67 +708,25 @@ define <3 x float> @v_fmaximum3_v3f32(<3 x float> %a, <3 x float> %b, <3 x float ; GFX9-LABEL: v_fmaximum3_v3f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v5, v2, vcc +; GFX9-NEXT: v_max_f32_e32 v9, v2, v5 ; GFX9-NEXT: v_mov_b32_e32 v10, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v9, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v9, vcc +; GFX9-NEXT: v_max_f32_e32 v5, v1, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v10, v5, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v5, vcc +; GFX9-NEXT: v_max_f32_e32 v4, v0, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v10, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v6, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v0, v6, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v4, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v6, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v6, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v7, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v7, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v3, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v7, v1 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v7, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v7, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v8, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v8, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v3, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v8, v2 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v8, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v8, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %max0 = call <3 x float> @llvm.maximum.v3f32(<3 x float> %a, <3 x float> %b) %max1 = call <3 x float> @llvm.maximum.v3f32(<3 x float> %c, <3 x float> %max0) @@ -935,67 +749,25 @@ define <3 x float> @v_fmaximum3_v3f32_commute(<3 x float> %a, <3 x float> %b, <3 ; GFX9-LABEL: v_fmaximum3_v3f32_commute: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v5, v2, vcc +; GFX9-NEXT: v_max_f32_e32 v9, v2, v5 ; GFX9-NEXT: v_mov_b32_e32 v10, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v9, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v9, vcc +; GFX9-NEXT: v_max_f32_e32 v5, v1, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v10, v5, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v5, vcc +; GFX9-NEXT: v_max_f32_e32 v4, v0, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v10, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v6, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v4, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v0, v6 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v3, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v1, v7 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v3, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v2, v8 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v8, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %max0 = call <3 x float> @llvm.maximum.v3f32(<3 x float> %a, <3 x float> %b) %max1 = call <3 x float> @llvm.maximum.v3f32(<3 x float> %max0, <3 x float> %c) @@ -1018,67 +790,25 @@ define <3 x float> @v_fmaximum3_v3f32__fabs_all(<3 x float> %a, <3 x float> %b, ; GFX9-LABEL: v_fmaximum3_v3f32__fabs_all: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e64 vcc, |v2|, |v5| -; GFX9-NEXT: v_cndmask_b32_e32 v9, v5, v2, vcc +; GFX9-NEXT: v_max_f32_e64 v9, |v2|, |v5| ; GFX9-NEXT: v_mov_b32_e32 v10, 0x7fc00000 -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], |v2|, |v5| -; GFX9-NEXT: v_cndmask_b32_e64 v9, v10, |v9|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v2|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v9, |v2|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v5|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, |v5|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX9-NEXT: v_cmp_gt_f32_e64 vcc, |v1|, |v4| -; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v1, vcc -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], |v1|, |v4| -; GFX9-NEXT: v_cndmask_b32_e64 v5, v10, |v5|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v1|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v5, |v1|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v4|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, |v4|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e64 vcc, |v0|, |v3| -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v0, vcc -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], |v0|, |v3| -; GFX9-NEXT: v_cndmask_b32_e64 v4, v10, |v4|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v0|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v4, |v0|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v3|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, |v3|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e64 s[4:5], v0, |v6| -; GFX9-NEXT: v_cndmask_b32_e64 v3, |v6|, v0, s[4:5] +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, |v2|, |v5| +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v9, vcc +; GFX9-NEXT: v_max_f32_e64 v5, |v1|, |v4| +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, |v1|, |v4| +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v5, vcc +; GFX9-NEXT: v_max_f32_e64 v4, |v0|, |v3| +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, |v0|, |v3| +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v4, vcc +; GFX9-NEXT: v_max_f32_e64 v3, v0, |v6| ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v0, |v6| -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v6|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, |v6|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cmp_gt_f32_e64 s[4:5], v1, |v7| -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v3, |v7|, v1, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v3, vcc +; GFX9-NEXT: v_max_f32_e64 v3, v1, |v7| ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v1, |v7| -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v7|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, |v7|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cmp_gt_f32_e64 s[4:5], v2, |v8| -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v3, |v8|, v2, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v3, vcc +; GFX9-NEXT: v_max_f32_e64 v3, v2, |v8| ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v2, |v8| -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v8|, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, |v8|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %a.fabs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %a) %b.fabs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %b) @@ -1104,67 +834,25 @@ define <3 x float> @v_fmaximum3_v3f32__fneg_all(<3 x float> %a, <3 x float> %b, ; GFX9-LABEL: v_fmaximum3_v3f32__fneg_all: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e64 vcc, -v2, -v5 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v5, v2, vcc +; GFX9-NEXT: v_max_f32_e64 v9, -v2, -v5 ; GFX9-NEXT: v_mov_b32_e32 v10, 0x7fc00000 -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], -v2, -v5 -; GFX9-NEXT: v_cndmask_b32_e64 v9, v10, -v9, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v2, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v9, -v2, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v5, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, -v5, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX9-NEXT: v_cmp_gt_f32_e64 vcc, -v1, -v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v1, vcc -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], -v1, -v4 -; GFX9-NEXT: v_cndmask_b32_e64 v5, v10, -v5, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v1, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v5, -v1, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v4, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, -v4, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e64 vcc, -v0, -v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v0, vcc -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], -v0, -v3 -; GFX9-NEXT: v_cndmask_b32_e64 v4, v10, -v4, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v0, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v4, -v0, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v3, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, -v3, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e64 s[4:5], v0, -v6 -; GFX9-NEXT: v_cndmask_b32_e64 v3, -v6, v0, s[4:5] +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, -v2, -v5 +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v9, vcc +; GFX9-NEXT: v_max_f32_e64 v5, -v1, -v4 +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, -v1, -v4 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v5, vcc +; GFX9-NEXT: v_max_f32_e64 v4, -v0, -v3 +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, -v0, -v3 +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v4, vcc +; GFX9-NEXT: v_max_f32_e64 v3, v0, -v6 ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v0, -v6 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v6, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, -v6, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cmp_gt_f32_e64 s[4:5], v1, -v7 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v3, -v7, v1, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v3, vcc +; GFX9-NEXT: v_max_f32_e64 v3, v1, -v7 ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v1, -v7 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v7, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, -v7, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cmp_gt_f32_e64 s[4:5], v2, -v8 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v3, -v8, v2, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v3, vcc +; GFX9-NEXT: v_max_f32_e64 v3, v2, -v8 ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v2, -v8 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v8, 64 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, -v8, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %a.fneg = fneg <3 x float> %a %b.fneg = fneg <3 x float> %b @@ -1190,49 +878,25 @@ define <3 x float> @v_fmaximum3_v3f32__inlineimm1(<3 x float> %a, <3 x float> %c ; GFX9-LABEL: v_fmaximum3_v3f32__inlineimm1: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, 2.0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v6, 2.0, v2, vcc +; GFX9-NEXT: v_max_f32_e32 v6, 2.0, v2 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v2 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, 2.0, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v6, 2.0, v1, vcc +; GFX9-NEXT: v_max_f32_e32 v6, 2.0, v1 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, 2.0, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v6, 2.0, v0, vcc +; GFX9-NEXT: v_max_f32_e32 v6, 2.0, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc +; GFX9-NEXT: v_max_f32_e32 v6, v0, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v1, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v2, v5 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %max0 = call <3 x float> @llvm.maximum.v3f32(<3 x float> %a, <3 x float> ) %max1 = call <3 x float> @llvm.maximum.v3f32(<3 x float> %max0, <3 x float> %c) @@ -1255,47 +919,23 @@ define <3 x float> @v_fmaximum3_v3f32__inlineimm2(<3 x float> %a, <3 x float> %b ; GFX9-LABEL: v_fmaximum3_v3f32__inlineimm2: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc +; GFX9-NEXT: v_max_f32_e32 v6, v2, v5 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc +; GFX9-NEXT: v_max_f32_e32 v5, v1, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v5, vcc +; GFX9-NEXT: v_max_f32_e32 v4, v0, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, 4.0, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v3, 4.0, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v4, vcc +; GFX9-NEXT: v_max_f32_e32 v3, 4.0, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v3, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, 4.0, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v3, 4.0, v1, vcc +; GFX9-NEXT: v_max_f32_e32 v3, 4.0, v1 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, 4.0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, 4.0, v2, vcc +; GFX9-NEXT: v_max_f32_e32 v3, 4.0, v2 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v2 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] diff --git a/llvm/test/CodeGen/AMDGPU/fminimum3.ll b/llvm/test/CodeGen/AMDGPU/fminimum3.ll index 0e0b73b88d2d..21074d58bdb7 100644 --- a/llvm/test/CodeGen/AMDGPU/fminimum3.ll +++ b/llvm/test/CodeGen/AMDGPU/fminimum3.ll @@ -497,47 +497,19 @@ define <2 x float> @v_fminimum3_v2f32(<2 x float> %a, <2 x float> %b, <2 x float ; GFX9-LABEL: v_fminimum3_v2f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v1, vcc +; GFX9-NEXT: v_min_f32_e32 v6, v1, v3 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v0, v2 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v4, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v0, v4, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v3, vcc +; GFX9-NEXT: v_min_f32_e32 v2, v4, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v4, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v5, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v1, v5, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc +; GFX9-NEXT: v_min_f32_e32 v2, v5, v1 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v5, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v5, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %max0 = call <2 x float> @llvm.minimum.v2f32(<2 x float> %a, <2 x float> %b) %max1 = call <2 x float> @llvm.minimum.v2f32(<2 x float> %c, <2 x float> %max0) @@ -559,47 +531,19 @@ define <2 x float> @v_fminimum3_v2f32_commute(<2 x float> %a, <2 x float> %b, <2 ; GFX9-LABEL: v_fminimum3_v2f32_commute: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v1, vcc +; GFX9-NEXT: v_min_f32_e32 v6, v1, v3 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v0, v2 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v3, vcc +; GFX9-NEXT: v_min_f32_e32 v2, v0, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc +; GFX9-NEXT: v_min_f32_e32 v2, v1, v5 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %max0 = call <2 x float> @llvm.minimum.v2f32(<2 x float> %a, <2 x float> %b) %max1 = call <2 x float> @llvm.minimum.v2f32(<2 x float> %max0, <2 x float> %c) @@ -621,47 +565,19 @@ define <2 x float> @v_fminimum3_v2f32__fabs_all(<2 x float> %a, <2 x float> %b, ; GFX9-LABEL: v_fminimum3_v2f32__fabs_all: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, |v3| -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v1, vcc +; GFX9-NEXT: v_min_f32_e64 v6, |v1|, |v3| ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], |v1|, |v3| -; GFX9-NEXT: v_cndmask_b32_e64 v6, v7, |v6|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v1|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v6, |v1|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v3|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, |v3|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, |v2| -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v0, vcc -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], |v0|, |v2| -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, |v3|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v0|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v3, |v0|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v2|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, |v2|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e64 s[4:5], v0, |v4| -; GFX9-NEXT: v_cndmask_b32_e64 v2, |v4|, v0, s[4:5] +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, |v1|, |v3| +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc +; GFX9-NEXT: v_min_f32_e64 v3, |v0|, |v2| +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, |v0|, |v2| +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v3, vcc +; GFX9-NEXT: v_min_f32_e64 v2, v0, |v4| ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v0, |v4| -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v4|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, |v4|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cmp_lt_f32_e64 s[4:5], v1, |v5| -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, |v5|, v1, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc +; GFX9-NEXT: v_min_f32_e64 v2, v1, |v5| ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v1, |v5| -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v5|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, |v5|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %a.fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %a) %b.fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %b) @@ -686,47 +602,19 @@ define <2 x float> @v_fminimum3_v2f32__fneg_all(<2 x float> %a, <2 x float> %b, ; GFX9-LABEL: v_fminimum3_v2f32__fneg_all: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e64 vcc, -v1, -v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v1, vcc +; GFX9-NEXT: v_min_f32_e64 v6, -v1, -v3 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], -v1, -v3 -; GFX9-NEXT: v_cndmask_b32_e64 v6, v7, -v6, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v1, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v6, -v1, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v3, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, -v3, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e64 vcc, -v0, -v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v0, vcc -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], -v0, -v2 -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, -v3, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v0, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v3, -v0, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v2, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, -v2, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e64 s[4:5], v0, -v4 -; GFX9-NEXT: v_cndmask_b32_e64 v2, -v4, v0, s[4:5] +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, -v1, -v3 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc +; GFX9-NEXT: v_min_f32_e64 v3, -v0, -v2 +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, -v0, -v2 +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v3, vcc +; GFX9-NEXT: v_min_f32_e64 v2, v0, -v4 ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v0, -v4 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v4, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, -v4, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cmp_lt_f32_e64 s[4:5], v1, -v5 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, -v5, v1, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc +; GFX9-NEXT: v_min_f32_e64 v2, v1, -v5 ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v1, -v5 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v5, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, -v5, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %a.fneg = fneg <2 x float> %a %b.fneg = fneg <2 x float> %b @@ -751,35 +639,19 @@ define <2 x float> @v_fminimum3_v2f32__inlineimm1(<2 x float> %a, <2 x float> %c ; GFX9-LABEL: v_fminimum3_v2f32__inlineimm1: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, 2.0, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v4, 2.0, v1, vcc +; GFX9-NEXT: v_min_f32_e32 v4, 2.0, v1 ; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, 2.0, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v4, 2.0, v0, vcc +; GFX9-NEXT: v_min_f32_e32 v4, 2.0, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc +; GFX9-NEXT: v_min_f32_e32 v4, v0, v2 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc +; GFX9-NEXT: v_min_f32_e32 v2, v1, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %max0 = call <2 x float> @llvm.minimum.v2f32(<2 x float> %a, <2 x float> ) %max1 = call <2 x float> @llvm.minimum.v2f32(<2 x float> %max0, <2 x float> %c) @@ -801,33 +673,17 @@ define <2 x float> @v_fminimum3_v2f32__inlineimm2(<2 x float> %a, <2 x float> %b ; GFX9-LABEL: v_fminimum3_v2f32__inlineimm2: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v1, vcc +; GFX9-NEXT: v_min_f32_e32 v4, v1, v3 ; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v0, v2 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, 4.0, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v2, 4.0, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v3, vcc +; GFX9-NEXT: v_min_f32_e32 v2, 4.0, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, 4.0, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v2, 4.0, v1, vcc +; GFX9-NEXT: v_min_f32_e32 v2, 4.0, v1 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] @@ -852,67 +708,25 @@ define <3 x float> @v_fminimum3_v3f32(<3 x float> %a, <3 x float> %b, <3 x float ; GFX9-LABEL: v_fminimum3_v3f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v5, v2, vcc +; GFX9-NEXT: v_min_f32_e32 v9, v2, v5 ; GFX9-NEXT: v_mov_b32_e32 v10, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v9, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v9, vcc +; GFX9-NEXT: v_min_f32_e32 v5, v1, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v10, v5, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v5, vcc +; GFX9-NEXT: v_min_f32_e32 v4, v0, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v10, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v6, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v0, v6, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v4, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v6, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v6, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v7, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v7, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v3, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v7, v1 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v7, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v7, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v8, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v8, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v3, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v8, v2 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v8, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v8, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %max0 = call <3 x float> @llvm.minimum.v3f32(<3 x float> %a, <3 x float> %b) %max1 = call <3 x float> @llvm.minimum.v3f32(<3 x float> %c, <3 x float> %max0) @@ -935,67 +749,25 @@ define <3 x float> @v_fminimum3_v3f32_commute(<3 x float> %a, <3 x float> %b, <3 ; GFX9-LABEL: v_fminimum3_v3f32_commute: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v5, v2, vcc +; GFX9-NEXT: v_min_f32_e32 v9, v2, v5 ; GFX9-NEXT: v_mov_b32_e32 v10, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v9, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v9, vcc +; GFX9-NEXT: v_min_f32_e32 v5, v1, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v10, v5, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v5, vcc +; GFX9-NEXT: v_min_f32_e32 v4, v0, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v10, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v6, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v4, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v0, v6 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v3, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v1, v7 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v3, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v2, v8 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v8, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %max0 = call <3 x float> @llvm.minimum.v3f32(<3 x float> %a, <3 x float> %b) %max1 = call <3 x float> @llvm.minimum.v3f32(<3 x float> %max0, <3 x float> %c) @@ -1018,67 +790,25 @@ define <3 x float> @v_fminimum3_v3f32__fabs_all(<3 x float> %a, <3 x float> %b, ; GFX9-LABEL: v_fminimum3_v3f32__fabs_all: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e64 vcc, |v2|, |v5| -; GFX9-NEXT: v_cndmask_b32_e32 v9, v5, v2, vcc +; GFX9-NEXT: v_min_f32_e64 v9, |v2|, |v5| ; GFX9-NEXT: v_mov_b32_e32 v10, 0x7fc00000 -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], |v2|, |v5| -; GFX9-NEXT: v_cndmask_b32_e64 v9, v10, |v9|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v2|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v9, |v2|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v5|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, |v5|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX9-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, |v4| -; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v1, vcc -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], |v1|, |v4| -; GFX9-NEXT: v_cndmask_b32_e64 v5, v10, |v5|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v1|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v5, |v1|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v4|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, |v4|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, |v3| -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v0, vcc -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], |v0|, |v3| -; GFX9-NEXT: v_cndmask_b32_e64 v4, v10, |v4|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v0|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v4, |v0|, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v3|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, |v3|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e64 s[4:5], v0, |v6| -; GFX9-NEXT: v_cndmask_b32_e64 v3, |v6|, v0, s[4:5] +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, |v2|, |v5| +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v9, vcc +; GFX9-NEXT: v_min_f32_e64 v5, |v1|, |v4| +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, |v1|, |v4| +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v5, vcc +; GFX9-NEXT: v_min_f32_e64 v4, |v0|, |v3| +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, |v0|, |v3| +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v4, vcc +; GFX9-NEXT: v_min_f32_e64 v3, v0, |v6| ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v0, |v6| -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v6|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, |v6|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cmp_lt_f32_e64 s[4:5], v1, |v7| -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v3, |v7|, v1, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v3, vcc +; GFX9-NEXT: v_min_f32_e64 v3, v1, |v7| ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v1, |v7| -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v7|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, |v7|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cmp_lt_f32_e64 s[4:5], v2, |v8| -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v3, |v8|, v2, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v3, vcc +; GFX9-NEXT: v_min_f32_e64 v3, v2, |v8| ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v2, |v8| -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], |v8|, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, |v8|, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %a.fabs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %a) %b.fabs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %b) @@ -1104,67 +834,25 @@ define <3 x float> @v_fminimum3_v3f32__fneg_all(<3 x float> %a, <3 x float> %b, ; GFX9-LABEL: v_fminimum3_v3f32__fneg_all: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e64 vcc, -v2, -v5 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v5, v2, vcc +; GFX9-NEXT: v_min_f32_e64 v9, -v2, -v5 ; GFX9-NEXT: v_mov_b32_e32 v10, 0x7fc00000 -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], -v2, -v5 -; GFX9-NEXT: v_cndmask_b32_e64 v9, v10, -v9, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v2, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v9, -v2, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v5, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, -v5, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX9-NEXT: v_cmp_lt_f32_e64 vcc, -v1, -v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v1, vcc -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], -v1, -v4 -; GFX9-NEXT: v_cndmask_b32_e64 v5, v10, -v5, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v1, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v5, -v1, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v4, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, -v4, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e64 vcc, -v0, -v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v0, vcc -; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], -v0, -v3 -; GFX9-NEXT: v_cndmask_b32_e64 v4, v10, -v4, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v0, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v4, -v0, s[4:5] -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v3, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, -v3, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e64 s[4:5], v0, -v6 -; GFX9-NEXT: v_cndmask_b32_e64 v3, -v6, v0, s[4:5] +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, -v2, -v5 +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v9, vcc +; GFX9-NEXT: v_min_f32_e64 v5, -v1, -v4 +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, -v1, -v4 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v5, vcc +; GFX9-NEXT: v_min_f32_e64 v4, -v0, -v3 +; GFX9-NEXT: v_cmp_o_f32_e64 vcc, -v0, -v3 +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v4, vcc +; GFX9-NEXT: v_min_f32_e64 v3, v0, -v6 ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v0, -v6 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v6, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, -v6, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cmp_lt_f32_e64 s[4:5], v1, -v7 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v3, -v7, v1, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v3, vcc +; GFX9-NEXT: v_min_f32_e64 v3, v1, -v7 ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v1, -v7 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v7, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, -v7, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cmp_lt_f32_e64 s[4:5], v2, -v8 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v3, -v8, v2, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v3, vcc +; GFX9-NEXT: v_min_f32_e64 v3, v2, -v8 ; GFX9-NEXT: v_cmp_o_f32_e64 vcc, v2, -v8 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 s[4:5], -v8, 32 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, -v8, s[4:5] -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %a.fneg = fneg <3 x float> %a %b.fneg = fneg <3 x float> %b @@ -1190,49 +878,25 @@ define <3 x float> @v_fminimum3_v3f32__inlineimm1(<3 x float> %a, <3 x float> %c ; GFX9-LABEL: v_fminimum3_v3f32__inlineimm1: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, 2.0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v6, 2.0, v2, vcc +; GFX9-NEXT: v_min_f32_e32 v6, 2.0, v2 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v2 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, 2.0, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v6, 2.0, v1, vcc +; GFX9-NEXT: v_min_f32_e32 v6, 2.0, v1 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, 2.0, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v6, 2.0, v0, vcc +; GFX9-NEXT: v_min_f32_e32 v6, 2.0, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc +; GFX9-NEXT: v_min_f32_e32 v6, v0, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v1, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v2, v5 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] %max0 = call <3 x float> @llvm.minimum.v3f32(<3 x float> %a, <3 x float> ) %max1 = call <3 x float> @llvm.minimum.v3f32(<3 x float> %max0, <3 x float> %c) @@ -1255,47 +919,23 @@ define <3 x float> @v_fminimum3_v3f32__inlineimm2(<3 x float> %a, <3 x float> %b ; GFX9-LABEL: v_fminimum3_v3f32__inlineimm2: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc +; GFX9-NEXT: v_min_f32_e32 v6, v2, v5 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc +; GFX9-NEXT: v_min_f32_e32 v5, v1, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v3, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v5, vcc +; GFX9-NEXT: v_min_f32_e32 v4, v0, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, 4.0, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v3, 4.0, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v4, vcc +; GFX9-NEXT: v_min_f32_e32 v3, 4.0, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v3, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, 4.0, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v3, 4.0, v1, vcc +; GFX9-NEXT: v_min_f32_e32 v3, 4.0, v1 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, 4.0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, 4.0, v2, vcc +; GFX9-NEXT: v_min_f32_e32 v3, 4.0, v2 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v2 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll index 0c13c5348879..fa7ee9e8d28f 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll @@ -554,28 +554,14 @@ define <2 x half> @v_maximum_v2f16(<2 x half> %src0, <2 x half> %src1) { ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX8-NEXT: v_cmp_gt_f16_e32 vcc, v3, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v3, vcc +; GFX8-NEXT: v_max_f16_e32 v4, v3, v2 ; GFX8-NEXT: v_mov_b32_e32 v5, 0x7e00 ; GFX8-NEXT: v_cmp_o_f16_e32 vcc, v3, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v3, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v2, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_eq_f16_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_cmp_gt_f16_e32 vcc, v0, v1 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v1, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc +; GFX8-NEXT: v_max_f16_e32 v3, v0, v1 ; GFX8-NEXT: v_cmp_o_f16_e32 vcc, v0, v1 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v0, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v1, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GFX8-NEXT: v_cmp_eq_f16_e32 vcc, 0, v3 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v3, vcc ; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: s_setpc_b64 s[30:31] ; @@ -669,26 +655,9 @@ define <2 x half> @v_maximum_v2f16__nnan(<2 x half> %src0, <2 x half> %src1) { ; GFX8-LABEL: v_maximum_v2f16__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX8-NEXT: v_cmp_gt_f16_e32 vcc, v3, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v3, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v3, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v2, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_eq_f16_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_cmp_gt_f16_e32 vcc, v0, v1 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v1, v0, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v0, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v1, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GFX8-NEXT: v_cmp_eq_f16_e32 vcc, 0, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_max_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 +; GFX8-NEXT: v_max_f16_e32 v0, v0, v1 +; GFX8-NEXT: v_or_b32_e32 v0, v0, v2 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v2f16__nnan: @@ -754,13 +723,11 @@ define <2 x half> @v_maximum_v2f16__nsz(<2 x half> %src0, <2 x half> %src1) { ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX8-NEXT: v_cmp_gt_f16_e32 vcc, v3, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v3, vcc +; GFX8-NEXT: v_max_f16_e32 v4, v3, v2 ; GFX8-NEXT: v_mov_b32_e32 v5, 0x7e00 ; GFX8-NEXT: v_cmp_o_f16_e32 vcc, v3, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc -; GFX8-NEXT: v_cmp_gt_f16_e32 vcc, v0, v1 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v1, v0, vcc +; GFX8-NEXT: v_max_f16_e32 v3, v0, v1 ; GFX8-NEXT: v_cmp_o_f16_e32 vcc, v0, v1 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v3, vcc @@ -857,14 +824,9 @@ define <2 x half> @v_maximum_v2f16__nnan_nsz(<2 x half> %src0, <2 x half> %src1) ; GFX8-LABEL: v_maximum_v2f16__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX8-NEXT: v_cmp_gt_f16_e32 vcc, v3, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc -; GFX8-NEXT: v_cmp_gt_f16_e32 vcc, v0, v1 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_max_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 +; GFX8-NEXT: v_max_f16_e32 v0, v0, v1 +; GFX8-NEXT: v_or_b32_e32 v0, v0, v2 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v2f16__nnan_nsz: @@ -938,31 +900,15 @@ define void @s_maximum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) { ; GFX8-NEXT: s_lshr_b32 s6, s5, 16 ; GFX8-NEXT: s_lshr_b32 s7, s4, 16 ; GFX8-NEXT: v_mov_b32_e32 v0, s6 -; GFX8-NEXT: v_mov_b32_e32 v1, s7 -; GFX8-NEXT: v_cmp_gt_f16_e32 vcc, s7, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc -; GFX8-NEXT: v_mov_b32_e32 v3, 0x7e00 +; GFX8-NEXT: v_max_f16_e32 v1, s7, v0 +; GFX8-NEXT: v_mov_b32_e32 v2, 0x7e00 ; GFX8-NEXT: v_cmp_o_f16_e32 vcc, s7, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, s7, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, s6, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc -; GFX8-NEXT: v_cmp_eq_f16_e32 vcc, 0, v2 +; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc ; GFX8-NEXT: v_mov_b32_e32 v1, s5 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX8-NEXT: v_mov_b32_e32 v2, s4 -; GFX8-NEXT: v_cmp_gt_f16_e32 vcc, s4, v1 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc +; GFX8-NEXT: v_max_f16_e32 v3, s4, v1 ; GFX8-NEXT: v_cmp_o_f16_e32 vcc, s4, v1 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, s4, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, s5, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX8-NEXT: v_cmp_eq_f16_e32 vcc, 0, v3 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc ; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: ;;#ASMSTART ; GFX8-NEXT: ; use v0 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll index 7c5bc7da4df2..f4aa40dbd9bc 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll @@ -495,167 +495,73 @@ define <2 x float> @v_maximum_v2f32(<2 x float> %src0, <2 x float> %src1) { ; GFX7-LABEL: v_maximum_v2f32: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v4, v0, v2 +; GFX7-NEXT: v_max_f32_e32 v4, v0, v2 ; GFX7-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v2, v1, v3 +; GFX7-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc +; GFX7-NEXT: v_max_f32_e32 v2, v1, v3 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v2f32: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc +; GFX8-NEXT: v_max_f32_e32 v4, v0, v2 ; GFX8-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc +; GFX8-NEXT: v_max_f32_e32 v2, v1, v3 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v2f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc +; GFX9-NEXT: v_max_f32_e32 v4, v0, v2 ; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc +; GFX9-NEXT: v_max_f32_e32 v2, v1, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v2f32: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 +; GFX940-NEXT: v_max_f32_e32 v4, v0, v2 ; GFX940-NEXT: v_mov_b32_e32 v5, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc +; GFX940-NEXT: v_max_f32_e32 v2, v1, v3 +; GFX940-NEXT: s_nop 0 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v2f32: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v4, v0, v2 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v5, v1, v3 +; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v5, 0x7fc00000, v5, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v2f32: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v4, v0, v2 :: v_dual_max_f32 v5, v1, v3 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v5, 0x7fc00000, v5, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v2f32: @@ -676,136 +582,42 @@ define <2 x float> @v_maximum_v2f32__nnan(<2 x float> %src0, <2 x float> %src1) ; GFX7-LABEL: v_maximum_v2f32__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v4, v0, v2 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v2, v1, v3 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX7-NEXT: v_max_f32_e32 v0, v0, v2 +; GFX7-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v2f32__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX8-NEXT: v_max_f32_e32 v0, v0, v2 +; GFX8-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v2f32__nnan: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_max_f32_e32 v0, v0, v2 +; GFX9-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v2f32__nnan: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX940-NEXT: v_max_f32_e32 v0, v0, v2 +; GFX940-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v2f32__nnan: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v0, v0, v2 +; GFX10-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v2f32__nnan: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v0, v0, v2 :: v_dual_max_f32 v1, v1, v3 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v2f32__nnan: @@ -826,11 +638,11 @@ define <2 x float> @v_maximum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) { ; GFX7-LABEL: v_maximum_v2f32__nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v4, v0, v2 +; GFX7-NEXT: v_max_f32_e32 v4, v0, v2 ; GFX7-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v2, v1, v3 +; GFX7-NEXT: v_max_f32_e32 v2, v1, v3 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] @@ -838,13 +650,11 @@ define <2 x float> @v_maximum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) { ; GFX8-LABEL: v_maximum_v2f32__nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc +; GFX8-NEXT: v_max_f32_e32 v4, v0, v2 ; GFX8-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc +; GFX8-NEXT: v_max_f32_e32 v2, v1, v3 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] @@ -852,13 +662,11 @@ define <2 x float> @v_maximum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) { ; GFX9-LABEL: v_maximum_v2f32__nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc +; GFX9-NEXT: v_max_f32_e32 v4, v0, v2 ; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc +; GFX9-NEXT: v_max_f32_e32 v2, v1, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] @@ -866,16 +674,12 @@ define <2 x float> @v_maximum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) { ; GFX940-LABEL: v_maximum_v2f32__nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 +; GFX940-NEXT: v_max_f32_e32 v4, v0, v2 ; GFX940-NEXT: v_mov_b32_e32 v5, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_max_f32_e32 v2, v1, v3 +; GFX940-NEXT: s_nop 0 ; GFX940-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 ; GFX940-NEXT: s_nop 1 ; GFX940-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc @@ -884,11 +688,9 @@ define <2 x float> @v_maximum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) { ; GFX10-LABEL: v_maximum_v2f32__nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v4, v0, v2 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v2 +; GFX10-NEXT: v_max_f32_e32 v5, v1, v3 ; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v3 ; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo @@ -897,12 +699,9 @@ define <2 x float> @v_maximum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) { ; GFX11-LABEL: v_maximum_v2f32__nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v4, v0, v2 :: v_dual_max_f32 v5, v1, v3 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v3 ; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo @@ -926,55 +725,42 @@ define <2 x float> @v_maximum_v2f32__nnan_nsz(<2 x float> %src0, <2 x float> %sr ; GFX7-LABEL: v_maximum_v2f32__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v0, v0, v2 -; GFX7-NEXT: v_max_legacy_f32_e32 v1, v1, v3 +; GFX7-NEXT: v_max_f32_e32 v0, v0, v2 +; GFX7-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v2f32__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX8-NEXT: v_max_f32_e32 v0, v0, v2 +; GFX8-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v2f32__nnan_nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX9-NEXT: v_max_f32_e32 v0, v0, v2 +; GFX9-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v2f32__nnan_nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v2 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX940-NEXT: v_max_f32_e32 v0, v0, v2 +; GFX940-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v2f32__nnan_nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v0, v0, v2 +; GFX10-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v2f32__nnan_nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v0, v0, v2 :: v_dual_max_f32 v1, v1, v3 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v2f32__nnan_nsz: @@ -996,28 +782,14 @@ define void @s_maximum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) { ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_mov_b32_e32 v0, s7 -; GFX7-NEXT: v_max_legacy_f32_e32 v1, s5, v0 +; GFX7-NEXT: v_max_f32_e32 v1, s5, v0 ; GFX7-NEXT: v_mov_b32_e32 v2, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, s5, v0 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX7-NEXT: v_mov_b32_e32 v3, s5 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, s5, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, s7, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v1 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc ; GFX7-NEXT: v_mov_b32_e32 v0, s6 -; GFX7-NEXT: v_max_legacy_f32_e32 v3, s4, v0 +; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX7-NEXT: v_max_f32_e32 v3, s4, v0 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, s4, v0 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc -; GFX7-NEXT: v_mov_b32_e32 v3, s4 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, s4, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v2, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, s6, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ; GFX7-NEXT: ;;#ASMSTART ; GFX7-NEXT: ; use v[0:1] ; GFX7-NEXT: ;;#ASMEND @@ -1027,30 +799,14 @@ define void @s_maximum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) { ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v0, s7 -; GFX8-NEXT: v_mov_b32_e32 v1, s5 -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, s5, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc -; GFX8-NEXT: v_mov_b32_e32 v3, 0x7fc00000 +; GFX8-NEXT: v_max_f32_e32 v1, s5, v0 +; GFX8-NEXT: v_mov_b32_e32 v2, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, s5, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, s5, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, s7, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v0, vcc ; GFX8-NEXT: v_mov_b32_e32 v0, s6 -; GFX8-NEXT: v_mov_b32_e32 v2, s4 -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, s4, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX8-NEXT: v_max_f32_e32 v3, s4, v0 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, s4, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, s4, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, s6, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ; GFX8-NEXT: ;;#ASMSTART ; GFX8-NEXT: ; use v[0:1] ; GFX8-NEXT: ;;#ASMEND @@ -1060,30 +816,14 @@ define void @s_maximum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) { ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v0, s7 -; GFX9-NEXT: v_mov_b32_e32 v1, s5 -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, s5, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc -; GFX9-NEXT: v_mov_b32_e32 v3, 0x7fc00000 +; GFX9-NEXT: v_max_f32_e32 v1, s5, v0 +; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, s5, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, s5, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, s7, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v0, vcc ; GFX9-NEXT: v_mov_b32_e32 v0, s6 -; GFX9-NEXT: v_mov_b32_e32 v2, s4 -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, s4, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_max_f32_e32 v3, s4, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, s4, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, s4, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, s6, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ; GFX9-NEXT: ;;#ASMSTART ; GFX9-NEXT: ; use v[0:1] ; GFX9-NEXT: ;;#ASMEND @@ -1093,40 +833,15 @@ define void @s_maximum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) { ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v0, s3 -; GFX940-NEXT: v_mov_b32_e32 v1, s1 -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, s1, v0 -; GFX940-NEXT: v_mov_b32_e32 v3, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc +; GFX940-NEXT: v_max_f32_e32 v1, s1, v0 +; GFX940-NEXT: v_mov_b32_e32 v2, 0x7fc00000 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, s1, v0 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, s1, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, s3, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v0, vcc ; GFX940-NEXT: v_mov_b32_e32 v0, s2 -; GFX940-NEXT: v_mov_b32_e32 v2, s0 -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, s0, v0 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc +; GFX940-NEXT: v_max_f32_e32 v3, s0, v0 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, s0, v0 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, s0, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, s2, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ; GFX940-NEXT: ;;#ASMSTART ; GFX940-NEXT: ; use v[0:1] ; GFX940-NEXT: ;;#ASMEND @@ -1135,28 +850,12 @@ define void @s_maximum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) { ; GFX10-LABEL: s_maximum_v2f32: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v0, s5 -; GFX10-NEXT: v_cmp_gt_f32_e64 vcc_lo, s5, s7 -; GFX10-NEXT: v_mov_b32_e32 v1, s4 -; GFX10-NEXT: v_cmp_class_f32_e64 s8, s5, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, s7, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e64 vcc_lo, s4, s6 -; GFX10-NEXT: v_cndmask_b32_e32 v1, s6, v1, vcc_lo +; GFX10-NEXT: v_max_f32_e64 v0, s5, s7 ; GFX10-NEXT: v_cmp_o_f32_e64 vcc_lo, s5, s7 -; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo +; GFX10-NEXT: v_max_f32_e64 v2, s4, s6 +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v0, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e64 vcc_lo, s4, s6 -; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v1, v0, s5, s8 -; GFX10-NEXT: v_cmp_class_f32_e64 s5, s4, 64 -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v2, s4, s5 -; GFX10-NEXT: v_cmp_class_f32_e64 s4, s7, 64 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s7, s4 -; GFX10-NEXT: v_cmp_class_f32_e64 s4, s6, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s6, s4 -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo ; GFX10-NEXT: ;;#ASMSTART ; GFX10-NEXT: ; use v[0:1] ; GFX10-NEXT: ;;#ASMEND @@ -1165,32 +864,13 @@ define void @s_maximum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) { ; GFX11-LABEL: s_maximum_v2f32: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0 -; GFX11-NEXT: v_cmp_gt_f32_e64 vcc_lo, s1, s3 -; GFX11-NEXT: v_cmp_class_f32_e64 s4, s1, 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, s3, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e64 vcc_lo, s0, s2 -; GFX11-NEXT: v_cndmask_b32_e32 v1, s2, v1, vcc_lo +; GFX11-NEXT: v_max_f32_e64 v0, s1, s3 ; GFX11-NEXT: v_cmp_o_f32_e64 vcc_lo, s1, s3 -; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo +; GFX11-NEXT: v_max_f32_e64 v2, s0, s2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v0, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e64 vcc_lo, s0, s2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v1, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v1, v0, s1, s4 -; GFX11-NEXT: v_cmp_class_f32_e64 s1, s0, 64 -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_cndmask_b32_e64 v3, v2, s0, s1 -; GFX11-NEXT: v_cmp_class_f32_e64 s0, s3, 64 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s3, s0 -; GFX11-NEXT: v_cmp_class_f32_e64 s0, s2, 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s2, s0 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo ; GFX11-NEXT: ;;#ASMSTART ; GFX11-NEXT: ; use v[0:1] ; GFX11-NEXT: ;;#ASMEND @@ -1218,227 +898,92 @@ define <3 x float> @v_maximum_v3f32(<3 x float> %src0, <3 x float> %src1) { ; GFX7-LABEL: v_maximum_v3f32: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v6, v0, v3 +; GFX7-NEXT: v_max_f32_e32 v6, v0, v3 ; GFX7-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v3, v1, v4 +; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc +; GFX7-NEXT: v_max_f32_e32 v3, v1, v4 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v3, v2, v5 +; GFX7-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc +; GFX7-NEXT: v_max_f32_e32 v3, v2, v5 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v3f32: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc +; GFX8-NEXT: v_max_f32_e32 v6, v0, v3 ; GFX8-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc +; GFX8-NEXT: v_max_f32_e32 v3, v1, v4 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc +; GFX8-NEXT: v_max_f32_e32 v3, v2, v5 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v3f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc +; GFX9-NEXT: v_max_f32_e32 v6, v0, v3 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v1, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v2, v5 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v3f32: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 +; GFX940-NEXT: v_max_f32_e32 v6, v0, v3 ; GFX940-NEXT: v_mov_b32_e32 v7, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc +; GFX940-NEXT: v_max_f32_e32 v3, v1, v4 +; GFX940-NEXT: s_nop 0 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc +; GFX940-NEXT: v_max_f32_e32 v3, v2, v5 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v3f32: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v4, v1, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v6, v0, v3 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v6, 0x7fc00000, v6, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v7, v1, v4 +; GFX10-NEXT: v_max_f32_e32 v8, v2, v5 +; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v7, 0x7fc00000, v7, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v7, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v3f32: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v4, v1, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v6, v0, v3 :: v_dual_max_f32 v7, v1, v4 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v6, 0x7fc00000, v6, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v7, 0x7fc00000, v7, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v8, v2, v5 :: v_dual_cndmask_b32 v1, 0x7fc00000, v7 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v3f32: @@ -1460,184 +1005,48 @@ define <3 x float> @v_maximum_v3f32__nnan(<3 x float> %src0, <3 x float> %src1) ; GFX7-LABEL: v_maximum_v3f32__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v6, v0, v3 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v3, v1, v4 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v3, v2, v5 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX7-NEXT: v_max_f32_e32 v0, v0, v3 +; GFX7-NEXT: v_max_f32_e32 v1, v1, v4 +; GFX7-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v3f32__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX8-NEXT: v_max_f32_e32 v0, v0, v3 +; GFX8-NEXT: v_max_f32_e32 v1, v1, v4 +; GFX8-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v3f32__nnan: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_max_f32_e32 v0, v0, v3 +; GFX9-NEXT: v_max_f32_e32 v1, v1, v4 +; GFX9-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v3f32__nnan: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX940-NEXT: v_max_f32_e32 v0, v0, v3 +; GFX940-NEXT: v_max_f32_e32 v1, v1, v4 +; GFX940-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v3f32__nnan: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v4, v1, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v0, v0, v3 +; GFX10-NEXT: v_max_f32_e32 v1, v1, v4 +; GFX10-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v3f32__nnan: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v4, v1, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v0, v0, v3 :: v_dual_max_f32 v1, v1, v4 +; GFX11-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v3f32__nnan: @@ -1659,14 +1068,14 @@ define <3 x float> @v_maximum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) { ; GFX7-LABEL: v_maximum_v3f32__nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v6, v0, v3 +; GFX7-NEXT: v_max_f32_e32 v6, v0, v3 ; GFX7-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v3, v1, v4 +; GFX7-NEXT: v_max_f32_e32 v3, v1, v4 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v3, v2, v5 +; GFX7-NEXT: v_max_f32_e32 v3, v2, v5 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 ; GFX7-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] @@ -1674,17 +1083,14 @@ define <3 x float> @v_maximum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) { ; GFX8-LABEL: v_maximum_v3f32__nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc +; GFX8-NEXT: v_max_f32_e32 v6, v0, v3 ; GFX8-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc +; GFX8-NEXT: v_max_f32_e32 v3, v1, v4 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX8-NEXT: v_max_f32_e32 v3, v2, v5 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] @@ -1692,17 +1098,14 @@ define <3 x float> @v_maximum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) { ; GFX9-LABEL: v_maximum_v3f32__nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc +; GFX9-NEXT: v_max_f32_e32 v6, v0, v3 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v1, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX9-NEXT: v_max_f32_e32 v3, v2, v5 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] @@ -1710,22 +1113,16 @@ define <3 x float> @v_maximum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) { ; GFX940-LABEL: v_maximum_v3f32__nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 +; GFX940-NEXT: v_max_f32_e32 v6, v0, v3 ; GFX940-NEXT: v_mov_b32_e32 v7, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_max_f32_e32 v3, v1, v4 +; GFX940-NEXT: s_nop 0 ; GFX940-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 ; GFX940-NEXT: s_nop 1 ; GFX940-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX940-NEXT: v_max_f32_e32 v3, v2, v5 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 ; GFX940-NEXT: s_nop 1 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc @@ -1734,13 +1131,10 @@ define <3 x float> @v_maximum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) { ; GFX10-LABEL: v_maximum_v3f32__nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v4, v1, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v6, v0, v3 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v3 +; GFX10-NEXT: v_max_f32_e32 v7, v1, v4 +; GFX10-NEXT: v_max_f32_e32 v8, v2, v5 ; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v4 ; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v7, vcc_lo @@ -1751,17 +1145,14 @@ define <3 x float> @v_maximum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) { ; GFX11-LABEL: v_maximum_v3f32__nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v4, v1, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v6, v0, v3 :: v_dual_max_f32 v7, v1, v4 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v7, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v8, v2, v5 :: v_dual_cndmask_b32 v1, 0x7fc00000, v7 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; @@ -1784,67 +1175,48 @@ define <3 x float> @v_maximum_v3f32__nnan_nsz(<3 x float> %src0, <3 x float> %sr ; GFX7-LABEL: v_maximum_v3f32__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v0, v0, v3 -; GFX7-NEXT: v_max_legacy_f32_e32 v1, v1, v4 -; GFX7-NEXT: v_max_legacy_f32_e32 v2, v2, v5 +; GFX7-NEXT: v_max_f32_e32 v0, v0, v3 +; GFX7-NEXT: v_max_f32_e32 v1, v1, v4 +; GFX7-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v3f32__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc +; GFX8-NEXT: v_max_f32_e32 v0, v0, v3 +; GFX8-NEXT: v_max_f32_e32 v1, v1, v4 +; GFX8-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v3f32__nnan_nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc +; GFX9-NEXT: v_max_f32_e32 v0, v0, v3 +; GFX9-NEXT: v_max_f32_e32 v1, v1, v4 +; GFX9-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v3f32__nnan_nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v2, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc +; GFX940-NEXT: v_max_f32_e32 v0, v0, v3 +; GFX940-NEXT: v_max_f32_e32 v1, v1, v4 +; GFX940-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v3f32__nnan_nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v0, v0, v3 +; GFX10-NEXT: v_max_f32_e32 v1, v1, v4 +; GFX10-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v3f32__nnan_nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v0, v0, v3 :: v_dual_max_f32 v1, v1, v4 +; GFX11-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v3f32__nnan_nsz: @@ -1866,292 +1238,111 @@ define <4 x float> @v_maximum_v4f32(<4 x float> %src0, <4 x float> %src1) { ; GFX7-LABEL: v_maximum_v4f32: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v8, v0, v4 +; GFX7-NEXT: v_max_f32_e32 v8, v0, v4 ; GFX7-NEXT: v_mov_b32_e32 v9, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v4, v1, v5 +; GFX7-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc +; GFX7-NEXT: v_max_f32_e32 v4, v1, v5 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v4, v2, v6 +; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc +; GFX7-NEXT: v_max_f32_e32 v4, v2, v6 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v4, v3, v7 +; GFX7-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc +; GFX7-NEXT: v_max_f32_e32 v4, v3, v7 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v4f32: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc +; GFX8-NEXT: v_max_f32_e32 v8, v0, v4 ; GFX8-NEXT: v_mov_b32_e32 v9, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc +; GFX8-NEXT: v_max_f32_e32 v4, v1, v5 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v2, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc +; GFX8-NEXT: v_max_f32_e32 v4, v2, v6 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v3, v7 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc +; GFX8-NEXT: v_max_f32_e32 v4, v3, v7 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v4f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc +; GFX9-NEXT: v_max_f32_e32 v8, v0, v4 ; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc +; GFX9-NEXT: v_max_f32_e32 v4, v1, v5 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc +; GFX9-NEXT: v_max_f32_e32 v4, v2, v6 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v3, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc +; GFX9-NEXT: v_max_f32_e32 v4, v3, v7 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v4f32: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4 +; GFX940-NEXT: v_max_f32_e32 v8, v0, v4 ; GFX940-NEXT: v_mov_b32_e32 v9, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc +; GFX940-NEXT: v_max_f32_e32 v4, v1, v5 +; GFX940-NEXT: s_nop 0 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v2, v6 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc +; GFX940-NEXT: v_max_f32_e32 v4, v2, v6 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v3, v7 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc +; GFX940-NEXT: v_max_f32_e32 v4, v3, v7 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v4f32: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v8, v0, v4 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v10, v6, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v9, v1, v5 +; GFX10-NEXT: v_max_f32_e32 v4, v2, v6 +; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v8, v3, v7 +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v9, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v10, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v4f32: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v8, v0, v4 :: v_dual_max_f32 v9, v1, v5 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v10, v6, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo +; GFX11-NEXT: v_max_f32_e32 v4, v2, v6 +; GFX11-NEXT: v_dual_max_f32 v8, v3, v7 :: v_dual_cndmask_b32 v1, 0x7fc00000, v9 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v10, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v4f32: @@ -2174,236 +1365,53 @@ define <4 x float> @v_maximum_v4f32__nnan(<4 x float> %src0, <4 x float> %src1) ; GFX7-LABEL: v_maximum_v4f32__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v8, v0, v4 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v4, v1, v5 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v4, v2, v6 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v4, v3, v7 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX7-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX7-NEXT: v_max_f32_e32 v1, v1, v5 +; GFX7-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX7-NEXT: v_max_f32_e32 v3, v3, v7 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v4f32__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v2, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v3, v7 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX8-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX8-NEXT: v_max_f32_e32 v1, v1, v5 +; GFX8-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX8-NEXT: v_max_f32_e32 v3, v3, v7 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v4f32__nnan: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v3, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX9-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX9-NEXT: v_max_f32_e32 v1, v1, v5 +; GFX9-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX9-NEXT: v_max_f32_e32 v3, v3, v7 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v4f32__nnan: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v2, v6 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v3, v7 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX940-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX940-NEXT: v_max_f32_e32 v1, v1, v5 +; GFX940-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX940-NEXT: v_max_f32_e32 v3, v3, v7 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v4f32__nnan: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v6, v2, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v7, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX10-NEXT: v_max_f32_e32 v1, v1, v5 +; GFX10-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX10-NEXT: v_max_f32_e32 v3, v3, v7 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v4f32__nnan: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v6, v2, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v7, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v0, v0, v4 :: v_dual_max_f32 v1, v1, v5 +; GFX11-NEXT: v_dual_max_f32 v2, v2, v6 :: v_dual_max_f32 v3, v3, v7 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v4f32__nnan: @@ -2426,17 +1434,17 @@ define <4 x float> @v_maximum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) { ; GFX7-LABEL: v_maximum_v4f32__nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v8, v0, v4 +; GFX7-NEXT: v_max_f32_e32 v8, v0, v4 ; GFX7-NEXT: v_mov_b32_e32 v9, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v4, v1, v5 +; GFX7-NEXT: v_max_f32_e32 v4, v1, v5 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v4, v2, v6 +; GFX7-NEXT: v_max_f32_e32 v4, v2, v6 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 ; GFX7-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v4, v3, v7 +; GFX7-NEXT: v_max_f32_e32 v4, v3, v7 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 ; GFX7-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] @@ -2444,21 +1452,17 @@ define <4 x float> @v_maximum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) { ; GFX8-LABEL: v_maximum_v4f32__nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc +; GFX8-NEXT: v_max_f32_e32 v8, v0, v4 ; GFX8-NEXT: v_mov_b32_e32 v9, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc +; GFX8-NEXT: v_max_f32_e32 v4, v1, v5 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v2, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc +; GFX8-NEXT: v_max_f32_e32 v4, v2, v6 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v3, v7 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc +; GFX8-NEXT: v_max_f32_e32 v4, v3, v7 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] @@ -2466,21 +1470,17 @@ define <4 x float> @v_maximum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) { ; GFX9-LABEL: v_maximum_v4f32__nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc +; GFX9-NEXT: v_max_f32_e32 v8, v0, v4 ; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc +; GFX9-NEXT: v_max_f32_e32 v4, v1, v5 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc +; GFX9-NEXT: v_max_f32_e32 v4, v2, v6 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v3, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc +; GFX9-NEXT: v_max_f32_e32 v4, v3, v7 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] @@ -2488,28 +1488,20 @@ define <4 x float> @v_maximum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) { ; GFX940-LABEL: v_maximum_v4f32__nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4 +; GFX940-NEXT: v_max_f32_e32 v8, v0, v4 ; GFX940-NEXT: v_mov_b32_e32 v9, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_max_f32_e32 v4, v1, v5 +; GFX940-NEXT: s_nop 0 ; GFX940-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 ; GFX940-NEXT: s_nop 1 ; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v2, v6 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc +; GFX940-NEXT: v_max_f32_e32 v4, v2, v6 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 ; GFX940-NEXT: s_nop 1 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v3, v7 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc +; GFX940-NEXT: v_max_f32_e32 v4, v3, v7 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 ; GFX940-NEXT: s_nop 1 ; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc @@ -2518,44 +1510,35 @@ define <4 x float> @v_maximum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) { ; GFX10-LABEL: v_maximum_v4f32__nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v8, v0, v4 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v4 +; GFX10-NEXT: v_max_f32_e32 v9, v1, v5 +; GFX10-NEXT: v_max_f32_e32 v4, v2, v6 ; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v6, v2, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v7, v3, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v4, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v8, v3, v7 +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v9, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v9, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v4f32__nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v8, v0, v4 :: v_dual_max_f32 v9, v1, v5 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3) ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v6, v2, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v7, v3, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v4, vcc_lo +; GFX11-NEXT: v_max_f32_e32 v4, v2, v6 +; GFX11-NEXT: v_dual_max_f32 v8, v3, v7 :: v_dual_cndmask_b32 v1, 0x7fc00000, v9 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v9, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v4f32__nsz: @@ -2578,79 +1561,53 @@ define <4 x float> @v_maximum_v4f32__nnan_nsz(<4 x float> %src0, <4 x float> %sr ; GFX7-LABEL: v_maximum_v4f32__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v0, v0, v4 -; GFX7-NEXT: v_max_legacy_f32_e32 v1, v1, v5 -; GFX7-NEXT: v_max_legacy_f32_e32 v2, v2, v6 -; GFX7-NEXT: v_max_legacy_f32_e32 v3, v3, v7 +; GFX7-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX7-NEXT: v_max_f32_e32 v1, v1, v5 +; GFX7-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX7-NEXT: v_max_f32_e32 v3, v3, v7 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v4f32__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v2, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v3, v7 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX8-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX8-NEXT: v_max_f32_e32 v1, v1, v5 +; GFX8-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX8-NEXT: v_max_f32_e32 v3, v3, v7 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v4f32__nnan_nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v3, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX9-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX9-NEXT: v_max_f32_e32 v1, v1, v5 +; GFX9-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX9-NEXT: v_max_f32_e32 v3, v3, v7 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v4f32__nnan_nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v2, v6 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v3, v7 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX940-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX940-NEXT: v_max_f32_e32 v1, v1, v5 +; GFX940-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX940-NEXT: v_max_f32_e32 v3, v3, v7 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v4f32__nnan_nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX10-NEXT: v_max_f32_e32 v1, v1, v5 +; GFX10-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX10-NEXT: v_max_f32_e32 v3, v3, v7 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v4f32__nnan_nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v0, v0, v4 :: v_dual_max_f32 v1, v1, v5 +; GFX11-NEXT: v_dual_max_f32 v2, v2, v6 :: v_dual_max_f32 v3, v3, v7 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v4f32__nnan_nsz: @@ -2673,551 +1630,185 @@ define <8 x float> @v_maximum_v8f32(<8 x float> %src0, <8 x float> %src1) { ; GFX7-LABEL: v_maximum_v8f32: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v0, v8 +; GFX7-NEXT: v_max_f32_e32 v16, v0, v8 ; GFX7-NEXT: v_mov_b32_e32 v17, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v8, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v8, v1, v9 +; GFX7-NEXT: v_cndmask_b32_e32 v0, v17, v16, vcc +; GFX7-NEXT: v_max_f32_e32 v8, v1, v9 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v9 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v9, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v8, v2, v10 +; GFX7-NEXT: v_cndmask_b32_e32 v1, v17, v8, vcc +; GFX7-NEXT: v_max_f32_e32 v8, v2, v10 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v10 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v10, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v8, v3, v11 +; GFX7-NEXT: v_cndmask_b32_e32 v2, v17, v8, vcc +; GFX7-NEXT: v_max_f32_e32 v8, v3, v11 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v3, v11 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v11, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v8, v4, v12 +; GFX7-NEXT: v_cndmask_b32_e32 v3, v17, v8, vcc +; GFX7-NEXT: v_max_f32_e32 v8, v4, v12 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v4, v12 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v12, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v12, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v8, v5, v13 +; GFX7-NEXT: v_cndmask_b32_e32 v4, v17, v8, vcc +; GFX7-NEXT: v_max_f32_e32 v8, v5, v13 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v5, v13 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v13, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v8, v6, v14 +; GFX7-NEXT: v_cndmask_b32_e32 v5, v17, v8, vcc +; GFX7-NEXT: v_max_f32_e32 v8, v6, v14 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v6, v14 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v14, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v8, v7, v15 +; GFX7-NEXT: v_cndmask_b32_e32 v6, v17, v8, vcc +; GFX7-NEXT: v_max_f32_e32 v8, v7, v15 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v7, v15 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v15, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v7, v17, v8, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v8f32: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc +; GFX8-NEXT: v_max_f32_e32 v16, v0, v8 ; GFX8-NEXT: v_mov_b32_e32 v17, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v8, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v9 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v9, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v0, v17, v16, vcc +; GFX8-NEXT: v_max_f32_e32 v8, v1, v9 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v9 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v9, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v2, v10 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v8, vcc +; GFX8-NEXT: v_max_f32_e32 v8, v2, v10 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v2, v10 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v10, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v3, v11 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v11, v3, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v2, v17, v8, vcc +; GFX8-NEXT: v_max_f32_e32 v8, v3, v11 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v3, v11 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v11, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v4, v12 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v12, v4, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v3, v17, v8, vcc +; GFX8-NEXT: v_max_f32_e32 v8, v4, v12 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v4, v12 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v12, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v12, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v5, v13 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v13, v5, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v4, v17, v8, vcc +; GFX8-NEXT: v_max_f32_e32 v8, v5, v13 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v5, v13 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v13, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v6, v14 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v14, v6, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v5, v17, v8, vcc +; GFX8-NEXT: v_max_f32_e32 v8, v6, v14 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v6, v14 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v14, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v7, v15 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v15, v7, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v6, v17, v8, vcc +; GFX8-NEXT: v_max_f32_e32 v8, v7, v15 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v7, v15 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v15, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v7, v17, v8, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v8f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc +; GFX9-NEXT: v_max_f32_e32 v16, v0, v8 ; GFX9-NEXT: v_mov_b32_e32 v17, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v8, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v17, v16, vcc +; GFX9-NEXT: v_max_f32_e32 v8, v1, v9 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v9, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v10 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v8, vcc +; GFX9-NEXT: v_max_f32_e32 v8, v2, v10 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v10 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v10, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v3, v11 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v11, v3, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v17, v8, vcc +; GFX9-NEXT: v_max_f32_e32 v8, v3, v11 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v3, v11 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v11, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v4, v12 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v12, v4, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v3, v17, v8, vcc +; GFX9-NEXT: v_max_f32_e32 v8, v4, v12 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v4, v12 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v12, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v12, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v5, v13 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v13, v5, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v4, v17, v8, vcc +; GFX9-NEXT: v_max_f32_e32 v8, v5, v13 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v5, v13 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v13, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v6, v14 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v14, v6, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v5, v17, v8, vcc +; GFX9-NEXT: v_max_f32_e32 v8, v6, v14 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v6, v14 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v14, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v7, v15 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v15, v7, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v6, v17, v8, vcc +; GFX9-NEXT: v_max_f32_e32 v8, v7, v15 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v7, v15 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v15, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v7, v17, v8, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v8f32: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v8 +; GFX940-NEXT: v_max_f32_e32 v16, v0, v8 ; GFX940-NEXT: v_mov_b32_e32 v17, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v8, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v9 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v9, v1, vcc +; GFX940-NEXT: v_max_f32_e32 v8, v1, v9 +; GFX940-NEXT: s_nop 0 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v17, v16, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v9 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v9, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v2, v10 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v1, v17, v8, vcc +; GFX940-NEXT: v_max_f32_e32 v8, v2, v10 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v2, v10 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v10, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v3, v11 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v11, v3, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v2, v17, v8, vcc +; GFX940-NEXT: v_max_f32_e32 v8, v3, v11 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v3, v11 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v11, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v4, v12 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v12, v4, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v17, v8, vcc +; GFX940-NEXT: v_max_f32_e32 v8, v4, v12 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v4, v12 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v12, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v4, v12, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v5, v13 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v13, v5, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v4, v17, v8, vcc +; GFX940-NEXT: v_max_f32_e32 v8, v5, v13 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v5, v13 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v13, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v6, v14 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v14, v6, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v5, v17, v8, vcc +; GFX940-NEXT: v_max_f32_e32 v8, v6, v14 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v6, v14 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v14, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v7, v15 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v15, v7, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v6, v17, v8, vcc +; GFX940-NEXT: v_max_f32_e32 v8, v7, v15 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v7, v15 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v15, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v7, v17, v8, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v8f32: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v9 -; GFX10-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v16, v0, v8 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v17, v1, v9 +; GFX10-NEXT: v_max_f32_e32 v8, v2, v10 +; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v16, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v9 -; GFX10-NEXT: v_cndmask_b32_e32 v17, 0x7fc00000, v17, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v8, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v9, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v10 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v11 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v11, v3, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v12 -; GFX10-NEXT: v_cndmask_b32_e32 v16, v12, v4, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v9, v3, v11 +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v17, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v10 -; GFX10-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v10, v7, v15 +; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v11 -; GFX10-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v8, v4, v12 +; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v9, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v4, v12 -; GFX10-NEXT: v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v10, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v11, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v12, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v12, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v13 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v13, v5, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v14 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v14, v6, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v15 -; GFX10-NEXT: v_cndmask_b32_e32 v10, v15, v7, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v9, v5, v13 +; GFX10-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v5, v13 -; GFX10-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v8, v6, v14 +; GFX10-NEXT: v_cndmask_b32_e32 v5, 0x7fc00000, v9, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v6, v14 -; GFX10-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v6, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v7, v15 -; GFX10-NEXT: v_cndmask_b32_e32 v10, 0x7fc00000, v10, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v13, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v14, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v15, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v7, 0x7fc00000, v10, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v8f32: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v9 -; GFX11-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v16, v0, v8 :: v_dual_max_f32 v17, v1, v9 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v8 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v16, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v9 -; GFX11-NEXT: v_cndmask_b32_e32 v17, 0x7fc00000, v17, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v8, 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v9, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v10 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v11 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v11, v3, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v12 -; GFX11-NEXT: v_cndmask_b32_e32 v16, v12, v4, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v9, v3, v11 :: v_dual_max_f32 v8, v2, v10 +; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v17, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v10 -; GFX11-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo +; GFX11-NEXT: v_max_f32_e32 v10, v7, v15 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v11 -; GFX11-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v8, v4, v12 :: v_dual_cndmask_b32 v3, 0x7fc00000, v9 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v4, v12 -; GFX11-NEXT: v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v10, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v11, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v12, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v12, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v13 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v13, v5, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v14 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v14, v6, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v15 -; GFX11-NEXT: v_cndmask_b32_e32 v10, v15, v7, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v9, v5, v13 :: v_dual_cndmask_b32 v4, 0x7fc00000, v8 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v5, v13 -; GFX11-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_dual_max_f32 v8, v6, v14 :: v_dual_cndmask_b32 v5, 0x7fc00000, v9 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v6, v14 -; GFX11-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v6, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v7, v15 -; GFX11-NEXT: v_cndmask_b32_e32 v10, 0x7fc00000, v10, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v13, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v14, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v15, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v7, 0x7fc00000, v10, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v8f32: @@ -3244,1071 +1835,371 @@ define <16 x float> @v_maximum_v16f32(<16 x float> %src0, <16 x float> %src1) { ; GFX7-LABEL: v_maximum_v16f32: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v32, v0, v16 -; GFX7-NEXT: v_mov_b32_e32 v31, 0x7fc00000 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v32, v31, v32, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v16, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v1, v17 +; GFX7-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX7-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX7-NEXT: s_mov_b64 exec, s[4:5] +; GFX7-NEXT: v_cmp_o_f32_e64 s[16:17], v0, v16 +; GFX7-NEXT: v_max_f32_e32 v0, v0, v16 +; GFX7-NEXT: buffer_load_dword v16, off, s[0:3], s32 +; GFX7-NEXT: v_writelane_b32 v31, s30, 0 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v17 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v17, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX7-NEXT: buffer_load_dword v17, off, s[0:3], s32 -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v2, v18 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v18 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v18, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v3, v19 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v3, v19 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v19, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v4, v20 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v4, v20 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v20, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v20, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v5, v21 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v5, v21 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v21, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v21, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v6, v22 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v6, v22 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v22, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v22, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v7, v23 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v7, v23 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v23, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v8, v24 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v8, v24 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v8, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v24, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v9, v25 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v9, v25 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v9, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v25, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v9, v9, v25, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v10, v26 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v10, v26 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v10, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v26, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v10, v10, v26, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v11, v27 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v11, v27 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v11, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v27, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v11, v11, v27, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v12, v28 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v12, v28 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v12, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v28, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v12, v12, v28, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v13, v29 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v13, v29 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v13, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v29, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v14, v30 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v14, v30 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v14, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v30, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc +; GFX7-NEXT: v_max_f32_e32 v1, v1, v17 +; GFX7-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v18 +; GFX7-NEXT: v_max_f32_e32 v2, v2, v18 +; GFX7-NEXT: v_mov_b32_e32 v17, 0x7fc00000 +; GFX7-NEXT: v_max_f32_e32 v18, v13, v29 +; GFX7-NEXT: v_cmp_o_f32_e64 s[28:29], v13, v29 +; GFX7-NEXT: v_writelane_b32 v31, s31, 1 +; GFX7-NEXT: v_cmp_o_f32_e64 s[6:7], v3, v19 +; GFX7-NEXT: v_max_f32_e32 v3, v3, v19 +; GFX7-NEXT: v_cmp_o_f32_e64 s[8:9], v4, v20 +; GFX7-NEXT: v_max_f32_e32 v4, v4, v20 +; GFX7-NEXT: v_cmp_o_f32_e64 s[10:11], v5, v21 +; GFX7-NEXT: v_max_f32_e32 v5, v5, v21 +; GFX7-NEXT: v_cmp_o_f32_e64 s[12:13], v6, v22 +; GFX7-NEXT: v_max_f32_e32 v6, v6, v22 +; GFX7-NEXT: v_cmp_o_f32_e64 s[14:15], v7, v23 +; GFX7-NEXT: v_max_f32_e32 v7, v7, v23 +; GFX7-NEXT: v_cmp_o_f32_e64 s[18:19], v8, v24 +; GFX7-NEXT: v_max_f32_e32 v8, v8, v24 +; GFX7-NEXT: v_cmp_o_f32_e64 s[20:21], v9, v25 +; GFX7-NEXT: v_max_f32_e32 v9, v9, v25 +; GFX7-NEXT: v_cmp_o_f32_e64 s[22:23], v10, v26 +; GFX7-NEXT: v_max_f32_e32 v10, v10, v26 +; GFX7-NEXT: v_cmp_o_f32_e64 s[24:25], v11, v27 +; GFX7-NEXT: v_max_f32_e32 v11, v11, v27 +; GFX7-NEXT: v_cmp_o_f32_e64 s[26:27], v12, v28 +; GFX7-NEXT: v_max_f32_e32 v12, v12, v28 +; GFX7-NEXT: v_max_f32_e32 v19, v14, v30 +; GFX7-NEXT: v_cmp_o_f32_e64 s[30:31], v14, v30 +; GFX7-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v13, v17, v18, s[28:29] +; GFX7-NEXT: v_cndmask_b32_e64 v0, v17, v0, s[16:17] +; GFX7-NEXT: v_cndmask_b32_e64 v2, v17, v2, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v17, v3, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v4, v17, v4, s[8:9] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[10:11] +; GFX7-NEXT: v_cndmask_b32_e64 v6, v17, v6, s[12:13] +; GFX7-NEXT: v_cndmask_b32_e64 v7, v17, v7, s[14:15] +; GFX7-NEXT: v_cndmask_b32_e64 v8, v17, v8, s[18:19] +; GFX7-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[20:21] +; GFX7-NEXT: v_cndmask_b32_e64 v10, v17, v10, s[22:23] +; GFX7-NEXT: v_cndmask_b32_e64 v11, v17, v11, s[24:25] +; GFX7-NEXT: v_cndmask_b32_e64 v12, v17, v12, s[26:27] +; GFX7-NEXT: v_cndmask_b32_e64 v14, v17, v19, s[30:31] +; GFX7-NEXT: v_readlane_b32 s31, v31, 1 +; GFX7-NEXT: v_readlane_b32 s30, v31, 0 +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_max_f32_e32 v18, v15, v16 +; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v15, v16 +; GFX7-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc +; GFX7-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload +; GFX7-NEXT: s_mov_b64 exec, s[4:5] ; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_max_legacy_f32_e32 v16, v15, v17 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v15, v17 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v15, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v17, 64 -; GFX7-NEXT: v_cndmask_b32_e32 v15, v15, v17, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v16f32: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v32, v16, v0, vcc -; GFX8-NEXT: v_mov_b32_e32 v31, 0x7fc00000 -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v32, v31, v32, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v16, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v1, v17 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v17, v1, vcc +; GFX8-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX8-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX8-NEXT: s_mov_b64 exec, s[4:5] +; GFX8-NEXT: v_cmp_o_f32_e64 s[16:17], v0, v16 +; GFX8-NEXT: v_max_f32_e32 v0, v0, v16 +; GFX8-NEXT: buffer_load_dword v16, off, s[0:3], s32 +; GFX8-NEXT: v_writelane_b32 v31, s30, 0 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v17 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v17, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX8-NEXT: buffer_load_dword v17, off, s[0:3], s32 -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v2, v18 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v2, v18 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v18, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v3, v19 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v3, v19 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v19, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v4, v20 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v20, v4, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v4, v20 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v20, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v20, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v5, v21 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v21, v5, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v5, v21 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v21, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v21, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v6, v22 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v6, v22 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v22, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v22, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v7, v23 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v23, v7, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v7, v23 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v23, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v8, v24 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v24, v8, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v8, v24 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v8, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v24, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v9, v25 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v25, v9, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v9, v25 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v9, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v25, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v9, v9, v25, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v10, v26 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v10, v26 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v10, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v26, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v10, v10, v26, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v11, v27 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v27, v11, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v11, v27 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v11, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v27, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v11, v11, v27, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v12, v28 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v28, v12, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v12, v28 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v12, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v28, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v12, v12, v28, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v13, v29 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v29, v13, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v13, v29 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v13, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v29, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v14, v30 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v30, v14, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v14, v30 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v14, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v30, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc +; GFX8-NEXT: v_max_f32_e32 v1, v1, v17 +; GFX8-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v18 +; GFX8-NEXT: v_max_f32_e32 v2, v2, v18 +; GFX8-NEXT: v_mov_b32_e32 v17, 0x7fc00000 +; GFX8-NEXT: v_max_f32_e32 v18, v13, v29 +; GFX8-NEXT: v_cmp_o_f32_e64 s[28:29], v13, v29 +; GFX8-NEXT: v_writelane_b32 v31, s31, 1 +; GFX8-NEXT: v_cmp_o_f32_e64 s[6:7], v3, v19 +; GFX8-NEXT: v_max_f32_e32 v3, v3, v19 +; GFX8-NEXT: v_cmp_o_f32_e64 s[8:9], v4, v20 +; GFX8-NEXT: v_max_f32_e32 v4, v4, v20 +; GFX8-NEXT: v_cmp_o_f32_e64 s[10:11], v5, v21 +; GFX8-NEXT: v_max_f32_e32 v5, v5, v21 +; GFX8-NEXT: v_cmp_o_f32_e64 s[12:13], v6, v22 +; GFX8-NEXT: v_max_f32_e32 v6, v6, v22 +; GFX8-NEXT: v_cmp_o_f32_e64 s[14:15], v7, v23 +; GFX8-NEXT: v_max_f32_e32 v7, v7, v23 +; GFX8-NEXT: v_cmp_o_f32_e64 s[18:19], v8, v24 +; GFX8-NEXT: v_max_f32_e32 v8, v8, v24 +; GFX8-NEXT: v_cmp_o_f32_e64 s[20:21], v9, v25 +; GFX8-NEXT: v_max_f32_e32 v9, v9, v25 +; GFX8-NEXT: v_cmp_o_f32_e64 s[22:23], v10, v26 +; GFX8-NEXT: v_max_f32_e32 v10, v10, v26 +; GFX8-NEXT: v_cmp_o_f32_e64 s[24:25], v11, v27 +; GFX8-NEXT: v_max_f32_e32 v11, v11, v27 +; GFX8-NEXT: v_cmp_o_f32_e64 s[26:27], v12, v28 +; GFX8-NEXT: v_max_f32_e32 v12, v12, v28 +; GFX8-NEXT: v_max_f32_e32 v19, v14, v30 +; GFX8-NEXT: v_cmp_o_f32_e64 s[30:31], v14, v30 +; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v13, v17, v18, s[28:29] +; GFX8-NEXT: v_cndmask_b32_e64 v0, v17, v0, s[16:17] +; GFX8-NEXT: v_cndmask_b32_e64 v2, v17, v2, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v17, v3, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v4, v17, v4, s[8:9] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[10:11] +; GFX8-NEXT: v_cndmask_b32_e64 v6, v17, v6, s[12:13] +; GFX8-NEXT: v_cndmask_b32_e64 v7, v17, v7, s[14:15] +; GFX8-NEXT: v_cndmask_b32_e64 v8, v17, v8, s[18:19] +; GFX8-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[20:21] +; GFX8-NEXT: v_cndmask_b32_e64 v10, v17, v10, s[22:23] +; GFX8-NEXT: v_cndmask_b32_e64 v11, v17, v11, s[24:25] +; GFX8-NEXT: v_cndmask_b32_e64 v12, v17, v12, s[26:27] +; GFX8-NEXT: v_cndmask_b32_e64 v14, v17, v19, s[30:31] +; GFX8-NEXT: v_readlane_b32 s31, v31, 1 +; GFX8-NEXT: v_readlane_b32 s30, v31, 0 +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_max_f32_e32 v18, v15, v16 +; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v15, v16 +; GFX8-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc +; GFX8-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload +; GFX8-NEXT: s_mov_b64 exec, s[4:5] ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v15, v17 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v17, v15, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v15, v17 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v15, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v17, 64 -; GFX8-NEXT: v_cndmask_b32_e32 v15, v15, v17, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v16f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v32, v16, v0, vcc -; GFX9-NEXT: v_mov_b32_e32 v31, 0x7fc00000 -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v32, v31, v32, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v16, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v1, v17 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v1, vcc +; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX9-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-NEXT: v_cmp_o_f32_e64 s[16:17], v0, v16 +; GFX9-NEXT: v_max_f32_e32 v0, v0, v16 +; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 +; GFX9-NEXT: v_writelane_b32 v31, s30, 0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v17 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v17, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX9-NEXT: buffer_load_dword v17, off, s[0:3], s32 -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v2, v18 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v18 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v18, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v3, v19 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v3, v19 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v19, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v4, v20 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v20, v4, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v4, v20 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v20, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v20, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v5, v21 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v21, v5, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v5, v21 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v21, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v21, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v6, v22 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v6, v22 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v22, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v22, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v7, v23 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v23, v7, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v7, v23 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v23, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v8, v24 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v24, v8, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v8, v24 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v8, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v24, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v9, v25 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v25, v9, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v9, v25 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v9, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v25, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v9, v25, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v10, v26 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v10, v26 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v10, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v26, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v10, v10, v26, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v11, v27 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v27, v11, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v11, v27 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v11, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v27, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v11, v11, v27, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v12, v28 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v28, v12, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v12, v28 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v12, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v28, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v12, v12, v28, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v13, v29 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v29, v13, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v13, v29 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v13, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v29, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v14, v30 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v30, v14, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v14, v30 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v14, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v30, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc +; GFX9-NEXT: v_max_f32_e32 v1, v1, v17 +; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v18 +; GFX9-NEXT: v_max_f32_e32 v2, v2, v18 +; GFX9-NEXT: v_mov_b32_e32 v17, 0x7fc00000 +; GFX9-NEXT: v_max_f32_e32 v18, v13, v29 +; GFX9-NEXT: v_cmp_o_f32_e64 s[28:29], v13, v29 +; GFX9-NEXT: v_writelane_b32 v31, s31, 1 +; GFX9-NEXT: v_cmp_o_f32_e64 s[6:7], v3, v19 +; GFX9-NEXT: v_max_f32_e32 v3, v3, v19 +; GFX9-NEXT: v_cmp_o_f32_e64 s[8:9], v4, v20 +; GFX9-NEXT: v_max_f32_e32 v4, v4, v20 +; GFX9-NEXT: v_cmp_o_f32_e64 s[10:11], v5, v21 +; GFX9-NEXT: v_max_f32_e32 v5, v5, v21 +; GFX9-NEXT: v_cmp_o_f32_e64 s[12:13], v6, v22 +; GFX9-NEXT: v_max_f32_e32 v6, v6, v22 +; GFX9-NEXT: v_cmp_o_f32_e64 s[14:15], v7, v23 +; GFX9-NEXT: v_max_f32_e32 v7, v7, v23 +; GFX9-NEXT: v_cmp_o_f32_e64 s[18:19], v8, v24 +; GFX9-NEXT: v_max_f32_e32 v8, v8, v24 +; GFX9-NEXT: v_cmp_o_f32_e64 s[20:21], v9, v25 +; GFX9-NEXT: v_max_f32_e32 v9, v9, v25 +; GFX9-NEXT: v_cmp_o_f32_e64 s[22:23], v10, v26 +; GFX9-NEXT: v_max_f32_e32 v10, v10, v26 +; GFX9-NEXT: v_cmp_o_f32_e64 s[24:25], v11, v27 +; GFX9-NEXT: v_max_f32_e32 v11, v11, v27 +; GFX9-NEXT: v_cmp_o_f32_e64 s[26:27], v12, v28 +; GFX9-NEXT: v_max_f32_e32 v12, v12, v28 +; GFX9-NEXT: v_max_f32_e32 v19, v14, v30 +; GFX9-NEXT: v_cmp_o_f32_e64 s[30:31], v14, v30 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v13, v17, v18, s[28:29] +; GFX9-NEXT: v_cndmask_b32_e64 v0, v17, v0, s[16:17] +; GFX9-NEXT: v_cndmask_b32_e64 v2, v17, v2, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v17, v3, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v4, v17, v4, s[8:9] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[10:11] +; GFX9-NEXT: v_cndmask_b32_e64 v6, v17, v6, s[12:13] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v17, v7, s[14:15] +; GFX9-NEXT: v_cndmask_b32_e64 v8, v17, v8, s[18:19] +; GFX9-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[20:21] +; GFX9-NEXT: v_cndmask_b32_e64 v10, v17, v10, s[22:23] +; GFX9-NEXT: v_cndmask_b32_e64 v11, v17, v11, s[24:25] +; GFX9-NEXT: v_cndmask_b32_e64 v12, v17, v12, s[26:27] +; GFX9-NEXT: v_cndmask_b32_e64 v14, v17, v19, s[30:31] +; GFX9-NEXT: v_readlane_b32 s31, v31, 1 +; GFX9-NEXT: v_readlane_b32 s30, v31, 0 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f32_e32 v18, v15, v16 +; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v15, v16 +; GFX9-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc +; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload +; GFX9-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_cmp_gt_f32_e32 vcc, v15, v17 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v15, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v15, v17 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v15, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v17, 64 -; GFX9-NEXT: v_cndmask_b32_e32 v15, v15, v17, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v16f32: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX940-NEXT: scratch_load_dword v31, off, s32 -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v0, v16 ; GFX940-NEXT: v_mov_b32_e32 v32, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v33, v16, v0, vcc +; GFX940-NEXT: v_max_f32_e32 v33, v0, v16 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v33, v32, v33, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v16, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v33 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v1, v17 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v17, v1, vcc +; GFX940-NEXT: v_max_f32_e32 v34, v1, v17 +; GFX940-NEXT: v_max_f32_e32 v35, v2, v18 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v32, v33, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v17 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v17, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v2, v18 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc +; GFX940-NEXT: v_max_f32_e32 v36, v3, v19 +; GFX940-NEXT: v_max_f32_e32 v37, v4, v20 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v32, v34, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v2, v18 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v18, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v3, v19 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc +; GFX940-NEXT: v_max_f32_e32 v38, v5, v21 +; GFX940-NEXT: v_max_f32_e32 v39, v6, v22 +; GFX940-NEXT: v_cndmask_b32_e32 v2, v32, v35, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v3, v19 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v19, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v4, v20 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v20, v4, vcc +; GFX940-NEXT: v_max_f32_e32 v48, v7, v23 +; GFX940-NEXT: v_max_f32_e32 v49, v8, v24 +; GFX940-NEXT: v_cndmask_b32_e32 v3, v32, v36, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v4, v20 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v4, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v20, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v4, v20, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v5, v21 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v21, v5, vcc +; GFX940-NEXT: v_max_f32_e32 v50, v9, v25 +; GFX940-NEXT: v_max_f32_e32 v51, v10, v26 +; GFX940-NEXT: v_cndmask_b32_e32 v4, v32, v37, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v5, v21 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v5, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v21, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v5, v5, v21, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v6, v22 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc +; GFX940-NEXT: v_max_f32_e32 v52, v11, v27 +; GFX940-NEXT: v_max_f32_e32 v53, v12, v28 +; GFX940-NEXT: v_cndmask_b32_e32 v5, v32, v38, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v6, v22 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v6, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v22, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v6, v22, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v7, v23 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v23, v7, vcc +; GFX940-NEXT: v_max_f32_e32 v54, v13, v29 +; GFX940-NEXT: v_max_f32_e32 v55, v14, v30 +; GFX940-NEXT: v_cndmask_b32_e32 v6, v32, v39, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v7, v23 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v7, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v23, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v8, v24 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v24, v8, vcc +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_max_f32_e32 v16, v15, v31 +; GFX940-NEXT: v_cndmask_b32_e32 v7, v32, v48, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v8, v24 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v8, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v24, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v9, v25 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v25, v9, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v8, v32, v49, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v9, v25 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v9, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v25, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v9, v9, v25, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v10, v26 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v9, v32, v50, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v10, v26 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v10, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v26, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v10, v10, v26, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v11, v27 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v27, v11, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v10, v32, v51, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v11, v27 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v11, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v27, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v11, v11, v27, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v12, v28 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v28, v12, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v11, v32, v52, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v12, v28 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v12, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v28, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v12, v12, v28, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v13, v29 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v29, v13, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v12, v32, v53, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v13, v29 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v13, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v29, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v14, v30 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v30, v14, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v13, v32, v54, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v14, v30 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v14, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v30, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc -; GFX940-NEXT: s_waitcnt vmcnt(0) -; GFX940-NEXT: v_cmp_gt_f32_e32 vcc, v15, v31 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v31, v15, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v14, v32, v55, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v15, v31 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v15, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v31, 64 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v15, v15, v31, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v15, v32, v16, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v16f32: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v16 ; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX10-NEXT: v_cndmask_b32_e32 v32, v16, v0, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v17 -; GFX10-NEXT: v_cndmask_b32_e32 v33, v17, v1, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v18 -; GFX10-NEXT: v_cndmask_b32_e32 v34, v18, v2, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v19 -; GFX10-NEXT: v_cndmask_b32_e32 v35, v19, v3, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v20 -; GFX10-NEXT: v_cndmask_b32_e32 v36, v20, v4, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v21 -; GFX10-NEXT: v_cndmask_b32_e32 v37, v21, v5, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v22 -; GFX10-NEXT: v_cndmask_b32_e32 v38, v22, v6, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v23 -; GFX10-NEXT: v_cndmask_b32_e32 v39, v23, v7, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v24 -; GFX10-NEXT: v_cndmask_b32_e32 v48, v24, v8, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v25 -; GFX10-NEXT: v_cndmask_b32_e32 v49, v25, v9, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v26 -; GFX10-NEXT: v_cndmask_b32_e32 v50, v26, v10, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v27 -; GFX10-NEXT: v_cndmask_b32_e32 v51, v27, v11, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v28 -; GFX10-NEXT: v_cndmask_b32_e32 v52, v28, v12, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v29 -; GFX10-NEXT: v_cndmask_b32_e32 v53, v29, v13, vcc_lo -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v30 -; GFX10-NEXT: v_cndmask_b32_e32 v54, v30, v14, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v32, v0, v16 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v16 -; GFX10-NEXT: v_cndmask_b32_e32 v32, 0x7fc00000, v32, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v33, v1, v17 +; GFX10-NEXT: v_max_f32_e32 v34, v2, v18 +; GFX10-NEXT: v_max_f32_e32 v35, v3, v19 +; GFX10-NEXT: v_max_f32_e32 v36, v4, v20 +; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v32, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v17 -; GFX10-NEXT: v_cndmask_b32_e32 v33, 0x7fc00000, v33, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v37, v5, v21 +; GFX10-NEXT: v_max_f32_e32 v38, v6, v22 +; GFX10-NEXT: v_max_f32_e32 v39, v7, v23 +; GFX10-NEXT: v_max_f32_e32 v48, v8, v24 +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v33, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v18 -; GFX10-NEXT: v_cndmask_b32_e32 v34, 0x7fc00000, v34, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v49, v9, v25 +; GFX10-NEXT: v_max_f32_e32 v50, v10, v26 +; GFX10-NEXT: v_max_f32_e32 v51, v11, v27 +; GFX10-NEXT: v_max_f32_e32 v52, v12, v28 +; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v34, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v19 -; GFX10-NEXT: v_cndmask_b32_e32 v35, 0x7fc00000, v35, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v53, v13, v29 +; GFX10-NEXT: v_max_f32_e32 v54, v14, v30 +; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v35, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v4, v20 -; GFX10-NEXT: v_cndmask_b32_e32 v36, 0x7fc00000, v36, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v36, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v5, v21 -; GFX10-NEXT: v_cndmask_b32_e32 v37, 0x7fc00000, v37, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v5, 0x7fc00000, v37, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v6, v22 -; GFX10-NEXT: v_cndmask_b32_e32 v38, 0x7fc00000, v38, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v6, 0x7fc00000, v38, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v7, v23 -; GFX10-NEXT: v_cndmask_b32_e32 v39, 0x7fc00000, v39, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v7, 0x7fc00000, v39, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v8, v24 -; GFX10-NEXT: v_cndmask_b32_e32 v48, 0x7fc00000, v48, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v48, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v9, v25 -; GFX10-NEXT: v_cndmask_b32_e32 v49, 0x7fc00000, v49, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v49, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v10, v26 -; GFX10-NEXT: v_cndmask_b32_e32 v50, 0x7fc00000, v50, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v10, 0x7fc00000, v50, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v11, v27 -; GFX10-NEXT: v_cndmask_b32_e32 v51, 0x7fc00000, v51, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v11, 0x7fc00000, v51, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v12, v28 -; GFX10-NEXT: v_cndmask_b32_e32 v52, 0x7fc00000, v52, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v12, 0x7fc00000, v52, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v13, v29 -; GFX10-NEXT: v_cndmask_b32_e32 v53, 0x7fc00000, v53, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v13, 0x7fc00000, v53, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v14, v30 -; GFX10-NEXT: v_cndmask_b32_e32 v54, 0x7fc00000, v54, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v33, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v34, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v35, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v36, v4, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v37, v5, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v38, v6, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v39, v7, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v8, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v48, v8, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v9, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v49, v9, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v10, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v11, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v12, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v12, v52, v12, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v13, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v13, v53, v13, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v14, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v14, v54, v14, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v16, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v17, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v18, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v19, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v20, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v20, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v21, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v21, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v22, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v22, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v23, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v24, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v25, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v9, v25, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v26, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v10, v10, v26, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v27, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v11, v11, v27, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v28, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v12, v12, v28, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v29, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v30, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v33 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v33, v1, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v34 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v34, v2, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v35 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v35, v3, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v36 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v36, v4, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v37, v5, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v38 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v38, v6, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v14, 0x7fc00000, v54, vcc_lo ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v31 -; GFX10-NEXT: v_cndmask_b32_e32 v16, v31, v15, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v39 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v39, v7, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v48, v8, vcc_lo +; GFX10-NEXT: v_max_f32_e32 v16, v15, v31 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v15, v31 -; GFX10-NEXT: v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v49 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v49, v9, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v50 -; GFX10-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v15, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v51 -; GFX10-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v52 -; GFX10-NEXT: v_cndmask_b32_e32 v12, v52, v12, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v31, 64 -; GFX10-NEXT: v_cndmask_b32_e32 v15, v15, v31, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v53 -; GFX10-NEXT: v_cndmask_b32_e32 v13, v53, v13, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v54 -; GFX10-NEXT: v_cndmask_b32_e32 v14, v54, v14, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 -; GFX10-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v15, 0x7fc00000, v16, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v16f32: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v16 ; GFX11-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-NEXT: v_cndmask_b32_e32 v32, v16, v0, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v17 -; GFX11-NEXT: v_cndmask_b32_e32 v33, v17, v1, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v18 -; GFX11-NEXT: v_cndmask_b32_e32 v34, v18, v2, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v19 -; GFX11-NEXT: v_cndmask_b32_e32 v35, v19, v3, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v20 -; GFX11-NEXT: v_cndmask_b32_e32 v36, v20, v4, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v21 -; GFX11-NEXT: v_cndmask_b32_e32 v37, v21, v5, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v22 -; GFX11-NEXT: v_cndmask_b32_e32 v38, v22, v6, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v23 -; GFX11-NEXT: v_cndmask_b32_e32 v39, v23, v7, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v24 -; GFX11-NEXT: v_cndmask_b32_e32 v48, v24, v8, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v25 -; GFX11-NEXT: v_cndmask_b32_e32 v49, v25, v9, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v26 -; GFX11-NEXT: v_cndmask_b32_e32 v50, v26, v10, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v27 -; GFX11-NEXT: v_cndmask_b32_e32 v51, v27, v11, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v28 -; GFX11-NEXT: v_cndmask_b32_e32 v52, v28, v12, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v29 -; GFX11-NEXT: v_cndmask_b32_e32 v53, v29, v13, vcc_lo -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v30 -; GFX11-NEXT: v_cndmask_b32_e32 v54, v30, v14, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v32, v0, v16 :: v_dual_max_f32 v33, v1, v17 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v16 -; GFX11-NEXT: v_cndmask_b32_e32 v32, 0x7fc00000, v32, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v34, v2, v18 :: v_dual_max_f32 v35, v3, v19 +; GFX11-NEXT: v_dual_max_f32 v36, v4, v20 :: v_dual_max_f32 v37, v5, v21 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v32, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v17 -; GFX11-NEXT: v_cndmask_b32_e32 v33, 0x7fc00000, v33, vcc_lo +; GFX11-NEXT: v_max_f32_e32 v54, v14, v30 +; GFX11-NEXT: v_dual_max_f32 v38, v6, v22 :: v_dual_max_f32 v39, v7, v23 +; GFX11-NEXT: v_dual_max_f32 v48, v8, v24 :: v_dual_max_f32 v49, v9, v25 +; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v33, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v18 -; GFX11-NEXT: v_cndmask_b32_e32 v34, 0x7fc00000, v34, vcc_lo +; GFX11-NEXT: v_dual_max_f32 v50, v10, v26 :: v_dual_max_f32 v51, v11, v27 +; GFX11-NEXT: v_dual_max_f32 v52, v12, v28 :: v_dual_max_f32 v53, v13, v29 +; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v34, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v19 -; GFX11-NEXT: v_cndmask_b32_e32 v35, 0x7fc00000, v35, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v35, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v4, v20 -; GFX11-NEXT: v_cndmask_b32_e32 v36, 0x7fc00000, v36, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v36, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v5, v21 -; GFX11-NEXT: v_cndmask_b32_e32 v37, 0x7fc00000, v37, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v5, 0x7fc00000, v37, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v6, v22 -; GFX11-NEXT: v_cndmask_b32_e32 v38, 0x7fc00000, v38, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v6, 0x7fc00000, v38, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v7, v23 -; GFX11-NEXT: v_cndmask_b32_e32 v39, 0x7fc00000, v39, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v7, 0x7fc00000, v39, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v8, v24 -; GFX11-NEXT: v_cndmask_b32_e32 v48, 0x7fc00000, v48, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v48, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v9, v25 -; GFX11-NEXT: v_cndmask_b32_e32 v49, 0x7fc00000, v49, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v49, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v10, v26 -; GFX11-NEXT: v_cndmask_b32_e32 v50, 0x7fc00000, v50, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v10, 0x7fc00000, v50, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v11, v27 -; GFX11-NEXT: v_cndmask_b32_e32 v51, 0x7fc00000, v51, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v11, 0x7fc00000, v51, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v12, v28 -; GFX11-NEXT: v_cndmask_b32_e32 v52, 0x7fc00000, v52, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v12, 0x7fc00000, v52, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v13, v29 -; GFX11-NEXT: v_cndmask_b32_e32 v53, 0x7fc00000, v53, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v13, 0x7fc00000, v53, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v14, v30 -; GFX11-NEXT: v_cndmask_b32_e32 v54, 0x7fc00000, v54, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v33, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v34, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v35, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v36, v4, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v37, v5, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v38, v6, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v39, v7, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v8, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v48, v8, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v9, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v49, v9, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v10, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v11, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v12, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v12, v52, v12, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v13, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v13, v53, v13, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v14, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v14, v54, v14, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v16, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v17, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v18, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v19, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v20, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v20, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v21, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v5, v21, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v22, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v6, v22, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v23, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v24, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v25, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v9, v25, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v26, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v10, v10, v26, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v27, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v11, v11, v27, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v28, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v12, v12, v28, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v29, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v30, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v32 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v33 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v33, v1, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v34 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v34, v2, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v35 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v35, v3, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v36 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v36, v4, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v37, v5, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v38 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v38, v6, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v14, 0x7fc00000, v54, vcc_lo ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v31 -; GFX11-NEXT: v_cndmask_b32_e32 v16, v31, v15, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v39 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v39, v7, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v48, v8, vcc_lo +; GFX11-NEXT: v_max_f32_e32 v16, v15, v31 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v15, v31 -; GFX11-NEXT: v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v49 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v49, v9, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v50 -; GFX11-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v15, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v51 -; GFX11-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v52 -; GFX11-NEXT: v_cndmask_b32_e32 v12, v52, v12, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v31, 64 -; GFX11-NEXT: v_cndmask_b32_e32 v15, v15, v31, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v53 -; GFX11-NEXT: v_cndmask_b32_e32 v13, v53, v13, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v54 -; GFX11-NEXT: v_cndmask_b32_e32 v14, v54, v14, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 -; GFX11-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-NEXT: v_cndmask_b32_e32 v15, 0x7fc00000, v16, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v16f32: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll index d60a28e74043..78fb23182f80 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll @@ -530,221 +530,86 @@ define <2 x double> @v_maximum_v2f64(<2 x double> %src0, <2 x double> %src1) { ; GFX7-LABEL: v_maximum_v2f64: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5] -; GFX7-NEXT: v_mov_b32_e32 v10, 0x7ff80000 -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[0:1], 64 -; GFX7-NEXT: v_cmp_class_f64_e64 s[8:9], v[4:5], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v9, v10, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[4:5] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[8:9] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v11, v7, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v11, v10, v11, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v10, 0, v4, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 64 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[12:13] +; GFX7-NEXT: v_max_f64 v[8:9], v[0:1], v[4:5] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX7-NEXT: v_max_f64 v[4:5], v[2:3], v[6:7] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7] +; GFX7-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[4:5] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v2f64: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5] -; GFX8-NEXT: v_mov_b32_e32 v10, 0x7ff80000 -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[0:1], 64 -; GFX8-NEXT: v_cmp_class_f64_e64 s[8:9], v[4:5], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v9, v10, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[4:5] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[8:9] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v11, v7, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v11, v10, v11, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v10, 0, v4, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 64 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[12:13] +; GFX8-NEXT: v_max_f64 v[8:9], v[0:1], v[4:5] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[6:7] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7] +; GFX8-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[4:5] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v2f64: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5] -; GFX9-NEXT: v_mov_b32_e32 v10, 0x7ff80000 -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[0:1], 64 -; GFX9-NEXT: v_cmp_class_f64_e64 s[8:9], v[4:5], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v9, v10, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[4:5] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[8:9] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v11, v7, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v11, v10, v11, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, v4, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 64 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[12:13] +; GFX9-NEXT: v_max_f64 v[8:9], v[0:1], v[4:5] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX9-NEXT: v_max_f64 v[4:5], v[2:3], v[6:7] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7] +; GFX9-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[4:5] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v2f64: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX940-NEXT: v_mov_b32_e32 v10, 0x7ff80000 -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[4:5] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc +; GFX940-NEXT: v_max_f64 v[8:9], v[0:1], v[4:5] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX940-NEXT: v_max_f64 v[4:5], v[2:3], v[6:7] ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v9, v10, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[4:5], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[6:7] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v10, v4, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, 0, v4, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[6:7], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[4:5] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v4, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[2:3] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX940-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v8, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[6:7] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v2, v4, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v8, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v2f64: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[2:3], v[6:7] -; GFX10-NEXT: v_cmp_o_f64_e64 s5, v[0:1], v[4:5] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[2:3], v[6:7] -; GFX10-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v10, v7, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v12, v4, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v13, v6, v2, s4 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[2:3], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v9, 0x7ff80000, v8, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v10, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, v12, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, v13, s6 -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[4:5], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[6:7], 64 -; GFX10-NEXT: v_cmp_eq_f64_e64 s7, 0, v[8:9] -; GFX10-NEXT: v_cmp_eq_f64_e64 s8, 0, v[10:11] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v7, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v8, v0, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v10, v2, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v9, v1, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, v3, s8 +; GFX10-NEXT: v_max_f64 v[8:9], v[0:1], v[4:5] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5] +; GFX10-NEXT: v_max_f64 v[4:5], v[2:3], v[6:7] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[6:7] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v4, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, 0x7ff80000, s4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v2f64: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX11-NEXT: v_cmp_o_f64_e64 s1, v[0:1], v[4:5] -; GFX11-NEXT: v_cmp_gt_f64_e64 s0, v[2:3], v[6:7] -; GFX11-NEXT: v_cmp_o_f64_e64 s2, v[2:3], v[6:7] -; GFX11-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v10, v7, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v13, v6, v2, s0 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v9, 0x7ff80000, v8, s1 -; GFX11-NEXT: v_cndmask_b32_e32 v12, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v10, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v10, 0, v13, s2 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[6:7], 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_cmp_eq_f64_e64 s4, 0, v[10:11] -; GFX11-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v8, 0, v12, s1 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[4:5], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v6, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v7, s2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cmp_eq_f64_e64 s3, 0, v[8:9] -; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v5, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v0, v8, v0, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v9, v1, s3 +; GFX11-NEXT: v_max_f64 v[8:9], v[0:1], v[4:5] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5] +; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[6:7] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[6:7] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v5, 0x7ff80000, s0 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v2f64: @@ -765,182 +630,43 @@ define <2 x double> @v_maximum_v2f64__nnan(<2 x double> %src0, <2 x double> %src ; GFX7-LABEL: v_maximum_v2f64__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[4:5], 64 -; GFX7-NEXT: v_cmp_class_f64_e64 s[10:11], v[6:7], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v11, v7, v3, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v10, v6, v2, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[2:3], 64 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[8:9] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[12:13] +; GFX7-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX7-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v2f64__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[4:5], 64 -; GFX8-NEXT: v_cmp_class_f64_e64 s[10:11], v[6:7], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v11, v7, v3, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v10, v6, v2, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[2:3], 64 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[8:9] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[12:13] +; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v2f64__nnan: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[4:5], 64 -; GFX9-NEXT: v_cmp_class_f64_e64 s[10:11], v[6:7], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v11, v7, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v10, v6, v2, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[2:3], 64 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[8:9] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[12:13] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX9-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v2f64__nnan: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[4:5], 64 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v5, v7, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[6:7], 64 -; GFX940-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[0:1] -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[4:5] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[0:1] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v2, v4, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[2:3] +; GFX940-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX940-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7] ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v2f64__nnan: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[2:3], v[6:7] -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[4:5], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[6:7], 64 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v11, v7, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v10, v6, v2, s4 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[2:3], 64 -; GFX10-NEXT: v_cmp_eq_f64_e64 s7, 0, v[8:9] -; GFX10-NEXT: v_cmp_eq_f64_e64 s8, 0, v[10:11] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v7, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v8, v0, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v10, v2, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v9, v1, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, v3, s8 +; GFX10-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX10-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v2f64__nnan: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX11-NEXT: v_cmp_gt_f64_e64 s0, v[2:3], v[6:7] -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[4:5], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[6:7], 64 -; GFX11-NEXT: v_dual_cndmask_b32 v9, v5, v1 :: v_dual_cndmask_b32 v8, v4, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v11, v7, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v6, v2, s0 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 64 -; GFX11-NEXT: v_cmp_eq_f64_e64 s3, 0, v[8:9] -; GFX11-NEXT: v_cmp_eq_f64_e64 s4, 0, v[10:11] -; GFX11-NEXT: v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v6, s2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v7, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v8, v0, s3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v1, v9, v1, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 +; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v2f64__nnan: @@ -961,111 +687,86 @@ define <2 x double> @v_maximum_v2f64__nsz(<2 x double> %src0, <2 x double> %src1 ; GFX7-LABEL: v_maximum_v2f64__nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[6:7], v[2:3], v[6:7] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[2:3], v[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v4, v6, v2, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX7-NEXT: v_mov_b32_e32 v5, 0x7ff80000 -; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v0, 0, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v5, v1, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v2, 0, v4, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[8:9] +; GFX7-NEXT: v_max_f64 v[8:9], v[0:1], v[4:5] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX7-NEXT: v_max_f64 v[4:5], v[2:3], v[6:7] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7] +; GFX7-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[4:5] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v2f64__nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[6:7], v[2:3], v[6:7] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[2:3], v[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v2, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8-NEXT: v_mov_b32_e32 v5, 0x7ff80000 -; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v5, v1, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, v4, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[8:9] +; GFX8-NEXT: v_max_f64 v[8:9], v[0:1], v[4:5] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[6:7] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7] +; GFX8-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[4:5] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v2f64__nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[6:7], v[2:3], v[6:7] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[2:3], v[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v4, v6, v2, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000 -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v5, v1, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, v4, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[8:9] +; GFX9-NEXT: v_max_f64 v[8:9], v[0:1], v[4:5] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX9-NEXT: v_max_f64 v[4:5], v[2:3], v[6:7] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7] +; GFX9-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[4:5] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v2f64__nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[4:5] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX940-NEXT: v_mov_b32_e32 v4, 0x7ff80000 -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7] -; GFX940-NEXT: v_cndmask_b32_e64 v0, 0, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v4, v1, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v2, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[6:7] -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX940-NEXT: v_max_f64 v[8:9], v[0:1], v[4:5] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX940-NEXT: v_max_f64 v[4:5], v[2:3], v[6:7] ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v2, 0, v5, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v4, v3, s[0:1] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX940-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v8, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[6:7] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v2, v4, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v8, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v2f64__nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[2:3], v[6:7] -; GFX10-NEXT: v_cmp_o_f64_e64 s5, v[0:1], v[4:5] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[2:3], v[6:7] -; GFX10-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v9, v6, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, v8, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, v9, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v1, 0x7ff80000, v1, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v3, s6 +; GFX10-NEXT: v_max_f64 v[8:9], v[0:1], v[4:5] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5] +; GFX10-NEXT: v_max_f64 v[4:5], v[2:3], v[6:7] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[6:7] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v4, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, 0x7ff80000, s4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v2f64__nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX11-NEXT: v_cmp_gt_f64_e64 s0, v[2:3], v[6:7] -; GFX11-NEXT: v_cmp_o_f64_e64 s1, v[0:1], v[4:5] -; GFX11-NEXT: v_cmp_o_f64_e64 s2, v[2:3], v[6:7] -; GFX11-NEXT: v_dual_cndmask_b32 v8, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX11-NEXT: v_max_f64 v[8:9], v[0:1], v[4:5] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5] +; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[6:7] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[6:7] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v9, v6, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, v8, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v1, 0x7ff80000, v1, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, v9, s2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v3, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v5, 0x7ff80000, s0 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v2f64__nsz: @@ -1086,69 +787,43 @@ define <2 x double> @v_maximum_v2f64__nnan_nsz(<2 x double> %src0, <2 x double> ; GFX7-LABEL: v_maximum_v2f64__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] +; GFX7-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX7-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v2f64__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] +; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v2f64__nnan_nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX9-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v2f64__nnan_nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[4:5] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[6:7] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX940-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX940-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7] ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v2f64__nnan_nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[2:3], v[6:7] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v3, s4 +; GFX10-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX10-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v2f64__nnan_nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX11-NEXT: v_cmp_gt_f64_e64 s0, v[2:3], v[6:7] -; GFX11-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v6, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0 +; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v2f64__nnan_nsz: @@ -1170,61 +845,20 @@ define void @s_maximum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1) ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_mov_b32_e32 v0, s10 +; GFX7-NEXT: v_mov_b32_e32 v4, s8 ; GFX7-NEXT: v_mov_b32_e32 v1, s11 -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, s[6:7], v[0:1] -; GFX7-NEXT: v_cmp_o_f64_e64 s[12:13], s[6:7], v[0:1] -; GFX7-NEXT: v_cmp_class_f64_e64 s[18:19], s[10:11], 64 -; GFX7-NEXT: v_mov_b32_e32 v0, s8 -; GFX7-NEXT: v_mov_b32_e32 v1, s9 -; GFX7-NEXT: s_and_b64 s[14:15], vcc, exec -; GFX7-NEXT: s_cselect_b32 s16, s7, s11 -; GFX7-NEXT: s_and_b64 s[14:15], s[12:13], exec -; GFX7-NEXT: s_cselect_b32 s15, s16, 0x7ff80000 -; GFX7-NEXT: s_and_b64 s[16:17], vcc, exec -; GFX7-NEXT: s_cselect_b32 s14, s6, s10 -; GFX7-NEXT: v_cmp_class_f64_e64 s[16:17], s[6:7], 64 -; GFX7-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX7-NEXT: s_cselect_b32 s14, s14, 0 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[20:21], s[14:15], 0 -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1] -; GFX7-NEXT: s_and_b64 s[12:13], s[16:17], exec -; GFX7-NEXT: s_cselect_b32 s7, s7, s15 -; GFX7-NEXT: s_and_b64 s[12:13], s[18:19], exec -; GFX7-NEXT: s_cselect_b32 s7, s11, s7 -; GFX7-NEXT: s_and_b64 s[12:13], s[20:21], exec -; GFX7-NEXT: s_cselect_b32 s7, s7, s15 -; GFX7-NEXT: s_and_b64 s[12:13], s[16:17], exec -; GFX7-NEXT: v_cmp_o_f64_e64 s[12:13], s[4:5], v[0:1] -; GFX7-NEXT: s_cselect_b32 s6, s6, s14 -; GFX7-NEXT: s_and_b64 s[16:17], s[18:19], exec -; GFX7-NEXT: s_cselect_b32 s6, s10, s6 -; GFX7-NEXT: s_and_b64 s[10:11], s[20:21], exec -; GFX7-NEXT: s_cselect_b32 s6, s6, s14 -; GFX7-NEXT: s_and_b64 s[10:11], vcc, exec -; GFX7-NEXT: s_cselect_b32 s14, s5, s9 -; GFX7-NEXT: s_and_b64 s[10:11], s[12:13], exec -; GFX7-NEXT: s_cselect_b32 s11, s14, 0x7ff80000 -; GFX7-NEXT: s_and_b64 s[14:15], vcc, exec -; GFX7-NEXT: s_cselect_b32 s10, s4, s8 -; GFX7-NEXT: v_cmp_class_f64_e64 s[14:15], s[4:5], 64 -; GFX7-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX7-NEXT: v_cmp_class_f64_e64 s[12:13], s[8:9], 64 -; GFX7-NEXT: s_cselect_b32 s10, s10, 0 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[16:17], s[10:11], 0 -; GFX7-NEXT: s_and_b64 s[18:19], s[14:15], exec -; GFX7-NEXT: s_cselect_b32 s5, s5, s11 -; GFX7-NEXT: s_and_b64 s[18:19], s[12:13], exec -; GFX7-NEXT: s_cselect_b32 s5, s9, s5 -; GFX7-NEXT: s_and_b64 s[18:19], s[16:17], exec -; GFX7-NEXT: s_cselect_b32 s5, s5, s11 -; GFX7-NEXT: s_and_b64 s[14:15], s[14:15], exec -; GFX7-NEXT: s_cselect_b32 s4, s4, s10 -; GFX7-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX7-NEXT: s_cselect_b32 s4, s8, s4 -; GFX7-NEXT: s_and_b64 s[8:9], s[16:17], exec -; GFX7-NEXT: s_cselect_b32 s4, s4, s10 +; GFX7-NEXT: v_mov_b32_e32 v5, s9 +; GFX7-NEXT: v_max_f64 v[2:3], s[6:7], v[0:1] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, s[6:7], v[0:1] +; GFX7-NEXT: v_max_f64 v[0:1], s[4:5], v[4:5] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], s[4:5], v[4:5] +; GFX7-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v6, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[4:5] ; GFX7-NEXT: ;;#ASMSTART -; GFX7-NEXT: ; use s[4:7] +; GFX7-NEXT: ; use v[0:3] ; GFX7-NEXT: ;;#ASMEND ; GFX7-NEXT: s_setpc_b64 s[30:31] ; @@ -1232,61 +866,20 @@ define void @s_maximum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1) ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v0, s10 +; GFX8-NEXT: v_mov_b32_e32 v4, s8 ; GFX8-NEXT: v_mov_b32_e32 v1, s11 -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, s[6:7], v[0:1] -; GFX8-NEXT: v_cmp_o_f64_e64 s[12:13], s[6:7], v[0:1] -; GFX8-NEXT: v_cmp_class_f64_e64 s[18:19], s[10:11], 64 -; GFX8-NEXT: v_mov_b32_e32 v0, s8 -; GFX8-NEXT: v_mov_b32_e32 v1, s9 -; GFX8-NEXT: s_and_b64 s[14:15], vcc, exec -; GFX8-NEXT: s_cselect_b32 s16, s7, s11 -; GFX8-NEXT: s_and_b64 s[14:15], s[12:13], exec -; GFX8-NEXT: s_cselect_b32 s15, s16, 0x7ff80000 -; GFX8-NEXT: s_and_b64 s[16:17], vcc, exec -; GFX8-NEXT: s_cselect_b32 s14, s6, s10 -; GFX8-NEXT: v_cmp_class_f64_e64 s[16:17], s[6:7], 64 -; GFX8-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX8-NEXT: s_cselect_b32 s14, s14, 0 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[20:21], s[14:15], 0 -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1] -; GFX8-NEXT: s_and_b64 s[12:13], s[16:17], exec -; GFX8-NEXT: s_cselect_b32 s7, s7, s15 -; GFX8-NEXT: s_and_b64 s[12:13], s[18:19], exec -; GFX8-NEXT: s_cselect_b32 s7, s11, s7 -; GFX8-NEXT: s_and_b64 s[12:13], s[20:21], exec -; GFX8-NEXT: s_cselect_b32 s7, s7, s15 -; GFX8-NEXT: s_and_b64 s[12:13], s[16:17], exec -; GFX8-NEXT: v_cmp_o_f64_e64 s[12:13], s[4:5], v[0:1] -; GFX8-NEXT: s_cselect_b32 s6, s6, s14 -; GFX8-NEXT: s_and_b64 s[16:17], s[18:19], exec -; GFX8-NEXT: s_cselect_b32 s6, s10, s6 -; GFX8-NEXT: s_and_b64 s[10:11], s[20:21], exec -; GFX8-NEXT: s_cselect_b32 s6, s6, s14 -; GFX8-NEXT: s_and_b64 s[10:11], vcc, exec -; GFX8-NEXT: s_cselect_b32 s14, s5, s9 -; GFX8-NEXT: s_and_b64 s[10:11], s[12:13], exec -; GFX8-NEXT: s_cselect_b32 s11, s14, 0x7ff80000 -; GFX8-NEXT: s_and_b64 s[14:15], vcc, exec -; GFX8-NEXT: s_cselect_b32 s10, s4, s8 -; GFX8-NEXT: v_cmp_class_f64_e64 s[14:15], s[4:5], 64 -; GFX8-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX8-NEXT: v_cmp_class_f64_e64 s[12:13], s[8:9], 64 -; GFX8-NEXT: s_cselect_b32 s10, s10, 0 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[16:17], s[10:11], 0 -; GFX8-NEXT: s_and_b64 s[18:19], s[14:15], exec -; GFX8-NEXT: s_cselect_b32 s5, s5, s11 -; GFX8-NEXT: s_and_b64 s[18:19], s[12:13], exec -; GFX8-NEXT: s_cselect_b32 s5, s9, s5 -; GFX8-NEXT: s_and_b64 s[18:19], s[16:17], exec -; GFX8-NEXT: s_cselect_b32 s5, s5, s11 -; GFX8-NEXT: s_and_b64 s[14:15], s[14:15], exec -; GFX8-NEXT: s_cselect_b32 s4, s4, s10 -; GFX8-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX8-NEXT: s_cselect_b32 s4, s8, s4 -; GFX8-NEXT: s_and_b64 s[8:9], s[16:17], exec -; GFX8-NEXT: s_cselect_b32 s4, s4, s10 +; GFX8-NEXT: v_mov_b32_e32 v5, s9 +; GFX8-NEXT: v_max_f64 v[2:3], s[6:7], v[0:1] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, s[6:7], v[0:1] +; GFX8-NEXT: v_max_f64 v[0:1], s[4:5], v[4:5] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], s[4:5], v[4:5] +; GFX8-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v6, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[4:5] ; GFX8-NEXT: ;;#ASMSTART -; GFX8-NEXT: ; use s[4:7] +; GFX8-NEXT: ; use v[0:3] ; GFX8-NEXT: ;;#ASMEND ; GFX8-NEXT: s_setpc_b64 s[30:31] ; @@ -1294,61 +887,20 @@ define void @s_maximum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1) ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v0, s10 +; GFX9-NEXT: v_mov_b32_e32 v4, s8 ; GFX9-NEXT: v_mov_b32_e32 v1, s11 -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, s[6:7], v[0:1] -; GFX9-NEXT: v_cmp_o_f64_e64 s[12:13], s[6:7], v[0:1] -; GFX9-NEXT: v_cmp_class_f64_e64 s[18:19], s[10:11], 64 -; GFX9-NEXT: v_mov_b32_e32 v0, s8 -; GFX9-NEXT: v_mov_b32_e32 v1, s9 -; GFX9-NEXT: s_and_b64 s[14:15], vcc, exec -; GFX9-NEXT: s_cselect_b32 s16, s7, s11 -; GFX9-NEXT: s_and_b64 s[14:15], s[12:13], exec -; GFX9-NEXT: s_cselect_b32 s15, s16, 0x7ff80000 -; GFX9-NEXT: s_and_b64 s[16:17], vcc, exec -; GFX9-NEXT: s_cselect_b32 s14, s6, s10 -; GFX9-NEXT: v_cmp_class_f64_e64 s[16:17], s[6:7], 64 -; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX9-NEXT: s_cselect_b32 s14, s14, 0 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[20:21], s[14:15], 0 -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1] -; GFX9-NEXT: s_and_b64 s[12:13], s[16:17], exec -; GFX9-NEXT: s_cselect_b32 s7, s7, s15 -; GFX9-NEXT: s_and_b64 s[12:13], s[18:19], exec -; GFX9-NEXT: s_cselect_b32 s7, s11, s7 -; GFX9-NEXT: s_and_b64 s[12:13], s[20:21], exec -; GFX9-NEXT: s_cselect_b32 s7, s7, s15 -; GFX9-NEXT: s_and_b64 s[12:13], s[16:17], exec -; GFX9-NEXT: v_cmp_o_f64_e64 s[12:13], s[4:5], v[0:1] -; GFX9-NEXT: s_cselect_b32 s6, s6, s14 -; GFX9-NEXT: s_and_b64 s[16:17], s[18:19], exec -; GFX9-NEXT: s_cselect_b32 s6, s10, s6 -; GFX9-NEXT: s_and_b64 s[10:11], s[20:21], exec -; GFX9-NEXT: s_cselect_b32 s6, s6, s14 -; GFX9-NEXT: s_and_b64 s[10:11], vcc, exec -; GFX9-NEXT: s_cselect_b32 s14, s5, s9 -; GFX9-NEXT: s_and_b64 s[10:11], s[12:13], exec -; GFX9-NEXT: s_cselect_b32 s11, s14, 0x7ff80000 -; GFX9-NEXT: s_and_b64 s[14:15], vcc, exec -; GFX9-NEXT: s_cselect_b32 s10, s4, s8 -; GFX9-NEXT: v_cmp_class_f64_e64 s[14:15], s[4:5], 64 -; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX9-NEXT: v_cmp_class_f64_e64 s[12:13], s[8:9], 64 -; GFX9-NEXT: s_cselect_b32 s10, s10, 0 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[16:17], s[10:11], 0 -; GFX9-NEXT: s_and_b64 s[18:19], s[14:15], exec -; GFX9-NEXT: s_cselect_b32 s5, s5, s11 -; GFX9-NEXT: s_and_b64 s[18:19], s[12:13], exec -; GFX9-NEXT: s_cselect_b32 s5, s9, s5 -; GFX9-NEXT: s_and_b64 s[18:19], s[16:17], exec -; GFX9-NEXT: s_cselect_b32 s5, s5, s11 -; GFX9-NEXT: s_and_b64 s[14:15], s[14:15], exec -; GFX9-NEXT: s_cselect_b32 s4, s4, s10 -; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX9-NEXT: s_cselect_b32 s4, s8, s4 -; GFX9-NEXT: s_and_b64 s[8:9], s[16:17], exec -; GFX9-NEXT: s_cselect_b32 s4, s4, s10 +; GFX9-NEXT: v_mov_b32_e32 v5, s9 +; GFX9-NEXT: v_max_f64 v[2:3], s[6:7], v[0:1] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, s[6:7], v[0:1] +; GFX9-NEXT: v_max_f64 v[0:1], s[4:5], v[4:5] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], s[4:5], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v6, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[4:5] ; GFX9-NEXT: ;;#ASMSTART -; GFX9-NEXT: ; use s[4:7] +; GFX9-NEXT: ; use v[0:3] ; GFX9-NEXT: ;;#ASMEND ; GFX9-NEXT: s_setpc_b64 s[30:31] ; @@ -1356,179 +908,52 @@ define void @s_maximum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1) ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX940-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, s[2:3], v[0:1] -; GFX940-NEXT: s_and_b64 s[8:9], vcc, exec -; GFX940-NEXT: v_cmp_o_f64_e64 s[8:9], s[2:3], v[0:1] -; GFX940-NEXT: s_cselect_b32 s12, s3, s7 -; GFX940-NEXT: s_and_b64 s[10:11], s[8:9], exec -; GFX940-NEXT: s_cselect_b32 s11, s12, 0x7ff80000 -; GFX940-NEXT: s_and_b64 s[12:13], vcc, exec -; GFX940-NEXT: s_cselect_b32 s10, s2, s6 -; GFX940-NEXT: s_and_b64 s[8:9], s[8:9], exec -; GFX940-NEXT: v_cmp_class_f64_e64 s[12:13], s[2:3], 64 -; GFX940-NEXT: s_cselect_b32 s10, s10, 0 -; GFX940-NEXT: s_and_b64 s[14:15], s[12:13], exec -; GFX940-NEXT: v_cmp_class_f64_e64 s[14:15], s[6:7], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[8:9], s[10:11], 0 -; GFX940-NEXT: s_cselect_b32 s3, s3, s11 -; GFX940-NEXT: s_and_b64 s[16:17], s[14:15], exec -; GFX940-NEXT: s_cselect_b32 s3, s7, s3 -; GFX940-NEXT: s_and_b64 s[16:17], s[8:9], exec -; GFX940-NEXT: s_cselect_b32 s7, s3, s11 -; GFX940-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX940-NEXT: s_cselect_b32 s11, s2, s10 -; GFX940-NEXT: s_and_b64 s[2:3], s[14:15], exec +; GFX940-NEXT: v_max_f64 v[2:3], s[2:3], v[0:1] +; GFX940-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, s[2:3], v[0:1] ; GFX940-NEXT: v_mov_b64_e32 v[0:1], s[4:5] -; GFX940-NEXT: s_cselect_b32 s6, s6, s11 -; GFX940-NEXT: s_and_b64 s[2:3], s[8:9], exec -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, s[0:1], v[0:1] -; GFX940-NEXT: s_cselect_b32 s6, s6, s10 -; GFX940-NEXT: s_and_b64 s[2:3], vcc, exec -; GFX940-NEXT: v_cmp_o_f64_e64 s[2:3], s[0:1], v[0:1] -; GFX940-NEXT: s_cselect_b32 s10, s1, s5 -; GFX940-NEXT: s_and_b64 s[8:9], s[2:3], exec -; GFX940-NEXT: s_cselect_b32 s9, s10, 0x7ff80000 -; GFX940-NEXT: s_and_b64 s[10:11], vcc, exec -; GFX940-NEXT: s_cselect_b32 s8, s0, s4 -; GFX940-NEXT: s_and_b64 s[2:3], s[2:3], exec -; GFX940-NEXT: v_cmp_class_f64_e64 s[10:11], s[0:1], 64 -; GFX940-NEXT: s_cselect_b32 s8, s8, 0 -; GFX940-NEXT: s_and_b64 s[12:13], s[10:11], exec -; GFX940-NEXT: v_cmp_class_f64_e64 s[12:13], s[4:5], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], s[8:9], 0 -; GFX940-NEXT: s_cselect_b32 s1, s1, s9 -; GFX940-NEXT: s_and_b64 s[14:15], s[12:13], exec -; GFX940-NEXT: s_cselect_b32 s1, s5, s1 -; GFX940-NEXT: s_and_b64 s[14:15], s[2:3], exec -; GFX940-NEXT: s_cselect_b32 s5, s1, s9 -; GFX940-NEXT: s_and_b64 s[10:11], s[10:11], exec -; GFX940-NEXT: s_cselect_b32 s9, s0, s8 -; GFX940-NEXT: s_and_b64 s[0:1], s[12:13], exec -; GFX940-NEXT: s_cselect_b32 s4, s4, s9 -; GFX940-NEXT: s_and_b64 s[0:1], s[2:3], exec -; GFX940-NEXT: s_cselect_b32 s4, s4, s8 +; GFX940-NEXT: v_max_f64 v[4:5], s[0:1], v[0:1] +; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc +; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, s[0:1], v[0:1] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc +; GFX940-NEXT: v_cndmask_b32_e64 v0, v4, 0, vcc ; GFX940-NEXT: ;;#ASMSTART -; GFX940-NEXT: ; use s[4:7] +; GFX940-NEXT: ; use v[0:3] ; GFX940-NEXT: ;;#ASMEND ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: s_maximum_v2f64: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e64 s12, s[6:7], s[10:11] -; GFX10-NEXT: v_cmp_o_f64_e64 s14, s[6:7], s[10:11] -; GFX10-NEXT: v_cmp_class_f64_e64 s15, s[6:7], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s16, s[10:11], 64 -; GFX10-NEXT: v_cmp_o_f64_e64 s18, s[4:5], s[8:9] -; GFX10-NEXT: v_cmp_class_f64_e64 s19, s[4:5], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s20, s[8:9], 64 -; GFX10-NEXT: s_and_b32 s13, s12, exec_lo -; GFX10-NEXT: s_cselect_b32 s13, s7, s11 -; GFX10-NEXT: s_and_b32 s17, s14, exec_lo -; GFX10-NEXT: s_cselect_b32 s13, s13, 0x7ff80000 -; GFX10-NEXT: s_and_b32 s12, s12, exec_lo -; GFX10-NEXT: s_cselect_b32 s12, s6, s10 -; GFX10-NEXT: s_and_b32 s14, s14, exec_lo -; GFX10-NEXT: s_cselect_b32 s12, s12, 0 -; GFX10-NEXT: v_cmp_gt_f64_e64 s17, s[4:5], s[8:9] -; GFX10-NEXT: v_cmp_eq_f64_e64 s14, s[12:13], 0 -; GFX10-NEXT: s_and_b32 s21, s15, exec_lo -; GFX10-NEXT: s_cselect_b32 s7, s7, s13 -; GFX10-NEXT: s_and_b32 s21, s16, exec_lo -; GFX10-NEXT: s_cselect_b32 s7, s11, s7 -; GFX10-NEXT: s_and_b32 s11, s14, exec_lo -; GFX10-NEXT: s_cselect_b32 s7, s7, s13 -; GFX10-NEXT: s_and_b32 s11, s15, exec_lo -; GFX10-NEXT: s_cselect_b32 s6, s6, s12 -; GFX10-NEXT: s_and_b32 s11, s16, exec_lo -; GFX10-NEXT: s_cselect_b32 s6, s10, s6 -; GFX10-NEXT: s_and_b32 s10, s14, exec_lo -; GFX10-NEXT: s_cselect_b32 s6, s6, s12 -; GFX10-NEXT: s_and_b32 s10, s17, exec_lo -; GFX10-NEXT: s_cselect_b32 s10, s5, s9 -; GFX10-NEXT: s_and_b32 s11, s18, exec_lo -; GFX10-NEXT: s_cselect_b32 s11, s10, 0x7ff80000 -; GFX10-NEXT: s_and_b32 s10, s17, exec_lo -; GFX10-NEXT: s_cselect_b32 s10, s4, s8 -; GFX10-NEXT: s_and_b32 s12, s18, exec_lo -; GFX10-NEXT: s_cselect_b32 s10, s10, 0 -; GFX10-NEXT: s_and_b32 s13, s19, exec_lo -; GFX10-NEXT: v_cmp_eq_f64_e64 s12, s[10:11], 0 -; GFX10-NEXT: s_cselect_b32 s5, s5, s11 -; GFX10-NEXT: s_and_b32 s13, s20, exec_lo -; GFX10-NEXT: s_cselect_b32 s5, s9, s5 -; GFX10-NEXT: s_and_b32 s9, s12, exec_lo -; GFX10-NEXT: s_cselect_b32 s5, s5, s11 -; GFX10-NEXT: s_and_b32 s9, s19, exec_lo -; GFX10-NEXT: s_cselect_b32 s4, s4, s10 -; GFX10-NEXT: s_and_b32 s9, s20, exec_lo -; GFX10-NEXT: s_cselect_b32 s4, s8, s4 -; GFX10-NEXT: s_and_b32 s8, s12, exec_lo -; GFX10-NEXT: s_cselect_b32 s4, s4, s10 +; GFX10-NEXT: v_max_f64 v[0:1], s[6:7], s[10:11] +; GFX10-NEXT: v_cmp_u_f64_e64 s6, s[6:7], s[10:11] +; GFX10-NEXT: v_max_f64 v[4:5], s[4:5], s[8:9] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, s[4:5], s[8:9] +; GFX10-NEXT: v_cndmask_b32_e64 v3, v1, 0x7ff80000, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v2, v0, 0, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v1, v5, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v0, v4, 0, s4 ; GFX10-NEXT: ;;#ASMSTART -; GFX10-NEXT: ; use s[4:7] +; GFX10-NEXT: ; use v[0:3] ; GFX10-NEXT: ;;#ASMEND ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: s_maximum_v2f64: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e64 s8, s[2:3], s[6:7] -; GFX11-NEXT: v_cmp_o_f64_e64 s10, s[2:3], s[6:7] -; GFX11-NEXT: v_cmp_class_f64_e64 s11, s[2:3], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s12, s[6:7], 64 -; GFX11-NEXT: v_cmp_o_f64_e64 s14, s[0:1], s[4:5] -; GFX11-NEXT: v_cmp_class_f64_e64 s15, s[0:1], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s16, s[4:5], 64 -; GFX11-NEXT: s_and_b32 s9, s8, exec_lo -; GFX11-NEXT: s_cselect_b32 s9, s3, s7 -; GFX11-NEXT: s_and_b32 s13, s10, exec_lo -; GFX11-NEXT: s_cselect_b32 s9, s9, 0x7ff80000 -; GFX11-NEXT: s_and_b32 s8, s8, exec_lo -; GFX11-NEXT: s_cselect_b32 s8, s2, s6 -; GFX11-NEXT: s_and_b32 s10, s10, exec_lo -; GFX11-NEXT: s_cselect_b32 s8, s8, 0 -; GFX11-NEXT: v_cmp_gt_f64_e64 s13, s[0:1], s[4:5] -; GFX11-NEXT: v_cmp_eq_f64_e64 s10, s[8:9], 0 -; GFX11-NEXT: s_and_b32 s17, s11, exec_lo -; GFX11-NEXT: s_cselect_b32 s3, s3, s9 -; GFX11-NEXT: s_and_b32 s17, s12, exec_lo -; GFX11-NEXT: s_cselect_b32 s3, s7, s3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: s_and_b32 s7, s10, exec_lo -; GFX11-NEXT: s_cselect_b32 s3, s3, s9 -; GFX11-NEXT: s_and_b32 s7, s11, exec_lo -; GFX11-NEXT: s_cselect_b32 s2, s2, s8 -; GFX11-NEXT: s_and_b32 s7, s12, exec_lo -; GFX11-NEXT: s_cselect_b32 s2, s6, s2 -; GFX11-NEXT: s_and_b32 s6, s10, exec_lo -; GFX11-NEXT: s_cselect_b32 s2, s2, s8 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: s_and_b32 s6, s13, exec_lo -; GFX11-NEXT: s_cselect_b32 s6, s1, s5 -; GFX11-NEXT: s_and_b32 s7, s14, exec_lo -; GFX11-NEXT: s_cselect_b32 s7, s6, 0x7ff80000 -; GFX11-NEXT: s_and_b32 s6, s13, exec_lo -; GFX11-NEXT: s_cselect_b32 s6, s0, s4 -; GFX11-NEXT: s_and_b32 s8, s14, exec_lo -; GFX11-NEXT: s_cselect_b32 s6, s6, 0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-NEXT: s_and_b32 s9, s15, exec_lo -; GFX11-NEXT: v_cmp_eq_f64_e64 s8, s[6:7], 0 -; GFX11-NEXT: s_cselect_b32 s1, s1, s7 -; GFX11-NEXT: s_and_b32 s9, s16, exec_lo -; GFX11-NEXT: s_cselect_b32 s1, s5, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: s_and_b32 s5, s8, exec_lo -; GFX11-NEXT: s_cselect_b32 s1, s1, s7 -; GFX11-NEXT: s_and_b32 s5, s15, exec_lo -; GFX11-NEXT: s_cselect_b32 s0, s0, s6 -; GFX11-NEXT: s_and_b32 s5, s16, exec_lo -; GFX11-NEXT: s_cselect_b32 s0, s4, s0 -; GFX11-NEXT: s_and_b32 s4, s8, exec_lo -; GFX11-NEXT: s_cselect_b32 s0, s0, s6 +; GFX11-NEXT: v_max_f64 v[0:1], s[2:3], s[6:7] +; GFX11-NEXT: v_cmp_u_f64_e64 s2, s[2:3], s[6:7] +; GFX11-NEXT: v_max_f64 v[4:5], s[0:1], s[4:5] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, s[0:1], s[4:5] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_cndmask_b32_e64 v3, v1, 0x7ff80000, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v2, v0, 0, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v1, v5, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v0, v4, 0, s0 ; GFX11-NEXT: ;;#ASMSTART -; GFX11-NEXT: ; use s[0:3] +; GFX11-NEXT: ; use v[0:3] ; GFX11-NEXT: ;;#ASMEND ; GFX11-NEXT: s_setpc_b64 s[30:31] ; @@ -1554,306 +979,110 @@ define <3 x double> @v_maximum_v3f64(<3 x double> %src0, <3 x double> %src1) { ; GFX7-LABEL: v_maximum_v3f64: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7] -; GFX7-NEXT: v_mov_b32_e32 v14, 0x7ff80000 -; GFX7-NEXT: v_cmp_gt_f64_e64 s[8:9], v[2:3], v[8:9] -; GFX7-NEXT: v_cmp_o_f64_e64 s[10:11], v[2:3], v[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v12, v7, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v12, 0, v12, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 64 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[12:13] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v9, v3, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[2:3], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v7, v14, v6, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v8, v2, s[8:9] -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11] -; GFX7-NEXT: v_cmp_class_f64_e64 s[8:9], v[8:9], 64 -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[4:5], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v6, 0, v6, s[10:11] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v12, v11, v5, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v8, v10, v4, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v12, 0, v8, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[10:11], 64 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[12:13] +; GFX7-NEXT: v_max_f64 v[12:13], v[0:1], v[6:7] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX7-NEXT: v_max_f64 v[6:7], v[2:3], v[8:9] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9] +; GFX7-NEXT: v_max_f64 v[8:9], v[4:5], v[10:11] +; GFX7-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11] +; GFX7-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v13, v5, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v5, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v4, v8, 0, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v3f64: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7] -; GFX8-NEXT: v_mov_b32_e32 v14, 0x7ff80000 -; GFX8-NEXT: v_cmp_gt_f64_e64 s[8:9], v[2:3], v[8:9] -; GFX8-NEXT: v_cmp_o_f64_e64 s[10:11], v[2:3], v[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v12, v7, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v12, 0, v12, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 64 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[12:13] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v9, v3, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[2:3], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v7, v14, v6, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v8, v2, s[8:9] -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11] -; GFX8-NEXT: v_cmp_class_f64_e64 s[8:9], v[8:9], 64 -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[4:5], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, v6, s[10:11] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v12, v11, v5, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v8, v10, v4, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v12, 0, v8, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[10:11], 64 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[12:13] +; GFX8-NEXT: v_max_f64 v[12:13], v[0:1], v[6:7] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[8:9] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9] +; GFX8-NEXT: v_max_f64 v[8:9], v[4:5], v[10:11] +; GFX8-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11] +; GFX8-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v13, v5, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v5, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, 0, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v3f64: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7] -; GFX9-NEXT: v_mov_b32_e32 v14, 0x7ff80000 -; GFX9-NEXT: v_cmp_gt_f64_e64 s[8:9], v[2:3], v[8:9] -; GFX9-NEXT: v_cmp_o_f64_e64 s[10:11], v[2:3], v[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v12, v7, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, v12, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 64 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[12:13] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v9, v3, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[2:3], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v7, v14, v6, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v8, v2, s[8:9] -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11] -; GFX9-NEXT: v_cmp_class_f64_e64 s[8:9], v[8:9], 64 -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[4:5], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, v6, s[10:11] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v12, v11, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v8, v10, v4, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, v8, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[10:11], 64 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[12:13] +; GFX9-NEXT: v_max_f64 v[12:13], v[0:1], v[6:7] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX9-NEXT: v_max_f64 v[6:7], v[2:3], v[8:9] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9] +; GFX9-NEXT: v_max_f64 v[8:9], v[4:5], v[10:11] +; GFX9-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11] +; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v13, v5, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v5, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, 0, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v3f64: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX940-NEXT: v_mov_b32_e32 v14, 0x7ff80000 -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[6:7] -; GFX940-NEXT: v_cndmask_b32_e32 v12, v7, v1, vcc -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v12, 0, v12, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[6:7], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[12:13] -; GFX940-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v3, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[8:9] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v14, v6, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v8, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v6, 0, v6, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[8:9], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[6:7] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v11, v5, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[4:5], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[2:3] +; GFX940-NEXT: v_max_f64 v[12:13], v[0:1], v[6:7] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX940-NEXT: v_max_f64 v[6:7], v[2:3], v[8:9] ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v7, v14, v6, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v10, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v6, 0, v6, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[10:11], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[6:7] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v4, v6, v4, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[2:3] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX940-NEXT: v_mov_b32_e32 v12, 0x7ff80000 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v13, v12, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[8:9] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v2, v6, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v12, vcc +; GFX940-NEXT: v_max_f64 v[6:7], v[4:5], v[10:11] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[4:5], v[10:11] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v4, v6, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v5, v7, v12, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v3f64: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[2:3], v[8:9] -; GFX10-NEXT: v_cmp_gt_f64_e64 s5, v[4:5], v[10:11] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[0:1], v[6:7] -; GFX10-NEXT: v_cmp_o_f64_e64 s7, v[2:3], v[8:9] -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[4:5], v[10:11] -; GFX10-NEXT: v_cndmask_b32_e32 v12, v7, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v14, v9, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v16, v11, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e32 v17, v6, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v18, v8, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v19, v10, v4, s5 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[2:3], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[4:5], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v13, 0x7ff80000, v12, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v15, 0x7ff80000, v14, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, v17, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v17, 0x7ff80000, v16, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v14, 0, v18, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v16, 0, v19, s8 -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[8:9], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s7, v[6:7], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s8, v[10:11], 64 -; GFX10-NEXT: v_cmp_eq_f64_e64 s9, 0, v[12:13] -; GFX10-NEXT: v_cmp_eq_f64_e64 s10, 0, v[14:15] -; GFX10-NEXT: v_cmp_eq_f64_e64 s11, 0, v[16:17] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v14, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v16, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v15, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v17, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v8, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v6, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v10, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v7, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v9, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v11, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v12, v0, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v14, v2, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v16, v4, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v13, v1, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v15, v3, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v17, v5, s11 +; GFX10-NEXT: v_max_f64 v[12:13], v[0:1], v[6:7] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7] +; GFX10-NEXT: v_max_f64 v[6:7], v[2:3], v[8:9] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[8:9] +; GFX10-NEXT: v_max_f64 v[8:9], v[4:5], v[10:11] +; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[4:5], v[10:11] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v4, v8, 0, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v9, 0x7ff80000, s5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v3f64: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX11-NEXT: v_cmp_gt_f64_e64 s0, v[2:3], v[8:9] -; GFX11-NEXT: v_cmp_gt_f64_e64 s1, v[4:5], v[10:11] -; GFX11-NEXT: v_cmp_o_f64_e64 s2, v[0:1], v[6:7] -; GFX11-NEXT: v_cmp_o_f64_e64 s3, v[2:3], v[8:9] -; GFX11-NEXT: v_cmp_o_f64_e64 s4, v[4:5], v[10:11] -; GFX11-NEXT: v_dual_cndmask_b32 v12, v7, v1 :: v_dual_cndmask_b32 v17, v6, v0 -; GFX11-NEXT: v_cndmask_b32_e64 v14, v9, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v16, v11, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v18, v8, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v19, v10, v4, s1 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[4:5], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v13, 0x7ff80000, v12, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v15, 0x7ff80000, v14, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v12, 0, v17, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v17, 0x7ff80000, v16, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v14, 0, v18, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v16, 0, v19, s4 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[8:9], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s3, v[6:7], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s4, v[10:11], 64 -; GFX11-NEXT: v_cmp_eq_f64_e64 s5, 0, v[12:13] -; GFX11-NEXT: v_cmp_eq_f64_e64 s6, 0, v[14:15] -; GFX11-NEXT: v_cmp_eq_f64_e64 s7, 0, v[16:17] -; GFX11-NEXT: v_dual_cndmask_b32 v0, v12, v0 :: v_dual_cndmask_b32 v1, v13, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v14, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v16, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v15, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v17, v5, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v8, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v6, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v10, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v7, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v9, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v11, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v12, v0, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v14, v2, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v16, v4, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v13, v1, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v15, v3, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v17, v5, s7 +; GFX11-NEXT: v_max_f64 v[12:13], v[0:1], v[6:7] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7] +; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[8:9] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[8:9] +; GFX11-NEXT: v_max_f64 v[8:9], v[4:5], v[10:11] +; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[10:11] +; GFX11-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v6, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v4, v8, 0, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v5, v9, 0x7ff80000, s1 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v3f64: @@ -1875,247 +1104,49 @@ define <3 x double> @v_maximum_v3f64__nnan(<3 x double> %src0, <3 x double> %src ; GFX7-LABEL: v_maximum_v3f64__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 64 -; GFX7-NEXT: v_cmp_gt_f64_e64 s[8:9], v[2:3], v[8:9] -; GFX7-NEXT: v_cmp_class_f64_e64 s[10:11], v[10:11], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v13, v7, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[12:13] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[2:3], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v7, v9, v3, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v8, v2, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v13, v11, v5, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v12, v10, v4, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[8:9], 64 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[6:7] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[12:13] +; GFX7-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7] +; GFX7-NEXT: v_max_f64 v[2:3], v[2:3], v[8:9] +; GFX7-NEXT: v_max_f64 v[4:5], v[4:5], v[10:11] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v3f64__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 64 -; GFX8-NEXT: v_cmp_gt_f64_e64 s[8:9], v[2:3], v[8:9] -; GFX8-NEXT: v_cmp_class_f64_e64 s[10:11], v[10:11], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v13, v7, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[12:13] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[2:3], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v7, v9, v3, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v8, v2, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v13, v11, v5, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v12, v10, v4, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[8:9], 64 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[6:7] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[12:13] +; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7] +; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[8:9] +; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[10:11] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v3f64__nnan: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 64 -; GFX9-NEXT: v_cmp_gt_f64_e64 s[8:9], v[2:3], v[8:9] -; GFX9-NEXT: v_cmp_class_f64_e64 s[10:11], v[10:11], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v13, v7, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[12:13] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[2:3], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v7, v9, v3, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v8, v2, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v13, v11, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v12, v10, v4, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[8:9], 64 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[6:7] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[12:13] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7] +; GFX9-NEXT: v_max_f64 v[2:3], v[2:3], v[8:9] +; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[10:11] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v3f64__nnan: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[6:7], 64 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v13, v7, v1, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[12:13] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v7, v9, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v6, v8, v2, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[8:9], 64 -; GFX940-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[0:1] -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[6:7] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[0:1] -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e32 v7, v11, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v6, v10, v4, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[10:11], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[6:7] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v4, v6, v4, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[2:3] +; GFX940-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7] +; GFX940-NEXT: v_max_f64 v[2:3], v[2:3], v[8:9] +; GFX940-NEXT: v_max_f64 v[4:5], v[4:5], v[10:11] ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v3f64__nnan: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[2:3], v[8:9] -; GFX10-NEXT: v_cmp_gt_f64_e64 s5, v[4:5], v[10:11] -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[8:9], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s7, v[6:7], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s8, v[10:11], 64 -; GFX10-NEXT: v_cndmask_b32_e32 v13, v7, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v15, v9, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v17, v11, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v14, v8, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v16, v10, v4, s5 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[2:3], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[4:5], 64 -; GFX10-NEXT: v_cmp_eq_f64_e64 s9, 0, v[12:13] -; GFX10-NEXT: v_cmp_eq_f64_e64 s10, 0, v[14:15] -; GFX10-NEXT: v_cmp_eq_f64_e64 s11, 0, v[16:17] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v14, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v16, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v15, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v17, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v6, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v8, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v10, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v7, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v9, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v11, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v12, v0, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v14, v2, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v16, v4, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v13, v1, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v15, v3, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v17, v5, s11 +; GFX10-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7] +; GFX10-NEXT: v_max_f64 v[2:3], v[2:3], v[8:9] +; GFX10-NEXT: v_max_f64 v[4:5], v[4:5], v[10:11] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v3f64__nnan: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX11-NEXT: v_cmp_gt_f64_e64 s0, v[2:3], v[8:9] -; GFX11-NEXT: v_cmp_gt_f64_e64 s1, v[4:5], v[10:11] -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[8:9], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s3, v[6:7], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s4, v[10:11], 64 -; GFX11-NEXT: v_dual_cndmask_b32 v13, v7, v1 :: v_dual_cndmask_b32 v12, v6, v0 -; GFX11-NEXT: v_cndmask_b32_e64 v15, v9, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v17, v11, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v14, v8, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v16, v10, v4, s1 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[4:5], 64 -; GFX11-NEXT: v_cmp_eq_f64_e64 s5, 0, v[12:13] -; GFX11-NEXT: v_cmp_eq_f64_e64 s6, 0, v[14:15] -; GFX11-NEXT: v_cmp_eq_f64_e64 s7, 0, v[16:17] -; GFX11-NEXT: v_dual_cndmask_b32 v0, v12, v0 :: v_dual_cndmask_b32 v1, v13, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v14, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v16, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v15, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v17, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v6, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v8, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v10, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v7, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v9, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v11, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v12, v0, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v14, v2, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v16, v4, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v13, v1, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v15, v3, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v17, v5, s7 +; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7] +; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[8:9] +; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[10:11] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v3f64__nnan: @@ -2137,144 +1168,110 @@ define <3 x double> @v_maximum_v3f64__nsz(<3 x double> %src0, <3 x double> %src1 ; GFX7-LABEL: v_maximum_v3f64__nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[8:9], v[4:5], v[10:11] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[2:3], v[8:9] -; GFX7-NEXT: v_cmp_o_f64_e64 s[10:11], v[4:5], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v10, v4, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v11, v5, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v0, 0, v12, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v4, 0, v7, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v8, v2, vcc -; GFX7-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v2, 0, v6, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[10:11] +; GFX7-NEXT: v_max_f64 v[12:13], v[0:1], v[6:7] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX7-NEXT: v_max_f64 v[6:7], v[2:3], v[8:9] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9] +; GFX7-NEXT: v_max_f64 v[8:9], v[4:5], v[10:11] +; GFX7-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11] +; GFX7-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v13, v5, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v5, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v4, v8, 0, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v3f64__nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[8:9], v[4:5], v[10:11] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[2:3], v[8:9] -; GFX8-NEXT: v_cmp_o_f64_e64 s[10:11], v[4:5], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v10, v4, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v11, v5, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, v12, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v4, 0, v7, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v8, v2, vcc -; GFX8-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, v6, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[10:11] +; GFX8-NEXT: v_max_f64 v[12:13], v[0:1], v[6:7] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[8:9] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9] +; GFX8-NEXT: v_max_f64 v[8:9], v[4:5], v[10:11] +; GFX8-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11] +; GFX8-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v13, v5, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v5, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, 0, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v3f64__nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[8:9], v[4:5], v[10:11] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[2:3], v[8:9] -; GFX9-NEXT: v_cmp_o_f64_e64 s[10:11], v[4:5], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v10, v4, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v11, v5, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, v12, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, v7, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v8, v2, vcc -; GFX9-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, v6, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[10:11] +; GFX9-NEXT: v_max_f64 v[12:13], v[0:1], v[6:7] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX9-NEXT: v_max_f64 v[6:7], v[2:3], v[8:9] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9] +; GFX9-NEXT: v_max_f64 v[8:9], v[4:5], v[10:11] +; GFX9-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11] +; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v13, v5, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v5, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, 0, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v3f64__nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[6:7] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX940-NEXT: v_mov_b32_e32 v6, 0x7ff80000 -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9] -; GFX940-NEXT: v_cndmask_b32_e64 v0, 0, v12, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v6, v1, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v7, v8, v2, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v2, 0, v7, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v3, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v7, v10, v4, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[4:5], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e32 v5, v11, v5, vcc +; GFX940-NEXT: v_max_f64 v[12:13], v[0:1], v[6:7] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX940-NEXT: v_max_f64 v[6:7], v[2:3], v[8:9] ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v4, 0, v7, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v6, v5, s[0:1] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX940-NEXT: v_mov_b32_e32 v12, 0x7ff80000 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v13, v12, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[8:9] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v2, v6, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v12, vcc +; GFX940-NEXT: v_max_f64 v[6:7], v[4:5], v[10:11] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[4:5], v[10:11] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v4, v6, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v5, v7, v12, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v3f64__nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[2:3], v[8:9] -; GFX10-NEXT: v_cmp_gt_f64_e64 s5, v[4:5], v[10:11] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[0:1], v[6:7] -; GFX10-NEXT: v_cmp_o_f64_e64 s7, v[2:3], v[8:9] -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[4:5], v[10:11] -; GFX10-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v6, v8, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v8, v10, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v9, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v11, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, v12, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, v6, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, v8, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v1, 0x7ff80000, v1, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v3, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v5, s8 +; GFX10-NEXT: v_max_f64 v[12:13], v[0:1], v[6:7] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7] +; GFX10-NEXT: v_max_f64 v[6:7], v[2:3], v[8:9] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[8:9] +; GFX10-NEXT: v_max_f64 v[8:9], v[4:5], v[10:11] +; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[4:5], v[10:11] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v4, v8, 0, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v9, 0x7ff80000, s5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v3f64__nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX11-NEXT: v_cmp_gt_f64_e64 s0, v[2:3], v[8:9] -; GFX11-NEXT: v_cmp_gt_f64_e64 s1, v[4:5], v[10:11] -; GFX11-NEXT: v_cmp_o_f64_e64 s2, v[0:1], v[6:7] -; GFX11-NEXT: v_cmp_o_f64_e64 s3, v[2:3], v[8:9] -; GFX11-NEXT: v_cmp_o_f64_e64 s4, v[4:5], v[10:11] -; GFX11-NEXT: v_dual_cndmask_b32 v12, v6, v0 :: v_dual_cndmask_b32 v1, v7, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v8, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v8, v10, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v9, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v11, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, v12, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, v6, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v4, 0, v8, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v1, 0x7ff80000, v1, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v3, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v5, s4 +; GFX11-NEXT: v_max_f64 v[12:13], v[0:1], v[6:7] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7] +; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[8:9] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[8:9] +; GFX11-NEXT: v_max_f64 v[8:9], v[4:5], v[10:11] +; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[10:11] +; GFX11-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v6, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v4, v8, 0, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v5, v9, 0x7ff80000, s1 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v3f64__nsz: @@ -2296,88 +1293,49 @@ define <3 x double> @v_maximum_v3f64__nnan_nsz(<3 x double> %src0, <3 x double> ; GFX7-LABEL: v_maximum_v3f64__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[4:5], v[2:3], v[8:9] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[6:7], v[4:5], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v10, v4, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v11, v5, s[6:7] +; GFX7-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7] +; GFX7-NEXT: v_max_f64 v[2:3], v[2:3], v[8:9] +; GFX7-NEXT: v_max_f64 v[4:5], v[4:5], v[10:11] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v3f64__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[4:5], v[2:3], v[8:9] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[6:7], v[4:5], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v10, v4, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v11, v5, s[6:7] +; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7] +; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[8:9] +; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[10:11] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v3f64__nnan_nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[4:5], v[2:3], v[8:9] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[6:7], v[4:5], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v10, v4, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v11, v5, s[6:7] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7] +; GFX9-NEXT: v_max_f64 v[2:3], v[2:3], v[8:9] +; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[10:11] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v3f64__nnan_nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[6:7] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[8:9] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[10:11] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v10, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v11, v5, vcc +; GFX940-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7] +; GFX940-NEXT: v_max_f64 v[2:3], v[2:3], v[8:9] +; GFX940-NEXT: v_max_f64 v[4:5], v[4:5], v[10:11] ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v3f64__nnan_nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[2:3], v[8:9] -; GFX10-NEXT: v_cmp_gt_f64_e64 s5, v[4:5], v[10:11] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v8, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v10, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v9, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v11, v5, s5 +; GFX10-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7] +; GFX10-NEXT: v_max_f64 v[2:3], v[2:3], v[8:9] +; GFX10-NEXT: v_max_f64 v[4:5], v[4:5], v[10:11] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v3f64__nnan_nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX11-NEXT: v_cmp_gt_f64_e64 s0, v[2:3], v[8:9] -; GFX11-NEXT: v_cmp_gt_f64_e64 s1, v[4:5], v[10:11] -; GFX11-NEXT: v_dual_cndmask_b32 v0, v6, v0 :: v_dual_cndmask_b32 v1, v7, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v8, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v10, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v9, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v11, v5, s1 +; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7] +; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[8:9] +; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[10:11] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v3f64__nnan_nsz: @@ -2399,404 +1357,135 @@ define <4 x double> @v_maximum_v4f64(<4 x double> %src0, <4 x double> %src1) { ; GFX7-LABEL: v_maximum_v4f64: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9] -; GFX7-NEXT: v_mov_b32_e32 v18, 0x7ff80000 -; GFX7-NEXT: v_cmp_gt_f64_e64 s[6:7], v[2:3], v[10:11] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[2:3], v[10:11] -; GFX7-NEXT: v_cmp_o_f64_e64 s[12:13], v[4:5], v[12:13] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v9, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v17, v18, v16, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[8:9], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v19, v11, v3, s[6:7] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[16:17] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[10:11], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v8, v10, v2, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v18, v19, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[8:9] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[8:9], v[4:5], v[12:13] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX7-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[4:5] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[6:7], v[14:15] -; GFX7-NEXT: v_cndmask_b32_e64 v10, v13, v5, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v18, v10, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e32 v10, v15, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v8, v12, v4, s[8:9] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[4:5], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v11, v18, v10, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v10, v14, v6, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[12:13] -; GFX7-NEXT: v_cmp_class_f64_e64 s[8:9], v[12:13], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v10, 0, v10, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[14:15], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[10:11] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[8:9] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v10, v6, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v11, v7, s[12:13] +; GFX7-NEXT: v_max_f64 v[16:17], v[0:1], v[8:9] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX7-NEXT: v_max_f64 v[8:9], v[2:3], v[10:11] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11] +; GFX7-NEXT: v_max_f64 v[10:11], v[4:5], v[12:13] +; GFX7-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13] +; GFX7-NEXT: v_max_f64 v[12:13], v[6:7], v[14:15] +; GFX7-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15] +; GFX7-NEXT: v_mov_b32_e32 v7, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v17, v7, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v8, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v9, v7, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v4, v10, 0, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v11, v7, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v6, v12, 0, s[8:9] +; GFX7-NEXT: v_cndmask_b32_e64 v7, v13, v7, s[8:9] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v4f64: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9] -; GFX8-NEXT: v_mov_b32_e32 v18, 0x7ff80000 -; GFX8-NEXT: v_cmp_gt_f64_e64 s[6:7], v[2:3], v[10:11] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[2:3], v[10:11] -; GFX8-NEXT: v_cmp_o_f64_e64 s[12:13], v[4:5], v[12:13] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v9, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v17, v18, v16, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[8:9], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v19, v11, v3, s[6:7] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[16:17] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[10:11], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v8, v10, v2, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v18, v19, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[8:9] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[8:9], v[4:5], v[12:13] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX8-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[4:5] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[6:7], v[14:15] -; GFX8-NEXT: v_cndmask_b32_e64 v10, v13, v5, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v18, v10, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e32 v10, v15, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v8, v12, v4, s[8:9] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[4:5], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v11, v18, v10, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v10, v14, v6, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[12:13] -; GFX8-NEXT: v_cmp_class_f64_e64 s[8:9], v[12:13], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v10, 0, v10, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[14:15], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[10:11] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[8:9] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v10, v6, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v11, v7, s[12:13] +; GFX8-NEXT: v_max_f64 v[16:17], v[0:1], v[8:9] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[10:11] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11] +; GFX8-NEXT: v_max_f64 v[10:11], v[4:5], v[12:13] +; GFX8-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13] +; GFX8-NEXT: v_max_f64 v[12:13], v[6:7], v[14:15] +; GFX8-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15] +; GFX8-NEXT: v_mov_b32_e32 v7, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v7, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v8, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v9, v7, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v4, v10, 0, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v11, v7, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v6, v12, 0, s[8:9] +; GFX8-NEXT: v_cndmask_b32_e64 v7, v13, v7, s[8:9] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v4f64: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9] -; GFX9-NEXT: v_mov_b32_e32 v18, 0x7ff80000 -; GFX9-NEXT: v_cmp_gt_f64_e64 s[6:7], v[2:3], v[10:11] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[2:3], v[10:11] -; GFX9-NEXT: v_cmp_o_f64_e64 s[12:13], v[4:5], v[12:13] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v9, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v17, v18, v16, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[8:9], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v19, v11, v3, s[6:7] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[16:17] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[10:11], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v8, v10, v2, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v18, v19, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[8:9] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[8:9], v[4:5], v[12:13] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX9-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[4:5] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[6:7], v[14:15] -; GFX9-NEXT: v_cndmask_b32_e64 v10, v13, v5, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v18, v10, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e32 v10, v15, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v8, v12, v4, s[8:9] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[4:5], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v11, v18, v10, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v10, v14, v6, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[12:13] -; GFX9-NEXT: v_cmp_class_f64_e64 s[8:9], v[12:13], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, v10, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[14:15], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[10:11] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[8:9] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v10, v6, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v11, v7, s[12:13] +; GFX9-NEXT: v_max_f64 v[16:17], v[0:1], v[8:9] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX9-NEXT: v_max_f64 v[8:9], v[2:3], v[10:11] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11] +; GFX9-NEXT: v_max_f64 v[10:11], v[4:5], v[12:13] +; GFX9-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13] +; GFX9-NEXT: v_max_f64 v[12:13], v[6:7], v[14:15] +; GFX9-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15] +; GFX9-NEXT: v_mov_b32_e32 v7, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v7, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v8, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v9, v7, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v4, v10, 0, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v11, v7, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v6, v12, 0, s[8:9] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v13, v7, s[8:9] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v4f64: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX940-NEXT: v_mov_b32_e32 v18, 0x7ff80000 -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v9, v1, vcc -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v18, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[8:9], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v11, v3, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v9, v18, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[10:11], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[12:13] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v13, v5, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[4:5], v[12:13] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[2:3] +; GFX940-NEXT: v_max_f64 v[16:17], v[0:1], v[8:9] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX940-NEXT: v_max_f64 v[8:9], v[2:3], v[10:11] ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v9, v18, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v12, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[12:13], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v5, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15] -; GFX940-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v15, v7, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[6:7], v[14:15] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v9, v18, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v14, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[14:15], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v6, v8, v6, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[2:3] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX940-NEXT: v_mov_b32_e32 v16, 0x7ff80000 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v17, v16, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[10:11] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v2, v8, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v16, vcc +; GFX940-NEXT: v_max_f64 v[8:9], v[4:5], v[12:13] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[4:5], v[12:13] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v4, v8, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v16, vcc +; GFX940-NEXT: v_max_f64 v[8:9], v[6:7], v[14:15] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[6:7], v[14:15] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v6, v8, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v7, v9, v16, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v4f64: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[2:3], v[10:11] -; GFX10-NEXT: v_cmp_o_f64_e64 s5, v[0:1], v[8:9] -; GFX10-NEXT: v_cmp_gt_f64_e64 s6, v[4:5], v[12:13] -; GFX10-NEXT: v_cmp_o_f64_e64 s7, v[2:3], v[10:11] -; GFX10-NEXT: v_cmp_gt_f64_e64 s8, v[6:7], v[14:15] -; GFX10-NEXT: v_cmp_o_f64_e64 s9, v[4:5], v[12:13] -; GFX10-NEXT: v_cmp_o_f64_e64 s10, v[6:7], v[14:15] -; GFX10-NEXT: v_cmp_class_f64_e64 s11, v[14:15], 64 -; GFX10-NEXT: v_cndmask_b32_e32 v16, v9, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v18, v11, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v19, v8, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v21, v10, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v20, v13, v5, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v22, v15, v7, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v17, 0x7ff80000, v16, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v16, 0, v19, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v19, 0x7ff80000, v18, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v23, v12, v4, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v18, 0, v21, s7 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v24, v14, v6, s8 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[2:3], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s7, v[4:5], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s8, v[6:7], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[8:9], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v21, 0x7ff80000, v20, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v20, 0, v23, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v23, 0x7ff80000, v22, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v22, 0, v24, s10 -; GFX10-NEXT: v_cmp_class_f64_e64 s9, v[10:11], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[12:13], 64 -; GFX10-NEXT: v_cmp_eq_f64_e64 s6, 0, v[16:17] -; GFX10-NEXT: v_cmp_eq_f64_e64 s12, 0, v[18:19] -; GFX10-NEXT: v_cmp_eq_f64_e64 s13, 0, v[20:21] -; GFX10-NEXT: v_cmp_eq_f64_e64 s14, 0, v[22:23] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v18, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v20, v4, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v22, v6, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v19, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v21, v5, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v23, v7, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v8, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v14, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v10, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v12, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v9, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v11, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v13, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v15, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v16, v0, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v17, v1, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v18, v2, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v19, v3, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v20, v4, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v21, v5, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v22, v6, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v23, v7, s14 +; GFX10-NEXT: v_max_f64 v[16:17], v[0:1], v[8:9] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9] +; GFX10-NEXT: v_max_f64 v[8:9], v[2:3], v[10:11] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[10:11] +; GFX10-NEXT: v_max_f64 v[10:11], v[4:5], v[12:13] +; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[4:5], v[12:13] +; GFX10-NEXT: v_max_f64 v[12:13], v[6:7], v[14:15] +; GFX10-NEXT: v_cmp_u_f64_e64 s6, v[6:7], v[14:15] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v8, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v9, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v4, v10, 0, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v11, 0x7ff80000, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v6, v12, 0, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v7, v13, 0x7ff80000, s6 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v4f64: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX11-NEXT: v_cmp_o_f64_e64 s0, v[0:1], v[8:9] -; GFX11-NEXT: v_cmp_gt_f64_e64 s1, v[2:3], v[10:11] -; GFX11-NEXT: v_cmp_gt_f64_e64 s2, v[4:5], v[12:13] -; GFX11-NEXT: v_cmp_gt_f64_e64 s3, v[6:7], v[14:15] -; GFX11-NEXT: v_cmp_o_f64_e64 s4, v[2:3], v[10:11] -; GFX11-NEXT: v_cmp_o_f64_e64 s5, v[4:5], v[12:13] -; GFX11-NEXT: v_cmp_o_f64_e64 s6, v[6:7], v[14:15] -; GFX11-NEXT: v_cndmask_b32_e32 v16, v9, v1, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v18, v11, v3, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v20, v13, v5, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v22, v15, v7, s3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v17, 0x7ff80000, v16, s0 -; GFX11-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v19, 0x7ff80000, v18, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v21, 0x7ff80000, v20, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v18, v10, v2, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v20, v12, v4, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v24, v14, v6, s3 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[4:5], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s3, v[6:7], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v23, 0x7ff80000, v22, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v18, 0, v18, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v20, 0, v20, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v22, 0, v24, s6 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[12:13], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s4, v[14:15], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s5, v[8:9], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s6, v[10:11], 64 -; GFX11-NEXT: v_cmp_eq_f64_e64 s8, 0, v[18:19] -; GFX11-NEXT: v_cmp_eq_f64_e64 s9, 0, v[20:21] -; GFX11-NEXT: v_cmp_eq_f64_e64 s10, 0, v[22:23] -; GFX11-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v16, 0, v16, s0 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_cmp_eq_f64_e64 s7, 0, v[16:17] -; GFX11-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v4, v20, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v22, v6, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v21, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v23, v7, s3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v12, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v8, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v14, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v9, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v13, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v7, v15, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v20, v4, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v22, v6, s10 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v5, v21, v5, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v23, v7, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v18, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v19, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v16, v0, s7 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v10, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v11, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v17, v1, s7 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v18, v2, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v19, v3, s8 +; GFX11-NEXT: v_max_f64 v[16:17], v[0:1], v[8:9] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9] +; GFX11-NEXT: v_max_f64 v[8:9], v[2:3], v[10:11] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[10:11] +; GFX11-NEXT: v_max_f64 v[10:11], v[4:5], v[12:13] +; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[12:13] +; GFX11-NEXT: v_max_f64 v[12:13], v[6:7], v[14:15] +; GFX11-NEXT: v_cmp_u_f64_e64 s2, v[6:7], v[14:15] +; GFX11-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v8, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v9, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v4, v10, 0, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v5, v11, 0x7ff80000, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v6, v12, 0, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v7, v13, 0x7ff80000, s2 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v4f64: @@ -2819,320 +1508,55 @@ define <4 x double> @v_maximum_v4f64__nnan(<4 x double> %src0, <4 x double> %src ; GFX7-LABEL: v_maximum_v4f64__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], 64 -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[8:9], 64 -; GFX7-NEXT: v_cmp_class_f64_e64 s[10:11], v[10:11], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[4:5] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[4:5], v[4:5], v[12:13] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[16:17] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[12:13], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v19, v11, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v18, v10, v2, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[18:19] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v13, v5, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v8, v12, v4, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[4:5], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[8:9] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[10:11] -; GFX7-NEXT: v_cmp_class_f64_e64 s[10:11], v[14:15], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v11, v15, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v10, v14, v6, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v10, v6, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v11, v7, s[12:13] +; GFX7-NEXT: v_max_f64 v[0:1], v[0:1], v[8:9] +; GFX7-NEXT: v_max_f64 v[2:3], v[2:3], v[10:11] +; GFX7-NEXT: v_max_f64 v[4:5], v[4:5], v[12:13] +; GFX7-NEXT: v_max_f64 v[6:7], v[6:7], v[14:15] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v4f64__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], 64 -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[8:9], 64 -; GFX8-NEXT: v_cmp_class_f64_e64 s[10:11], v[10:11], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[4:5] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[4:5], v[4:5], v[12:13] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[16:17] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[12:13], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v19, v11, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v18, v10, v2, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[18:19] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v13, v5, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v8, v12, v4, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[4:5], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[8:9] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[10:11] -; GFX8-NEXT: v_cmp_class_f64_e64 s[10:11], v[14:15], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v11, v15, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v10, v14, v6, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v10, v6, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v11, v7, s[12:13] +; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[8:9] +; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[10:11] +; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[12:13] +; GFX8-NEXT: v_max_f64 v[6:7], v[6:7], v[14:15] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v4f64__nnan: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], 64 -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[8:9], 64 -; GFX9-NEXT: v_cmp_class_f64_e64 s[10:11], v[10:11], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[4:5] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[4:5], v[4:5], v[12:13] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[16:17] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[12:13], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v19, v11, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v18, v10, v2, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[18:19] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v13, v5, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v8, v12, v4, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[4:5], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[8:9] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[10:11] -; GFX9-NEXT: v_cmp_class_f64_e64 s[10:11], v[14:15], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v11, v15, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v10, v14, v6, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v10, v6, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v11, v7, s[12:13] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[8:9] +; GFX9-NEXT: v_max_f64 v[2:3], v[2:3], v[10:11] +; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[12:13] +; GFX9-NEXT: v_max_f64 v[6:7], v[6:7], v[14:15] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v4f64__nnan: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[8:9], 64 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v11, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[10:11], 64 -; GFX940-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[0:1] -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[0:1] -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[12:13] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v13, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v8, v12, v4, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[12:13], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[0:1] -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15] -; GFX940-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v15, v7, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v8, v14, v6, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[14:15], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v6, v8, v6, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[2:3] +; GFX940-NEXT: v_max_f64 v[0:1], v[0:1], v[8:9] +; GFX940-NEXT: v_max_f64 v[2:3], v[2:3], v[10:11] +; GFX940-NEXT: v_max_f64 v[4:5], v[4:5], v[12:13] +; GFX940-NEXT: v_max_f64 v[6:7], v[6:7], v[14:15] ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v4f64__nnan: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[2:3], v[10:11] -; GFX10-NEXT: v_cmp_gt_f64_e64 s5, v[4:5], v[12:13] -; GFX10-NEXT: v_cmp_gt_f64_e64 s6, v[6:7], v[14:15] -; GFX10-NEXT: v_cmp_class_f64_e64 s7, v[10:11], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s8, v[8:9], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[12:13], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s11, v[14:15], 64 -; GFX10-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v19, v11, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v21, v13, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v23, v15, v7, s6 -; GFX10-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v18, v10, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v20, v12, v4, s5 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[4:5], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v22, v14, v6, s6 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[6:7], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[0:1], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[2:3], 64 -; GFX10-NEXT: v_cmp_eq_f64_e64 s9, 0, v[16:17] -; GFX10-NEXT: v_cmp_eq_f64_e64 s12, 0, v[18:19] -; GFX10-NEXT: v_cmp_eq_f64_e64 s13, 0, v[20:21] -; GFX10-NEXT: v_cmp_eq_f64_e64 s14, 0, v[22:23] -; GFX10-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v6, v22, v6, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v16, v0, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v18, v2, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v17, v1, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v19, v3, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v23, v7, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v8, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v10, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v12, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v14, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v9, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v11, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v13, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v15, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v16, v0, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v18, v2, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v17, v1, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v19, v3, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v20, v4, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v21, v5, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v22, v6, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v23, v7, s14 +; GFX10-NEXT: v_max_f64 v[0:1], v[0:1], v[8:9] +; GFX10-NEXT: v_max_f64 v[2:3], v[2:3], v[10:11] +; GFX10-NEXT: v_max_f64 v[4:5], v[4:5], v[12:13] +; GFX10-NEXT: v_max_f64 v[6:7], v[6:7], v[14:15] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v4f64__nnan: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX11-NEXT: v_cmp_gt_f64_e64 s0, v[2:3], v[10:11] -; GFX11-NEXT: v_cmp_gt_f64_e64 s1, v[4:5], v[12:13] -; GFX11-NEXT: v_cmp_gt_f64_e64 s2, v[6:7], v[14:15] -; GFX11-NEXT: v_cmp_class_f64_e64 s3, v[6:7], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s4, v[14:15], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s5, v[8:9], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s6, v[10:11], 64 -; GFX11-NEXT: v_dual_cndmask_b32 v17, v9, v1 :: v_dual_cndmask_b32 v16, v8, v0 -; GFX11-NEXT: v_cndmask_b32_e64 v19, v11, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v21, v13, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v18, v10, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v20, v12, v4, s1 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[4:5], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v23, v15, v7, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v22, v14, v6, s2 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[12:13], 64 -; GFX11-NEXT: v_cmp_eq_f64_e64 s7, 0, v[16:17] -; GFX11-NEXT: v_cmp_eq_f64_e64 s8, 0, v[18:19] -; GFX11-NEXT: v_cmp_eq_f64_e64 s9, 0, v[20:21] -; GFX11-NEXT: v_cmp_eq_f64_e64 s10, 0, v[22:23] -; GFX11-NEXT: v_cndmask_b32_e64 v6, v22, v6, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v23, v7, s3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v14, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v7, v15, s4 -; GFX11-NEXT: v_dual_cndmask_b32 v0, v16, v0 :: v_dual_cndmask_b32 v1, v17, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v18, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v20, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v19, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v21, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v8, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v10, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v12, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v9, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v11, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v13, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v16, v0, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v18, v2, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v20, v4, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v22, v6, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v17, v1, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v19, v3, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v21, v5, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v23, v7, s10 +; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[8:9] +; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[10:11] +; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[12:13] +; GFX11-NEXT: v_max_f64 v[6:7], v[6:7], v[14:15] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v4f64__nnan: @@ -3155,180 +1579,135 @@ define <4 x double> @v_maximum_v4f64__nsz(<4 x double> %src0, <4 x double> %src1 ; GFX7-LABEL: v_maximum_v4f64__nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[6:7], v[4:5], v[12:13] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[10:11], v[6:7], v[14:15] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[4:5], v[12:13] -; GFX7-NEXT: v_cmp_o_f64_e64 s[12:13], v[6:7], v[14:15] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX7-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v0, 0, v16, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[4:5] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v15, v7, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v8, v7, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e32 v9, v10, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v10, v14, v6, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, 0, v9, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v12, v4, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v4, 0, v9, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v6, 0, v10, s[12:13] +; GFX7-NEXT: v_max_f64 v[16:17], v[0:1], v[8:9] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX7-NEXT: v_max_f64 v[8:9], v[2:3], v[10:11] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11] +; GFX7-NEXT: v_max_f64 v[10:11], v[4:5], v[12:13] +; GFX7-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13] +; GFX7-NEXT: v_max_f64 v[12:13], v[6:7], v[14:15] +; GFX7-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15] +; GFX7-NEXT: v_mov_b32_e32 v7, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v17, v7, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v8, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v9, v7, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v4, v10, 0, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v11, v7, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v6, v12, 0, s[8:9] +; GFX7-NEXT: v_cndmask_b32_e64 v7, v13, v7, s[8:9] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v4f64__nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[6:7], v[4:5], v[12:13] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[10:11], v[6:7], v[14:15] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[4:5], v[12:13] -; GFX8-NEXT: v_cmp_o_f64_e64 s[12:13], v[6:7], v[14:15] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX8-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, v16, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[4:5] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v15, v7, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v8, v7, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e32 v9, v10, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v10, v14, v6, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, v9, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v12, v4, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v4, 0, v9, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, v10, s[12:13] +; GFX8-NEXT: v_max_f64 v[16:17], v[0:1], v[8:9] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[10:11] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11] +; GFX8-NEXT: v_max_f64 v[10:11], v[4:5], v[12:13] +; GFX8-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13] +; GFX8-NEXT: v_max_f64 v[12:13], v[6:7], v[14:15] +; GFX8-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15] +; GFX8-NEXT: v_mov_b32_e32 v7, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v7, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v8, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v9, v7, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v4, v10, 0, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v11, v7, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v6, v12, 0, s[8:9] +; GFX8-NEXT: v_cndmask_b32_e64 v7, v13, v7, s[8:9] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v4f64__nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[6:7], v[4:5], v[12:13] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[10:11], v[6:7], v[14:15] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[4:5], v[12:13] -; GFX9-NEXT: v_cmp_o_f64_e64 s[12:13], v[6:7], v[14:15] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX9-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, v16, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[4:5] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v15, v7, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v8, v7, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v10, v14, v6, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, v9, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v12, v4, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, v9, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, v10, s[12:13] +; GFX9-NEXT: v_max_f64 v[16:17], v[0:1], v[8:9] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX9-NEXT: v_max_f64 v[8:9], v[2:3], v[10:11] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11] +; GFX9-NEXT: v_max_f64 v[10:11], v[4:5], v[12:13] +; GFX9-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13] +; GFX9-NEXT: v_max_f64 v[12:13], v[6:7], v[14:15] +; GFX9-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15] +; GFX9-NEXT: v_mov_b32_e32 v7, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v7, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v8, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v9, v7, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v4, v10, 0, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v11, v7, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v6, v12, 0, s[8:9] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v13, v7, s[8:9] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v4f64__nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[8:9] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX940-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v0, 0, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v10, v2, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[12:13] -; GFX940-NEXT: v_cndmask_b32_e64 v2, 0, v9, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v12, v4, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[4:5], v[12:13] -; GFX940-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15] -; GFX940-NEXT: v_cndmask_b32_e64 v4, 0, v9, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v14, v6, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[6:7], v[14:15] -; GFX940-NEXT: v_cndmask_b32_e32 v7, v15, v7, vcc +; GFX940-NEXT: v_max_f64 v[16:17], v[0:1], v[8:9] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX940-NEXT: v_max_f64 v[8:9], v[2:3], v[10:11] ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v6, 0, v9, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v8, v7, s[0:1] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX940-NEXT: v_mov_b32_e32 v16, 0x7ff80000 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v17, v16, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[10:11] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v2, v8, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v16, vcc +; GFX940-NEXT: v_max_f64 v[8:9], v[4:5], v[12:13] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[4:5], v[12:13] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v4, v8, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v16, vcc +; GFX940-NEXT: v_max_f64 v[8:9], v[6:7], v[14:15] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[6:7], v[14:15] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v6, v8, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v7, v9, v16, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v4f64__nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[2:3], v[10:11] -; GFX10-NEXT: v_cmp_gt_f64_e64 s5, v[4:5], v[12:13] -; GFX10-NEXT: v_cmp_gt_f64_e64 s7, v[6:7], v[14:15] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[0:1], v[8:9] -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[2:3], v[10:11] -; GFX10-NEXT: v_cmp_o_f64_e64 s9, v[4:5], v[12:13] -; GFX10-NEXT: v_cmp_o_f64_e64 s10, v[6:7], v[14:15] -; GFX10-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v8, v10, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v10, v12, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v12, v14, v6, s7 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v13, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v15, v7, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, v16, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v1, 0x7ff80000, v1, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, v8, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v3, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, v10, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v5, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, v12, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0x7ff80000, v7, s10 +; GFX10-NEXT: v_max_f64 v[16:17], v[0:1], v[8:9] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9] +; GFX10-NEXT: v_max_f64 v[8:9], v[2:3], v[10:11] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[10:11] +; GFX10-NEXT: v_max_f64 v[10:11], v[4:5], v[12:13] +; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[4:5], v[12:13] +; GFX10-NEXT: v_max_f64 v[12:13], v[6:7], v[14:15] +; GFX10-NEXT: v_cmp_u_f64_e64 s6, v[6:7], v[14:15] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v8, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v9, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v4, v10, 0, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v11, 0x7ff80000, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v6, v12, 0, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v7, v13, 0x7ff80000, s6 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v4f64__nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX11-NEXT: v_cmp_gt_f64_e64 s1, v[2:3], v[10:11] -; GFX11-NEXT: v_cmp_gt_f64_e64 s2, v[4:5], v[12:13] -; GFX11-NEXT: v_cmp_gt_f64_e64 s3, v[6:7], v[14:15] -; GFX11-NEXT: v_cmp_o_f64_e64 s0, v[0:1], v[8:9] -; GFX11-NEXT: v_cmp_o_f64_e64 s4, v[2:3], v[10:11] -; GFX11-NEXT: v_cmp_o_f64_e64 s5, v[4:5], v[12:13] -; GFX11-NEXT: v_cmp_o_f64_e64 s6, v[6:7], v[14:15] -; GFX11-NEXT: v_dual_cndmask_b32 v16, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v8, v10, v2, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v12, v4, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v12, v14, v6, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, v3, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v13, v5, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v15, v7, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, v16, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, v8, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v4, 0, v10, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v6, 0, v12, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v1, 0x7ff80000, v1, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v3, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v5, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v7, 0x7ff80000, v7, s6 +; GFX11-NEXT: v_max_f64 v[16:17], v[0:1], v[8:9] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9] +; GFX11-NEXT: v_max_f64 v[8:9], v[2:3], v[10:11] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[10:11] +; GFX11-NEXT: v_max_f64 v[10:11], v[4:5], v[12:13] +; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[12:13] +; GFX11-NEXT: v_max_f64 v[12:13], v[6:7], v[14:15] +; GFX11-NEXT: v_cmp_u_f64_e64 s2, v[6:7], v[14:15] +; GFX11-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v8, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v9, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v4, v10, 0, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v5, v11, 0x7ff80000, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v6, v12, 0, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v7, v13, 0x7ff80000, s2 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v4f64__nsz: @@ -3351,108 +1730,55 @@ define <4 x double> @v_maximum_v4f64__nnan_nsz(<4 x double> %src0, <4 x double> ; GFX7-LABEL: v_maximum_v4f64__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[4:5], v[2:3], v[10:11] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[6:7], v[4:5], v[12:13] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[8:9], v[6:7], v[14:15] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v14, v6, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v15, v7, s[8:9] +; GFX7-NEXT: v_max_f64 v[0:1], v[0:1], v[8:9] +; GFX7-NEXT: v_max_f64 v[2:3], v[2:3], v[10:11] +; GFX7-NEXT: v_max_f64 v[4:5], v[4:5], v[12:13] +; GFX7-NEXT: v_max_f64 v[6:7], v[6:7], v[14:15] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v4f64__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[4:5], v[2:3], v[10:11] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[6:7], v[4:5], v[12:13] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[8:9], v[6:7], v[14:15] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v14, v6, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v15, v7, s[8:9] +; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[8:9] +; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[10:11] +; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[12:13] +; GFX8-NEXT: v_max_f64 v[6:7], v[6:7], v[14:15] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v4f64__nnan_nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[4:5], v[2:3], v[10:11] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[6:7], v[4:5], v[12:13] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[8:9], v[6:7], v[14:15] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v14, v6, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v15, v7, s[8:9] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[8:9] +; GFX9-NEXT: v_max_f64 v[2:3], v[2:3], v[10:11] +; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[12:13] +; GFX9-NEXT: v_max_f64 v[6:7], v[6:7], v[14:15] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v4f64__nnan_nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[8:9] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[10:11] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[12:13] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[14:15] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v14, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v7, v15, v7, vcc +; GFX940-NEXT: v_max_f64 v[0:1], v[0:1], v[8:9] +; GFX940-NEXT: v_max_f64 v[2:3], v[2:3], v[10:11] +; GFX940-NEXT: v_max_f64 v[4:5], v[4:5], v[12:13] +; GFX940-NEXT: v_max_f64 v[6:7], v[6:7], v[14:15] ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v4f64__nnan_nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[2:3], v[10:11] -; GFX10-NEXT: v_cmp_gt_f64_e64 s5, v[4:5], v[12:13] -; GFX10-NEXT: v_cmp_gt_f64_e64 s6, v[6:7], v[14:15] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v12, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v14, v6, s6 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v13, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v15, v7, s6 +; GFX10-NEXT: v_max_f64 v[0:1], v[0:1], v[8:9] +; GFX10-NEXT: v_max_f64 v[2:3], v[2:3], v[10:11] +; GFX10-NEXT: v_max_f64 v[4:5], v[4:5], v[12:13] +; GFX10-NEXT: v_max_f64 v[6:7], v[6:7], v[14:15] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v4f64__nnan_nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX11-NEXT: v_cmp_gt_f64_e64 s0, v[2:3], v[10:11] -; GFX11-NEXT: v_cmp_gt_f64_e64 s1, v[4:5], v[12:13] -; GFX11-NEXT: v_cmp_gt_f64_e64 s2, v[6:7], v[14:15] -; GFX11-NEXT: v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v12, v4, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v6, v14, v6, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v13, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v15, v7, s2 +; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[8:9] +; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[10:11] +; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[12:13] +; GFX11-NEXT: v_max_f64 v[6:7], v[6:7], v[14:15] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v4f64__nnan_nsz: @@ -3475,782 +1801,244 @@ define <8 x double> @v_maximum_v8f64(<8 x double> %src0, <8 x double> %src1) { ; GFX7-LABEL: v_maximum_v8f64: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[16:17] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[16:17] -; GFX7-NEXT: v_mov_b32_e32 v32, 0x7ff80000 -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[4:5], v[20:21] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[8:9], v[24:25] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[10:11], v[12:13], v[28:29] -; GFX7-NEXT: v_cmp_o_f64_e64 s[12:13], v[12:13], v[28:29] -; GFX7-NEXT: v_cndmask_b32_e32 v31, v17, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v34, v32, v31, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v31, v16, v0, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v33, 0, v31, s[4:5] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[18:19] ; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v34, v1, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[33:34] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v34, v1, vcc -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[18:19] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[4:5] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[4:5], v[4:5], v[20:21] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[18:19], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[16:17] -; GFX7-NEXT: v_cndmask_b32_e64 v18, v21, v5, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v16, v20, v4, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[20:21], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v17, v32, v18, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[6:7] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[6:7], v[22:23] -; GFX7-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v5, v17, v5, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v20, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v21, s[4:5] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[16:17] -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[22:23] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v16, v4, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[4:5] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[4:5], v[8:9], v[24:25] -; GFX7-NEXT: v_cndmask_b32_e32 v18, v23, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v18, 0, v16, s[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[22:23], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v16, v25, v9, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v16, v24, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v18, v6, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[18:19] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[8:9], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, v22, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v23, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[8:9] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[24:25], 64 -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[10:11], v[26:27] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v18, v6, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[10:11], v[26:27] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[4:5] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[16:17] -; GFX7-NEXT: v_cndmask_b32_e64 v8, v8, v24, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v9, v25, s[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[26:27], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v18, v27, v11, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v18, 0, v16, s[8:9] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[18:19] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[4:5] +; GFX7-NEXT: v_max_f64 v[32:33], v[2:3], v[18:19] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[18:19] +; GFX7-NEXT: v_max_f64 v[18:19], v[4:5], v[20:21] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[4:5], v[20:21] +; GFX7-NEXT: v_max_f64 v[2:3], v[0:1], v[16:17] +; GFX7-NEXT: v_cmp_u_f64_e64 s[8:9], v[0:1], v[16:17] +; GFX7-NEXT: v_mov_b32_e32 v34, 0x7ff80000 +; GFX7-NEXT: v_max_f64 v[20:21], v[6:7], v[22:23] +; GFX7-NEXT: v_cmp_u_f64_e64 s[6:7], v[6:7], v[22:23] +; GFX7-NEXT: v_max_f64 v[16:17], v[8:9], v[24:25] +; GFX7-NEXT: v_cmp_u_f64_e64 s[10:11], v[8:9], v[24:25] +; GFX7-NEXT: v_max_f64 v[22:23], v[10:11], v[26:27] +; GFX7-NEXT: v_cmp_u_f64_e64 s[12:13], v[10:11], v[26:27] +; GFX7-NEXT: v_max_f64 v[24:25], v[12:13], v[28:29] +; GFX7-NEXT: v_cmp_u_f64_e64 s[14:15], v[12:13], v[28:29] +; GFX7-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[8:9] +; GFX7-NEXT: v_cndmask_b32_e64 v1, v3, v34, s[8:9] +; GFX7-NEXT: v_cndmask_b32_e64 v2, v32, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v3, v33, v34, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v4, v18, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v19, v34, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v6, v20, 0, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v7, v21, v34, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v8, v16, 0, s[10:11] +; GFX7-NEXT: v_cndmask_b32_e64 v9, v17, v34, s[10:11] +; GFX7-NEXT: v_cndmask_b32_e64 v10, v22, 0, s[12:13] +; GFX7-NEXT: v_cndmask_b32_e64 v11, v23, v34, s[12:13] +; GFX7-NEXT: v_cndmask_b32_e64 v12, v24, 0, s[14:15] +; GFX7-NEXT: v_cndmask_b32_e64 v13, v25, v34, s[14:15] ; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[14:15], v[30:31] -; GFX7-NEXT: v_cndmask_b32_e64 v16, v29, v13, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v16, v28, v12, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v10, v18, v10, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v11, v19, v11, vcc -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[14:15], v[30:31] -; GFX7-NEXT: v_cndmask_b32_e64 v10, v10, v26, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v27, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v10, v18, v10, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v11, v19, v11, s[8:9] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[12:13], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[12:13] -; GFX7-NEXT: v_cmp_class_f64_e64 s[8:9], v[28:29], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v18, v31, v15, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v18, v30, v14, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v18, 0, v18, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[30:31], 64 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[16:17] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[18:19] -; GFX7-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v12, v12, v28, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v14, v18, v14, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v15, v19, v15, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v14, v14, v30, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v13, v13, v29, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v15, v15, v31, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v14, v18, v14, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v15, v19, v15, s[12:13] +; GFX7-NEXT: v_max_f64 v[18:19], v[14:15], v[30:31] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[14:15], v[30:31] +; GFX7-NEXT: v_cndmask_b32_e64 v14, v18, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v15, v19, v34, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v8f64: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[16:17] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[16:17] -; GFX8-NEXT: v_mov_b32_e32 v32, 0x7ff80000 -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[4:5], v[20:21] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[8:9], v[24:25] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[10:11], v[12:13], v[28:29] -; GFX8-NEXT: v_cmp_o_f64_e64 s[12:13], v[12:13], v[28:29] -; GFX8-NEXT: v_cndmask_b32_e32 v31, v17, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v34, v32, v31, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v31, v16, v0, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v33, 0, v31, s[4:5] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[18:19] ; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v34, v1, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[33:34] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v34, v1, vcc -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[18:19] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[4:5] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[4:5], v[4:5], v[20:21] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[18:19], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[16:17] -; GFX8-NEXT: v_cndmask_b32_e64 v18, v21, v5, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v16, v20, v4, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[20:21], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v17, v32, v18, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[6:7] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[6:7], v[22:23] -; GFX8-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v5, v17, v5, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v20, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v21, s[4:5] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[16:17] -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[22:23] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v16, v4, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[4:5] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[4:5], v[8:9], v[24:25] -; GFX8-NEXT: v_cndmask_b32_e32 v18, v23, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v18, 0, v16, s[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[22:23], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v16, v25, v9, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v16, v24, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v18, v6, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[18:19] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[8:9], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v22, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v23, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[8:9] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[24:25], 64 -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[10:11], v[26:27] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v18, v6, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[10:11], v[26:27] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[4:5] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[16:17] -; GFX8-NEXT: v_cndmask_b32_e64 v8, v8, v24, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v9, v25, s[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[26:27], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v18, v27, v11, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v18, 0, v16, s[8:9] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[18:19] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[4:5] +; GFX8-NEXT: v_max_f64 v[32:33], v[2:3], v[18:19] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[18:19] +; GFX8-NEXT: v_max_f64 v[18:19], v[4:5], v[20:21] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[4:5], v[20:21] +; GFX8-NEXT: v_max_f64 v[2:3], v[0:1], v[16:17] +; GFX8-NEXT: v_cmp_u_f64_e64 s[8:9], v[0:1], v[16:17] +; GFX8-NEXT: v_mov_b32_e32 v34, 0x7ff80000 +; GFX8-NEXT: v_max_f64 v[20:21], v[6:7], v[22:23] +; GFX8-NEXT: v_cmp_u_f64_e64 s[6:7], v[6:7], v[22:23] +; GFX8-NEXT: v_max_f64 v[16:17], v[8:9], v[24:25] +; GFX8-NEXT: v_cmp_u_f64_e64 s[10:11], v[8:9], v[24:25] +; GFX8-NEXT: v_max_f64 v[22:23], v[10:11], v[26:27] +; GFX8-NEXT: v_cmp_u_f64_e64 s[12:13], v[10:11], v[26:27] +; GFX8-NEXT: v_max_f64 v[24:25], v[12:13], v[28:29] +; GFX8-NEXT: v_cmp_u_f64_e64 s[14:15], v[12:13], v[28:29] +; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[8:9] +; GFX8-NEXT: v_cndmask_b32_e64 v1, v3, v34, s[8:9] +; GFX8-NEXT: v_cndmask_b32_e64 v2, v32, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v3, v33, v34, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v4, v18, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v19, v34, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v6, v20, 0, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v7, v21, v34, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v8, v16, 0, s[10:11] +; GFX8-NEXT: v_cndmask_b32_e64 v9, v17, v34, s[10:11] +; GFX8-NEXT: v_cndmask_b32_e64 v10, v22, 0, s[12:13] +; GFX8-NEXT: v_cndmask_b32_e64 v11, v23, v34, s[12:13] +; GFX8-NEXT: v_cndmask_b32_e64 v12, v24, 0, s[14:15] +; GFX8-NEXT: v_cndmask_b32_e64 v13, v25, v34, s[14:15] ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[14:15], v[30:31] -; GFX8-NEXT: v_cndmask_b32_e64 v16, v29, v13, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v16, v28, v12, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v10, v18, v10, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v11, v19, v11, vcc -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[14:15], v[30:31] -; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, v26, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v27, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v10, v18, v10, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v11, v19, v11, s[8:9] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[12:13], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[12:13] -; GFX8-NEXT: v_cmp_class_f64_e64 s[8:9], v[28:29], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v18, v31, v15, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v18, v30, v14, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v18, 0, v18, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[30:31], 64 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[16:17] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[18:19] -; GFX8-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v12, v12, v28, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v14, v18, v14, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v15, v19, v15, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v14, v14, v30, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v13, v13, v29, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v15, v15, v31, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v14, v18, v14, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v15, v19, v15, s[12:13] +; GFX8-NEXT: v_max_f64 v[18:19], v[14:15], v[30:31] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[14:15], v[30:31] +; GFX8-NEXT: v_cndmask_b32_e64 v14, v18, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v15, v19, v34, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v8f64: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[16:17] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[16:17] -; GFX9-NEXT: v_mov_b32_e32 v32, 0x7ff80000 -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[4:5], v[20:21] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[8:9], v[24:25] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[10:11], v[12:13], v[28:29] -; GFX9-NEXT: v_cmp_o_f64_e64 s[12:13], v[12:13], v[28:29] -; GFX9-NEXT: v_cndmask_b32_e32 v31, v17, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v34, v32, v31, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v31, v16, v0, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v33, 0, v31, s[4:5] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[18:19] ; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v34, v1, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[33:34] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v34, v1, vcc -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[18:19] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[4:5] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[4:5], v[4:5], v[20:21] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[18:19], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[16:17] -; GFX9-NEXT: v_cndmask_b32_e64 v18, v21, v5, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v16, v20, v4, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[20:21], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v17, v32, v18, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[6:7] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[6:7], v[22:23] -; GFX9-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v5, v17, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v20, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v21, s[4:5] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[16:17] -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[22:23] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v16, v4, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[4:5] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[4:5], v[8:9], v[24:25] -; GFX9-NEXT: v_cndmask_b32_e32 v18, v23, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v18, 0, v16, s[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[22:23], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v16, v25, v9, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v16, v24, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v18, v6, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[18:19] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[8:9], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v22, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v23, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[8:9] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[24:25], 64 -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[10:11], v[26:27] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v18, v6, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[10:11], v[26:27] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[4:5] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[16:17] -; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v24, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v25, s[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[26:27], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v18, v27, v11, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v18, 0, v16, s[8:9] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[18:19] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[4:5] +; GFX9-NEXT: v_max_f64 v[32:33], v[2:3], v[18:19] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[18:19] +; GFX9-NEXT: v_max_f64 v[18:19], v[4:5], v[20:21] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[4:5], v[20:21] +; GFX9-NEXT: v_max_f64 v[2:3], v[0:1], v[16:17] +; GFX9-NEXT: v_cmp_u_f64_e64 s[8:9], v[0:1], v[16:17] +; GFX9-NEXT: v_mov_b32_e32 v34, 0x7ff80000 +; GFX9-NEXT: v_max_f64 v[20:21], v[6:7], v[22:23] +; GFX9-NEXT: v_cmp_u_f64_e64 s[6:7], v[6:7], v[22:23] +; GFX9-NEXT: v_max_f64 v[16:17], v[8:9], v[24:25] +; GFX9-NEXT: v_cmp_u_f64_e64 s[10:11], v[8:9], v[24:25] +; GFX9-NEXT: v_max_f64 v[22:23], v[10:11], v[26:27] +; GFX9-NEXT: v_cmp_u_f64_e64 s[12:13], v[10:11], v[26:27] +; GFX9-NEXT: v_max_f64 v[24:25], v[12:13], v[28:29] +; GFX9-NEXT: v_cmp_u_f64_e64 s[14:15], v[12:13], v[28:29] +; GFX9-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[8:9] +; GFX9-NEXT: v_cndmask_b32_e64 v1, v3, v34, s[8:9] +; GFX9-NEXT: v_cndmask_b32_e64 v2, v32, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v3, v33, v34, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v4, v18, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v19, v34, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v6, v20, 0, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v21, v34, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v8, v16, 0, s[10:11] +; GFX9-NEXT: v_cndmask_b32_e64 v9, v17, v34, s[10:11] +; GFX9-NEXT: v_cndmask_b32_e64 v10, v22, 0, s[12:13] +; GFX9-NEXT: v_cndmask_b32_e64 v11, v23, v34, s[12:13] +; GFX9-NEXT: v_cndmask_b32_e64 v12, v24, 0, s[14:15] +; GFX9-NEXT: v_cndmask_b32_e64 v13, v25, v34, s[14:15] ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[14:15], v[30:31] -; GFX9-NEXT: v_cndmask_b32_e64 v16, v29, v13, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v16, v28, v12, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v10, v18, v10, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v11, v19, v11, vcc -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[14:15], v[30:31] -; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v26, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v27, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v10, v18, v10, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v11, v19, v11, s[8:9] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[12:13], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[12:13] -; GFX9-NEXT: v_cmp_class_f64_e64 s[8:9], v[28:29], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v18, v31, v15, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v18, v30, v14, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v18, 0, v18, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[30:31], 64 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[16:17] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[18:19] -; GFX9-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, v28, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v14, v18, v14, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v15, v19, v15, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v14, v14, v30, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v13, v13, v29, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v15, v15, v31, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v14, v18, v14, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v15, v19, v15, s[12:13] +; GFX9-NEXT: v_max_f64 v[18:19], v[14:15], v[30:31] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[14:15], v[30:31] +; GFX9-NEXT: v_cndmask_b32_e64 v14, v18, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v15, v19, v34, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v8f64: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX940-NEXT: scratch_load_dword v31, off, s32 -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[16:17] -; GFX940-NEXT: v_mov_b32_e32 v32, 0x7ff80000 -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v33, v17, v1, vcc -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v35, v32, v33, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v33, v16, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v34, 0, v33, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[16:17], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[34:35] -; GFX940-NEXT: v_cndmask_b32_e32 v0, v34, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v35, v1, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[18:19] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v17, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[18:19] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v34, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v35, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[18:19], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v18, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[20:21] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v16, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v19, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v21, v5, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[4:5], v[20:21] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v17, v3, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v20, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[20:21], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, v4, v20, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v5, v17, v5, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[22:23] -; GFX940-NEXT: v_cndmask_b32_e64 v4, v16, v4, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v5, v21, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v23, v7, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[6:7], v[22:23] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[22:23], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v6, v6, v22, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v7, v17, v7, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[8:9], v[24:25] -; GFX940-NEXT: v_cndmask_b32_e64 v6, v16, v6, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v7, v23, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v25, v9, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[8:9], v[24:25] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v17, v7, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v24, v8, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[8:9], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[24:25], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v8, v8, v24, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v17, v9, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[10:11], v[26:27] -; GFX940-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v9, v9, v25, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v27, v11, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[10:11], v[26:27] -; GFX940-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[26:27], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v10, v10, v26, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v11, v17, v11, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[12:13], v[28:29] -; GFX940-NEXT: v_cndmask_b32_e64 v10, v16, v10, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v11, v11, v27, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v29, v13, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[12:13], v[28:29] -; GFX940-NEXT: v_cndmask_b32_e64 v11, v17, v11, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v28, v12, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[12:13], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[28:29], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v12, v12, v28, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v13, v17, v13, vcc +; GFX940-NEXT: v_mov_b32_e32 v54, 0x7ff80000 +; GFX940-NEXT: v_max_f64 v[32:33], v[0:1], v[16:17] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[16:17] +; GFX940-NEXT: v_max_f64 v[34:35], v[2:3], v[18:19] +; GFX940-NEXT: v_max_f64 v[36:37], v[4:5], v[20:21] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v32, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v1, v33, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[18:19] +; GFX940-NEXT: v_max_f64 v[38:39], v[6:7], v[22:23] +; GFX940-NEXT: v_max_f64 v[48:49], v[8:9], v[24:25] +; GFX940-NEXT: v_cndmask_b32_e64 v2, v34, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v35, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[4:5], v[20:21] +; GFX940-NEXT: v_max_f64 v[50:51], v[10:11], v[26:27] +; GFX940-NEXT: v_max_f64 v[52:53], v[12:13], v[28:29] +; GFX940-NEXT: v_cndmask_b32_e64 v4, v36, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v5, v37, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[6:7], v[22:23] ; GFX940-NEXT: s_waitcnt vmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[14:15], v[30:31] -; GFX940-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v13, v13, v29, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v31, v15, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[14:15], v[30:31] -; GFX940-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v30, v14, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[30:31], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v15, v17, v15, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v14, v14, v30, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v15, v15, v31, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v14, v16, v14, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v15, v17, v15, s[2:3] +; GFX940-NEXT: v_max_f64 v[16:17], v[14:15], v[30:31] +; GFX940-NEXT: v_cndmask_b32_e64 v6, v38, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v7, v39, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[8:9], v[24:25] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v8, v48, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v9, v49, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[10:11], v[26:27] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v10, v50, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v11, v51, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[12:13], v[28:29] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v12, v52, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v13, v53, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[14:15], v[30:31] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v14, v16, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v15, v17, v54, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v8f64: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[16:17] -; GFX10-NEXT: v_cmp_o_f64_e64 s4, v[0:1], v[16:17] -; GFX10-NEXT: v_cmp_gt_f64_e64 s9, v[6:7], v[22:23] -; GFX10-NEXT: v_cmp_gt_f64_e64 s10, v[8:9], v[24:25] -; GFX10-NEXT: v_cmp_gt_f64_e64 s11, v[10:11], v[26:27] -; GFX10-NEXT: v_cmp_gt_f64_e64 s12, v[12:13], v[28:29] -; GFX10-NEXT: v_cmp_o_f64_e64 s13, v[6:7], v[22:23] -; GFX10-NEXT: v_cmp_o_f64_e64 s14, v[8:9], v[24:25] -; GFX10-NEXT: v_cmp_o_f64_e64 s15, v[10:11], v[26:27] -; GFX10-NEXT: v_cmp_o_f64_e64 s16, v[12:13], v[28:29] -; GFX10-NEXT: v_cmp_gt_f64_e64 s5, v[2:3], v[18:19] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[2:3], v[18:19] -; GFX10-NEXT: v_cmp_gt_f64_e64 s7, v[4:5], v[20:21] -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[4:5], v[20:21] -; GFX10-NEXT: v_cmp_class_f64_e64 s17, v[26:27], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s18, v[28:29], 64 -; GFX10-NEXT: v_cndmask_b32_e32 v32, v17, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v38, v23, v7, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v48, v25, v9, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v50, v27, v11, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v33, 0x7ff80000, v32, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v32, v16, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v52, v29, v13, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v39, 0x7ff80000, v38, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v49, 0x7ff80000, v48, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v32, 0, v32, s4 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[2:3], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v51, 0x7ff80000, v50, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v53, 0x7ff80000, v52, s16 -; GFX10-NEXT: v_cndmask_b32_e64 v38, v22, v6, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v48, v24, v8, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v50, v26, v10, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v52, v28, v12, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 s11, v[16:17], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s12, v[18:19], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v34, v19, v3, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v38, 0, v38, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v36, v21, v5, s7 -; GFX10-NEXT: v_cmp_class_f64_e64 s9, v[12:13], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v48, 0, v48, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v35, 0x7ff80000, v34, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v34, v18, v2, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v37, 0x7ff80000, v36, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v36, v20, v4, s7 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[4:5], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v34, 0, v34, s6 -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[6:7], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v36, 0, v36, s8 -; GFX10-NEXT: v_cmp_class_f64_e64 s7, v[8:9], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s8, v[10:11], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v34, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v50, 0, v50, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v52, 0, v52, s16 -; GFX10-NEXT: v_cmp_class_f64_e64 s14, v[20:21], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v16, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v18, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 s15, v[22:23], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s16, v[24:25], 64 -; GFX10-NEXT: v_cmp_eq_f64_e64 s19, 0, v[32:33] -; GFX10-NEXT: v_cmp_eq_f64_e64 s20, 0, v[34:35] -; GFX10-NEXT: v_cmp_eq_f64_e64 s21, 0, v[36:37] -; GFX10-NEXT: v_cmp_eq_f64_e64 s22, 0, v[48:49] -; GFX10-NEXT: v_cmp_eq_f64_e64 s23, 0, v[50:51] -; GFX10-NEXT: v_cmp_eq_f64_e64 s24, 0, v[52:53] -; GFX10-NEXT: v_cndmask_b32_e32 v1, v33, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v12, v52, v12, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v36, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v35, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v38, v6, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v37, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v8, v48, v8, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v10, v50, v10, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v39, v7, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v9, v49, v9, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v51, v11, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v53, v13, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v20, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v26, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v22, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, v24, s16 -; GFX10-NEXT: v_cndmask_b32_e64 v12, v12, v28, s18 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v17, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v19, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v21, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v23, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v25, s16 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v11, v27, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v13, v29, s18 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v32, v0, s19 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v34, v2, s20 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v36, v4, s21 -; GFX10-NEXT: v_cndmask_b32_e64 v8, v48, v8, s22 -; GFX10-NEXT: v_cndmask_b32_e64 v10, v50, v10, s23 -; GFX10-NEXT: v_cndmask_b32_e64 v12, v52, v12, s24 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v33, v1, s19 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v35, v3, s20 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v37, v5, s21 -; GFX10-NEXT: v_cndmask_b32_e64 v9, v49, v9, s22 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v51, v11, s23 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v53, v13, s24 +; GFX10-NEXT: v_max_f64 v[32:33], v[0:1], v[16:17] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[16:17] +; GFX10-NEXT: v_max_f64 v[16:17], v[2:3], v[18:19] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[18:19] +; GFX10-NEXT: v_max_f64 v[18:19], v[4:5], v[20:21] +; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[4:5], v[20:21] +; GFX10-NEXT: v_max_f64 v[20:21], v[6:7], v[22:23] +; GFX10-NEXT: v_cmp_u_f64_e64 s6, v[6:7], v[22:23] +; GFX10-NEXT: v_max_f64 v[22:23], v[8:9], v[24:25] +; GFX10-NEXT: v_cmp_u_f64_e64 s7, v[8:9], v[24:25] +; GFX10-NEXT: v_max_f64 v[24:25], v[10:11], v[26:27] +; GFX10-NEXT: v_cmp_u_f64_e64 s8, v[10:11], v[26:27] +; GFX10-NEXT: v_max_f64 v[26:27], v[12:13], v[28:29] +; GFX10-NEXT: v_cmp_u_f64_e64 s9, v[12:13], v[28:29] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v32, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v33, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v16, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v17, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v4, v18, 0, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v19, 0x7ff80000, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v6, v20, 0, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v7, v21, 0x7ff80000, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v8, v22, 0, s7 +; GFX10-NEXT: v_cndmask_b32_e64 v9, v23, 0x7ff80000, s7 +; GFX10-NEXT: v_cndmask_b32_e64 v10, v24, 0, s8 +; GFX10-NEXT: v_cndmask_b32_e64 v11, v25, 0x7ff80000, s8 +; GFX10-NEXT: v_cndmask_b32_e64 v12, v26, 0, s9 +; GFX10-NEXT: v_cndmask_b32_e64 v13, v27, 0x7ff80000, s9 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_cmp_gt_f64_e64 s10, v[14:15], v[30:31] -; GFX10-NEXT: v_cmp_o_f64_e64 s13, v[14:15], v[30:31] -; GFX10-NEXT: v_cmp_class_f64_e64 s25, v[30:31], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v16, v31, v15, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v18, v30, v14, s10 -; GFX10-NEXT: v_cmp_eq_f64_e64 s10, 0, v[38:39] -; GFX10-NEXT: v_cndmask_b32_e64 v55, 0x7ff80000, v16, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v54, 0, v18, s13 -; GFX10-NEXT: v_cmp_class_f64_e64 s13, v[14:15], 64 -; GFX10-NEXT: v_cmp_eq_f64_e32 vcc_lo, 0, v[54:55] -; GFX10-NEXT: v_cndmask_b32_e64 v6, v38, v6, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v39, v7, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v14, v54, v14, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v15, v55, v15, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v14, v14, v30, s25 -; GFX10-NEXT: v_cndmask_b32_e64 v15, v15, v31, s25 -; GFX10-NEXT: v_cndmask_b32_e32 v14, v54, v14, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v15, v55, v15, vcc_lo +; GFX10-NEXT: v_max_f64 v[28:29], v[14:15], v[30:31] +; GFX10-NEXT: v_cmp_u_f64_e64 s10, v[14:15], v[30:31] +; GFX10-NEXT: v_cndmask_b32_e64 v14, v28, 0, s10 +; GFX10-NEXT: v_cndmask_b32_e64 v15, v29, 0x7ff80000, s10 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v8f64: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-NEXT: v_cmp_gt_f64_e64 s4, v[6:7], v[22:23] -; GFX11-NEXT: v_cmp_o_f64_e64 s9, v[6:7], v[22:23] -; GFX11-NEXT: v_cmp_gt_f64_e64 s1, v[2:3], v[18:19] -; GFX11-NEXT: v_cmp_gt_f64_e64 s6, v[10:11], v[26:27] -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[16:17] -; GFX11-NEXT: v_cmp_o_f64_e64 s2, v[2:3], v[18:19] -; GFX11-NEXT: v_cmp_o_f64_e64 s11, v[10:11], v[26:27] -; GFX11-NEXT: v_cmp_o_f64_e64 s0, v[0:1], v[16:17] -; GFX11-NEXT: v_cmp_gt_f64_e64 s3, v[4:5], v[20:21] -; GFX11-NEXT: v_cmp_gt_f64_e64 s5, v[8:9], v[24:25] -; GFX11-NEXT: v_cmp_gt_f64_e64 s7, v[12:13], v[28:29] -; GFX11-NEXT: v_cmp_o_f64_e64 s8, v[4:5], v[20:21] -; GFX11-NEXT: v_cmp_o_f64_e64 s10, v[8:9], v[24:25] -; GFX11-NEXT: v_cmp_o_f64_e64 s12, v[12:13], v[28:29] -; GFX11-NEXT: v_cmp_class_f64_e64 s13, v[18:19], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s15, v[20:21], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v38, v23, v7, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v34, v19, v3, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v50, v27, v11, s6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v39, 0x7ff80000, v38, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v38, v22, v6, s4 -; GFX11-NEXT: v_cmp_class_f64_e64 s4, v[6:7], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v35, 0x7ff80000, v34, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v51, 0x7ff80000, v50, s11 -; GFX11-NEXT: v_cndmask_b32_e64 v34, v18, v2, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v50, v26, v10, s6 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[0:1], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v36, v21, v5, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v48, v25, v9, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v52, v29, v13, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v50, 0, v50, s11 -; GFX11-NEXT: v_cmp_class_f64_e64 s11, v[16:17], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v37, 0x7ff80000, v36, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v49, 0x7ff80000, v48, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v53, 0x7ff80000, v52, s12 -; GFX11-NEXT: v_cndmask_b32_e64 v36, v20, v4, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v48, v24, v8, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v52, v28, v12, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v34, 0, v34, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v38, 0, v38, s9 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[2:3], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s3, v[4:5], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s5, v[8:9], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s7, v[10:11], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s9, v[12:13], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v36, 0, v36, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v48, 0, v48, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v52, 0, v52, s12 -; GFX11-NEXT: v_cmp_class_f64_e64 s6, v[24:25], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s8, v[26:27], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s10, v[28:29], 64 -; GFX11-NEXT: v_cmp_eq_f64_e64 s14, 0, v[34:35] -; GFX11-NEXT: v_cmp_eq_f64_e64 s16, 0, v[36:37] -; GFX11-NEXT: v_cmp_eq_f64_e64 s17, 0, v[38:39] -; GFX11-NEXT: v_cmp_eq_f64_e64 s18, 0, v[48:49] -; GFX11-NEXT: v_cmp_eq_f64_e64 s20, 0, v[50:51] -; GFX11-NEXT: v_cmp_eq_f64_e64 s21, 0, v[52:53] -; GFX11-NEXT: v_cndmask_b32_e64 v7, v39, v7, s4 -; GFX11-NEXT: v_cndmask_b32_e32 v32, v17, v1, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v6, v38, v6, s4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v33, 0x7ff80000, v32, s0 -; GFX11-NEXT: v_cndmask_b32_e32 v32, v16, v0, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v1, v33, v1, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v32, 0, v32, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v17, s11 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v0, v32, v0, s1 -; GFX11-NEXT: v_cmp_eq_f64_e64 s12, 0, v[32:33] -; GFX11-NEXT: v_cndmask_b32_e64 v2, v34, v2, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v36, v4, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v8, v48, v8, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v16, s11 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v50, v10, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v12, v52, v12, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v35, v3, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v37, v5, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v9, v49, v9, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v11, v51, v11, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v13, v53, v13, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v18, s13 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v20, s15 -; GFX11-NEXT: v_cndmask_b32_e64 v8, v8, v24, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v26, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v12, v12, v28, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v19, s13 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v21, s15 -; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v25, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v11, v11, v27, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v13, v13, v29, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v34, v2, s14 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v36, v4, s16 -; GFX11-NEXT: v_cndmask_b32_e64 v8, v48, v8, s18 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v50, v10, s20 -; GFX11-NEXT: v_cndmask_b32_e64 v12, v52, v12, s21 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v35, v3, s14 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v37, v5, s16 -; GFX11-NEXT: v_cndmask_b32_e64 v9, v49, v9, s18 -; GFX11-NEXT: v_cndmask_b32_e64 v11, v51, v11, s20 -; GFX11-NEXT: v_cndmask_b32_e64 v13, v53, v13, s21 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v32, v0, s12 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v33, v1, s12 +; GFX11-NEXT: v_max_f64 v[32:33], v[0:1], v[16:17] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[16:17] +; GFX11-NEXT: v_max_f64 v[16:17], v[2:3], v[18:19] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[18:19] +; GFX11-NEXT: v_max_f64 v[18:19], v[4:5], v[20:21] +; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[20:21] +; GFX11-NEXT: v_max_f64 v[20:21], v[6:7], v[22:23] +; GFX11-NEXT: v_cmp_u_f64_e64 s2, v[6:7], v[22:23] +; GFX11-NEXT: v_max_f64 v[22:23], v[8:9], v[24:25] +; GFX11-NEXT: v_cmp_u_f64_e64 s3, v[8:9], v[24:25] +; GFX11-NEXT: v_max_f64 v[24:25], v[10:11], v[26:27] +; GFX11-NEXT: v_cmp_u_f64_e64 s4, v[10:11], v[26:27] +; GFX11-NEXT: v_max_f64 v[26:27], v[12:13], v[28:29] +; GFX11-NEXT: v_cmp_u_f64_e64 s5, v[12:13], v[28:29] +; GFX11-NEXT: v_cndmask_b32_e64 v0, v32, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v33, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v16, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v17, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v4, v18, 0, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v5, v19, 0x7ff80000, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v6, v20, 0, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v7, v21, 0x7ff80000, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v8, v22, 0, s3 +; GFX11-NEXT: v_cndmask_b32_e64 v9, v23, 0x7ff80000, s3 +; GFX11-NEXT: v_cndmask_b32_e64 v10, v24, 0, s4 +; GFX11-NEXT: v_cndmask_b32_e64 v11, v25, 0x7ff80000, s4 +; GFX11-NEXT: v_cndmask_b32_e64 v12, v26, 0, s5 +; GFX11-NEXT: v_cndmask_b32_e64 v13, v27, 0x7ff80000, s5 ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[14:15], v[30:31] -; GFX11-NEXT: v_cmp_o_f64_e64 s0, v[14:15], v[30:31] -; GFX11-NEXT: v_cmp_class_f64_e64 s19, v[30:31], 64 -; GFX11-NEXT: v_cndmask_b32_e32 v54, v31, v15, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e32 v16, v30, v14, vcc_lo -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[22:23], 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v55, 0x7ff80000, v54, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v54, 0, v16, s0 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[14:15], 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cmp_eq_f64_e64 s22, 0, v[54:55] -; GFX11-NEXT: v_dual_cndmask_b32 v7, v7, v23 :: v_dual_cndmask_b32 v6, v6, v22 -; GFX11-NEXT: v_cndmask_b32_e64 v14, v54, v14, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v15, v55, v15, s0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v6, v38, v6, s17 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v39, v7, s17 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v14, v14, v30, s19 -; GFX11-NEXT: v_cndmask_b32_e64 v15, v15, v31, s19 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v14, v54, v14, s22 -; GFX11-NEXT: v_cndmask_b32_e64 v15, v55, v15, s22 +; GFX11-NEXT: v_max_f64 v[28:29], v[14:15], v[30:31] +; GFX11-NEXT: v_cmp_u_f64_e64 s6, v[14:15], v[30:31] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_cndmask_b32_e64 v14, v28, 0, s6 +; GFX11-NEXT: v_cndmask_b32_e64 v15, v29, 0x7ff80000, s6 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v8f64: @@ -4279,1799 +2067,798 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1) ; GFX7-LABEL: v_maximum_v16f64: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX7-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill +; GFX7-NEXT: s_mov_b64 exec, s[4:5] ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 ; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4 -; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:16 -; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12 -; GFX7-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:24 -; GFX7-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:20 -; GFX7-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:32 -; GFX7-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:28 -; GFX7-NEXT: v_mov_b32_e32 v39, 0x7ff80000 -; GFX7-NEXT: s_waitcnt vmcnt(6) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[31:32] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[31:32] -; GFX7-NEXT: s_waitcnt vmcnt(4) -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[2:3], v[33:34] -; GFX7-NEXT: v_cndmask_b32_e32 v48, v32, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v49, v39, v48, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v48, v31, v0, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[31:32], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v48, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v49, v1, vcc -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[33:34] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v31, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v32, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v50, v34, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v31, v33, v2, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v32, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v31, 0, v31, s[6:7] -; GFX7-NEXT: s_waitcnt vmcnt(2) -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[4:5], v[35:36] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v31, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v32, v3, vcc -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[35:36] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v33, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v34, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[35:36], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v50, v36, v5, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v33, v35, v4, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v34, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v33, 0, v33, s[6:7] +; GFX7-NEXT: v_writelane_b32 v34, s30, 0 +; GFX7-NEXT: v_writelane_b32 v34, s31, 1 +; GFX7-NEXT: v_writelane_b32 v34, s34, 2 +; GFX7-NEXT: v_writelane_b32 v34, s35, 3 ; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[6:7], v[37:38] -; GFX7-NEXT: v_cndmask_b32_e32 v4, v33, v4, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v5, v34, v5, vcc -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[37:38] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v35, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v36, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v50, v38, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v35, v37, v6, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v36, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v35, v6, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v7, v36, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, v37, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v38, s[4:5] -; GFX7-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36 -; GFX7-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:40 -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v48, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v49, v1, vcc -; GFX7-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:48 -; GFX7-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:44 -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[31:32] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v31, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v32, v3, vcc +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[31:32] +; GFX7-NEXT: v_max_f64 v[0:1], v[0:1], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:12 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[31:32] +; GFX7-NEXT: v_max_f64 v[2:3], v[2:3], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:20 +; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[4:5] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[31:32] +; GFX7-NEXT: v_max_f64 v[4:5], v[4:5], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:28 +; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, 0, s[6:7] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[31:32] +; GFX7-NEXT: v_max_f64 v[6:7], v[6:7], v[31:32] +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:36 +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 +; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, 0, s[8:9] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[10:11], v[8:9], v[31:32] +; GFX7-NEXT: v_max_f64 v[8:9], v[8:9], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:44 +; GFX7-NEXT: v_cndmask_b32_e64 v8, v8, 0, s[10:11] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[12:13], v[10:11], v[31:32] +; GFX7-NEXT: v_max_f64 v[10:11], v[10:11], v[31:32] ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 ; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:52 -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[33:34] -; GFX7-NEXT: v_cndmask_b32_e32 v4, v33, v4, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v5, v34, v5, vcc -; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 -; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[35:36] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v35, v6, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v7, v36, v7, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[8:9], 64 -; GFX7-NEXT: s_waitcnt vmcnt(6) -; GFX7-NEXT: v_cmp_gt_f64_e64 s[4:5], v[8:9], v[37:38] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[8:9], v[37:38] -; GFX7-NEXT: v_cndmask_b32_e64 v50, v38, v9, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v35, v37, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v36, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v35, v8, vcc -; GFX7-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[35:36] -; GFX7-NEXT: v_cndmask_b32_e32 v9, v36, v9, vcc -; GFX7-NEXT: s_waitcnt vmcnt(4) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[10:11], v[48:49] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[10:11], v[48:49] -; GFX7-NEXT: v_cndmask_b32_e64 v8, v8, v37, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v9, v38, s[4:5] -; GFX7-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:68 -; GFX7-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:72 -; GFX7-NEXT: v_cndmask_b32_e64 v8, v35, v8, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v50, v49, v11, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v35, v48, v10, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 64 -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[48:49], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v51, v39, v50, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v50, 0, v35, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v36, v9, s[6:7] -; GFX7-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:80 -; GFX7-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:76 -; GFX7-NEXT: s_waitcnt vmcnt(6) -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[12:13], v[31:32] -; GFX7-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v10, v10, v48, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v49, s[4:5] -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[50:51] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[4:5], v[12:13], v[31:32] -; GFX7-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v48, v32, v13, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[12:13], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v49, v39, v48, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v48, v31, v12, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[31:32], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX7-NEXT: s_waitcnt vmcnt(4) -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[14:15], v[33:34] -; GFX7-NEXT: v_cndmask_b32_e32 v12, v48, v12, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v13, v49, v13, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX7-NEXT: v_cndmask_b32_e64 v12, v12, v31, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v13, v13, v32, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v10, v10, 0, s[12:13] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[14:15], v[12:13], v[31:32] +; GFX7-NEXT: v_max_f64 v[12:13], v[12:13], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:60 +; GFX7-NEXT: v_cndmask_b32_e64 v12, v12, 0, s[14:15] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[16:17], v[14:15], v[31:32] +; GFX7-NEXT: v_max_f64 v[14:15], v[14:15], v[31:32] +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:68 +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 +; GFX7-NEXT: v_cndmask_b32_e64 v14, v14, 0, s[16:17] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[18:19], v[16:17], v[31:32] +; GFX7-NEXT: v_max_f64 v[16:17], v[16:17], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:76 +; GFX7-NEXT: v_cndmask_b32_e64 v16, v16, 0, s[18:19] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[20:21], v[18:19], v[31:32] +; GFX7-NEXT: v_max_f64 v[18:19], v[18:19], v[31:32] ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:84 -; GFX7-NEXT: v_cmp_gt_f64_e64 s[4:5], v[14:15], v[33:34] -; GFX7-NEXT: v_cndmask_b32_e32 v12, v48, v12, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v13, v49, v13, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v50, v34, v15, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v48, v33, v14, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v49, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v14, v48, v14, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v15, v49, v15, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX7-NEXT: v_cndmask_b32_e64 v14, v14, v33, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v15, v15, v34, s[4:5] -; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:96 -; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:92 -; GFX7-NEXT: v_cndmask_b32_e32 v14, v48, v14, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v15, v49, v15, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 64 -; GFX7-NEXT: s_waitcnt vmcnt(6) -; GFX7-NEXT: v_cmp_gt_f64_e64 s[4:5], v[16:17], v[37:38] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[16:17], v[37:38] -; GFX7-NEXT: v_cndmask_b32_e64 v50, v38, v17, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v48, v37, v16, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v49, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v48, v16, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v17, v49, v17, vcc -; GFX7-NEXT: s_waitcnt vmcnt(4) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[18:19], v[35:36] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[18:19], v[35:36] -; GFX7-NEXT: v_cndmask_b32_e64 v16, v16, v37, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v17, v17, v38, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v50, v36, v19, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v37, v35, v18, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[18:19], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v51, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v50, 0, v37, s[6:7] -; GFX7-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:100 -; GFX7-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:104 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v48, v16, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v17, v49, v17, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[35:36], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v18, v50, v18, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v19, v51, v19, s[4:5] -; GFX7-NEXT: s_waitcnt vmcnt(4) -; GFX7-NEXT: v_cmp_gt_f64_e64 s[4:5], v[20:21], v[31:32] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[20:21], v[31:32] -; GFX7-NEXT: v_cndmask_b32_e32 v18, v18, v35, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v19, v19, v36, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[50:51] -; GFX7-NEXT: v_cndmask_b32_e64 v48, v32, v21, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v35, v31, v20, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[20:21], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v36, v39, v48, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v18, v50, v18, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v19, v51, v19, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[31:32], 64 -; GFX7-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:112 -; GFX7-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:108 -; GFX7-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:120 -; GFX7-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:116 -; GFX7-NEXT: v_cndmask_b32_e64 v20, v35, v20, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v21, v36, v21, s[4:5] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[35:36] -; GFX7-NEXT: s_waitcnt vmcnt(6) -; GFX7-NEXT: v_cmp_gt_f64_e64 s[6:7], v[22:23], v[33:34] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[22:23], v[33:34] -; GFX7-NEXT: v_cndmask_b32_e32 v20, v20, v31, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v21, v21, v32, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[22:23], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v20, v35, v20, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v21, v36, v21, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v31, v34, v23, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v36, v39, v31, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v31, v33, v22, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v35, 0, v31, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[35:36] -; GFX7-NEXT: v_cndmask_b32_e64 v22, v22, v33, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v18, v18, 0, s[20:21] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[22:23], v[20:21], v[31:32] +; GFX7-NEXT: v_max_f64 v[20:21], v[20:21], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:92 +; GFX7-NEXT: v_cndmask_b32_e64 v20, v20, 0, s[22:23] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[24:25], v[22:23], v[31:32] +; GFX7-NEXT: v_max_f64 v[22:23], v[22:23], v[31:32] +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:100 +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 +; GFX7-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[24:25] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[26:27], v[24:25], v[31:32] +; GFX7-NEXT: v_max_f64 v[24:25], v[24:25], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:108 +; GFX7-NEXT: v_cndmask_b32_e64 v24, v24, 0, s[26:27] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[28:29], v[26:27], v[31:32] +; GFX7-NEXT: v_max_f64 v[26:27], v[26:27], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:116 +; GFX7-NEXT: v_cndmask_b32_e64 v26, v26, 0, s[28:29] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32] +; GFX7-NEXT: v_max_f64 v[28:29], v[28:29], v[31:32] ; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:128 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 -; GFX7-NEXT: v_cndmask_b32_e64 v23, v23, v34, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[24:25], 64 -; GFX7-NEXT: s_waitcnt vmcnt(7) -; GFX7-NEXT: v_cmp_gt_f64_e64 s[4:5], v[24:25], v[37:38] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[24:25], v[37:38] -; GFX7-NEXT: v_cndmask_b32_e64 v34, v38, v25, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v35, v39, v34, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v34, v37, v24, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v24, v34, v24, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v25, v35, v25, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[37:38], 64 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[34:35] -; GFX7-NEXT: v_cndmask_b32_e32 v24, v24, v37, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v25, v25, v38, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[26:27], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v24, v34, v24, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v25, v35, v25, s[4:5] -; GFX7-NEXT: s_waitcnt vmcnt(5) -; GFX7-NEXT: v_cmp_gt_f64_e64 s[6:7], v[26:27], v[48:49] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[26:27], v[48:49] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[48:49], 64 -; GFX7-NEXT: s_waitcnt vmcnt(3) -; GFX7-NEXT: v_cmp_o_f64_e64 s[10:11], v[28:29], v[50:51] -; GFX7-NEXT: v_cndmask_b32_e64 v36, v49, v27, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v34, v48, v26, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v35, v39, v36, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[8:9] -; GFX7-NEXT: v_cmp_gt_f64_e64 s[8:9], v[28:29], v[50:51] -; GFX7-NEXT: v_cndmask_b32_e32 v26, v34, v26, vcc -; GFX7-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[34:35] -; GFX7-NEXT: v_cndmask_b32_e32 v27, v35, v27, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v26, v26, v48, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v27, v27, v49, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v36, v51, v29, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v27, v35, v27, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v28, v28, 0, s[30:31] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33] +; GFX7-NEXT: v_max_f64 v[30:31], v[30:31], v[32:33] +; GFX7-NEXT: v_mov_b32_e32 v32, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v32, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v32, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v32, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v32, s[8:9] +; GFX7-NEXT: v_cndmask_b32_e64 v9, v9, v32, s[10:11] +; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v32, s[12:13] +; GFX7-NEXT: v_cndmask_b32_e64 v13, v13, v32, s[14:15] +; GFX7-NEXT: v_cndmask_b32_e64 v15, v15, v32, s[16:17] +; GFX7-NEXT: v_cndmask_b32_e64 v17, v17, v32, s[18:19] +; GFX7-NEXT: v_cndmask_b32_e64 v19, v19, v32, s[20:21] +; GFX7-NEXT: v_cndmask_b32_e64 v21, v21, v32, s[22:23] +; GFX7-NEXT: v_cndmask_b32_e64 v23, v23, v32, s[24:25] +; GFX7-NEXT: v_cndmask_b32_e64 v25, v25, v32, s[26:27] +; GFX7-NEXT: v_cndmask_b32_e64 v27, v27, v32, s[28:29] +; GFX7-NEXT: v_cndmask_b32_e64 v29, v29, v32, s[30:31] +; GFX7-NEXT: v_cndmask_b32_e64 v31, v31, v32, s[34:35] +; GFX7-NEXT: v_cndmask_b32_e64 v30, v30, 0, s[34:35] +; GFX7-NEXT: v_readlane_b32 s35, v34, 3 +; GFX7-NEXT: v_readlane_b32 s34, v34, 2 +; GFX7-NEXT: v_readlane_b32 s31, v34, 1 +; GFX7-NEXT: v_readlane_b32 s30, v34, 0 +; GFX7-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload +; GFX7-NEXT: s_mov_b64 exec, s[4:5] ; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_cmp_gt_f64_e32 vcc, v[30:31], v[32:33] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[30:31], v[32:33] -; GFX7-NEXT: v_cndmask_b32_e64 v35, v39, v36, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v26, v34, v26, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v34, v50, v28, s[8:9] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[28:29], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[10:11] -; GFX7-NEXT: v_cmp_class_f64_e64 s[8:9], v[50:51], 64 -; GFX7-NEXT: v_cndmask_b32_e32 v36, v33, v31, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v37, v39, v36, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v36, v32, v30, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[30:31], 64 -; GFX7-NEXT: v_cndmask_b32_e64 v36, 0, v36, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[32:33], 64 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[34:35] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[36:37] -; GFX7-NEXT: v_cndmask_b32_e64 v28, v34, v28, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v29, v35, v29, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v28, v28, v50, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v30, v36, v30, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v31, v37, v31, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v30, v30, v32, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v29, v29, v51, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v31, v31, v33, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v28, v34, v28, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v29, v35, v29, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v30, v36, v30, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v31, v37, v31, s[12:13] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_maximum_v16f64: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX8-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill +; GFX8-NEXT: s_mov_b64 exec, s[4:5] ; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 ; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4 -; GFX8-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:16 -; GFX8-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12 -; GFX8-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:24 -; GFX8-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:20 -; GFX8-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:32 -; GFX8-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:28 -; GFX8-NEXT: v_mov_b32_e32 v39, 0x7ff80000 -; GFX8-NEXT: s_waitcnt vmcnt(6) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[31:32] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[31:32] -; GFX8-NEXT: s_waitcnt vmcnt(4) -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[2:3], v[33:34] -; GFX8-NEXT: v_cndmask_b32_e32 v48, v32, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v49, v39, v48, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v48, v31, v0, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[31:32], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v48, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v49, v1, vcc -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[33:34] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v31, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v32, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v50, v34, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v31, v33, v2, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v32, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v31, 0, v31, s[6:7] -; GFX8-NEXT: s_waitcnt vmcnt(2) -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[4:5], v[35:36] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v31, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v32, v3, vcc -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[35:36] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v33, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v34, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[35:36], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v50, v36, v5, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v33, v35, v4, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v34, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v33, 0, v33, s[6:7] +; GFX8-NEXT: v_writelane_b32 v34, s30, 0 +; GFX8-NEXT: v_writelane_b32 v34, s31, 1 +; GFX8-NEXT: v_writelane_b32 v34, s34, 2 +; GFX8-NEXT: v_writelane_b32 v34, s35, 3 +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[31:32] +; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:12 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[31:32] +; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:20 +; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[4:5] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[31:32] +; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:28 +; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, 0, s[6:7] ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[6:7], v[37:38] -; GFX8-NEXT: v_cndmask_b32_e32 v4, v33, v4, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v5, v34, v5, vcc -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[37:38] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v35, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v36, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v50, v38, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v35, v37, v6, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v36, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v35, v6, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v7, v36, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v37, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v38, s[4:5] -; GFX8-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36 -; GFX8-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:40 -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v48, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v49, v1, vcc -; GFX8-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:48 -; GFX8-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:44 -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[31:32] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v31, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v32, v3, vcc +; GFX8-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[31:32] +; GFX8-NEXT: v_max_f64 v[6:7], v[6:7], v[31:32] +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:36 +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 +; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, 0, s[8:9] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[10:11], v[8:9], v[31:32] +; GFX8-NEXT: v_max_f64 v[8:9], v[8:9], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:44 +; GFX8-NEXT: v_cndmask_b32_e64 v8, v8, 0, s[10:11] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[12:13], v[10:11], v[31:32] +; GFX8-NEXT: v_max_f64 v[10:11], v[10:11], v[31:32] ; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 ; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:52 -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[33:34] -; GFX8-NEXT: v_cndmask_b32_e32 v4, v33, v4, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v5, v34, v5, vcc -; GFX8-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 -; GFX8-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[35:36] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v35, v6, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v7, v36, v7, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[8:9], 64 -; GFX8-NEXT: s_waitcnt vmcnt(6) -; GFX8-NEXT: v_cmp_gt_f64_e64 s[4:5], v[8:9], v[37:38] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[8:9], v[37:38] -; GFX8-NEXT: v_cndmask_b32_e64 v50, v38, v9, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v35, v37, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v36, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v35, v8, vcc -; GFX8-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[35:36] -; GFX8-NEXT: v_cndmask_b32_e32 v9, v36, v9, vcc -; GFX8-NEXT: s_waitcnt vmcnt(4) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[10:11], v[48:49] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[10:11], v[48:49] -; GFX8-NEXT: v_cndmask_b32_e64 v8, v8, v37, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v9, v38, s[4:5] -; GFX8-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:68 -; GFX8-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:72 -; GFX8-NEXT: v_cndmask_b32_e64 v8, v35, v8, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v50, v49, v11, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v35, v48, v10, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 64 -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[48:49], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v51, v39, v50, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v50, 0, v35, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v36, v9, s[6:7] -; GFX8-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:80 -; GFX8-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:76 -; GFX8-NEXT: s_waitcnt vmcnt(6) -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[12:13], v[31:32] -; GFX8-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, v48, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v49, s[4:5] -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[50:51] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[4:5], v[12:13], v[31:32] -; GFX8-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v48, v32, v13, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[12:13], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v49, v39, v48, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v48, v31, v12, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[31:32], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX8-NEXT: s_waitcnt vmcnt(4) -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[14:15], v[33:34] -; GFX8-NEXT: v_cndmask_b32_e32 v12, v48, v12, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v13, v49, v13, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX8-NEXT: v_cndmask_b32_e64 v12, v12, v31, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v13, v13, v32, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, 0, s[12:13] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[14:15], v[12:13], v[31:32] +; GFX8-NEXT: v_max_f64 v[12:13], v[12:13], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:60 +; GFX8-NEXT: v_cndmask_b32_e64 v12, v12, 0, s[14:15] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[16:17], v[14:15], v[31:32] +; GFX8-NEXT: v_max_f64 v[14:15], v[14:15], v[31:32] +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:68 +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 +; GFX8-NEXT: v_cndmask_b32_e64 v14, v14, 0, s[16:17] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[18:19], v[16:17], v[31:32] +; GFX8-NEXT: v_max_f64 v[16:17], v[16:17], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:76 +; GFX8-NEXT: v_cndmask_b32_e64 v16, v16, 0, s[18:19] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[20:21], v[18:19], v[31:32] +; GFX8-NEXT: v_max_f64 v[18:19], v[18:19], v[31:32] ; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:84 -; GFX8-NEXT: v_cmp_gt_f64_e64 s[4:5], v[14:15], v[33:34] -; GFX8-NEXT: v_cndmask_b32_e32 v12, v48, v12, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v13, v49, v13, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v50, v34, v15, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v48, v33, v14, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v49, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v14, v48, v14, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v15, v49, v15, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX8-NEXT: v_cndmask_b32_e64 v14, v14, v33, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v15, v15, v34, s[4:5] -; GFX8-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:96 -; GFX8-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:92 -; GFX8-NEXT: v_cndmask_b32_e32 v14, v48, v14, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v15, v49, v15, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 64 -; GFX8-NEXT: s_waitcnt vmcnt(6) -; GFX8-NEXT: v_cmp_gt_f64_e64 s[4:5], v[16:17], v[37:38] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[16:17], v[37:38] -; GFX8-NEXT: v_cndmask_b32_e64 v50, v38, v17, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v48, v37, v16, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v49, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v48, v16, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v17, v49, v17, vcc -; GFX8-NEXT: s_waitcnt vmcnt(4) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[18:19], v[35:36] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[18:19], v[35:36] -; GFX8-NEXT: v_cndmask_b32_e64 v16, v16, v37, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v17, v17, v38, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v50, v36, v19, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v37, v35, v18, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[18:19], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v51, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v50, 0, v37, s[6:7] -; GFX8-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:100 -; GFX8-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:104 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v48, v16, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v17, v49, v17, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[35:36], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v18, v50, v18, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v19, v51, v19, s[4:5] -; GFX8-NEXT: s_waitcnt vmcnt(4) -; GFX8-NEXT: v_cmp_gt_f64_e64 s[4:5], v[20:21], v[31:32] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[20:21], v[31:32] -; GFX8-NEXT: v_cndmask_b32_e32 v18, v18, v35, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v19, v19, v36, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[50:51] -; GFX8-NEXT: v_cndmask_b32_e64 v48, v32, v21, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v35, v31, v20, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[20:21], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v36, v39, v48, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v18, v50, v18, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v19, v51, v19, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[31:32], 64 -; GFX8-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:112 -; GFX8-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:108 -; GFX8-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:120 -; GFX8-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:116 -; GFX8-NEXT: v_cndmask_b32_e64 v20, v35, v20, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v21, v36, v21, s[4:5] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[35:36] -; GFX8-NEXT: s_waitcnt vmcnt(6) -; GFX8-NEXT: v_cmp_gt_f64_e64 s[6:7], v[22:23], v[33:34] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[22:23], v[33:34] -; GFX8-NEXT: v_cndmask_b32_e32 v20, v20, v31, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v21, v21, v32, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[22:23], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v20, v35, v20, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v21, v36, v21, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v31, v34, v23, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v36, v39, v31, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v31, v33, v22, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v35, 0, v31, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[35:36] -; GFX8-NEXT: v_cndmask_b32_e64 v22, v22, v33, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v18, v18, 0, s[20:21] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[22:23], v[20:21], v[31:32] +; GFX8-NEXT: v_max_f64 v[20:21], v[20:21], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:92 +; GFX8-NEXT: v_cndmask_b32_e64 v20, v20, 0, s[22:23] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[24:25], v[22:23], v[31:32] +; GFX8-NEXT: v_max_f64 v[22:23], v[22:23], v[31:32] +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:100 +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 +; GFX8-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[24:25] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[26:27], v[24:25], v[31:32] +; GFX8-NEXT: v_max_f64 v[24:25], v[24:25], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:108 +; GFX8-NEXT: v_cndmask_b32_e64 v24, v24, 0, s[26:27] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[28:29], v[26:27], v[31:32] +; GFX8-NEXT: v_max_f64 v[26:27], v[26:27], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:116 +; GFX8-NEXT: v_cndmask_b32_e64 v26, v26, 0, s[28:29] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32] +; GFX8-NEXT: v_max_f64 v[28:29], v[28:29], v[31:32] ; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; GFX8-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:128 ; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 -; GFX8-NEXT: v_cndmask_b32_e64 v23, v23, v34, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[24:25], 64 -; GFX8-NEXT: s_waitcnt vmcnt(7) -; GFX8-NEXT: v_cmp_gt_f64_e64 s[4:5], v[24:25], v[37:38] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[24:25], v[37:38] -; GFX8-NEXT: v_cndmask_b32_e64 v34, v38, v25, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v35, v39, v34, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v34, v37, v24, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v24, v34, v24, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v25, v35, v25, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[37:38], 64 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[34:35] -; GFX8-NEXT: v_cndmask_b32_e32 v24, v24, v37, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v25, v25, v38, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[26:27], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v24, v34, v24, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v25, v35, v25, s[4:5] -; GFX8-NEXT: s_waitcnt vmcnt(5) -; GFX8-NEXT: v_cmp_gt_f64_e64 s[6:7], v[26:27], v[48:49] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[26:27], v[48:49] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[48:49], 64 -; GFX8-NEXT: s_waitcnt vmcnt(3) -; GFX8-NEXT: v_cmp_o_f64_e64 s[10:11], v[28:29], v[50:51] -; GFX8-NEXT: v_cndmask_b32_e64 v36, v49, v27, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v34, v48, v26, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v35, v39, v36, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[8:9] -; GFX8-NEXT: v_cmp_gt_f64_e64 s[8:9], v[28:29], v[50:51] -; GFX8-NEXT: v_cndmask_b32_e32 v26, v34, v26, vcc -; GFX8-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[34:35] -; GFX8-NEXT: v_cndmask_b32_e32 v27, v35, v27, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v26, v26, v48, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v27, v27, v49, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v36, v51, v29, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v27, v35, v27, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v28, v28, 0, s[30:31] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33] +; GFX8-NEXT: v_max_f64 v[30:31], v[30:31], v[32:33] +; GFX8-NEXT: v_mov_b32_e32 v32, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v32, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v32, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v32, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v32, s[8:9] +; GFX8-NEXT: v_cndmask_b32_e64 v9, v9, v32, s[10:11] +; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v32, s[12:13] +; GFX8-NEXT: v_cndmask_b32_e64 v13, v13, v32, s[14:15] +; GFX8-NEXT: v_cndmask_b32_e64 v15, v15, v32, s[16:17] +; GFX8-NEXT: v_cndmask_b32_e64 v17, v17, v32, s[18:19] +; GFX8-NEXT: v_cndmask_b32_e64 v19, v19, v32, s[20:21] +; GFX8-NEXT: v_cndmask_b32_e64 v21, v21, v32, s[22:23] +; GFX8-NEXT: v_cndmask_b32_e64 v23, v23, v32, s[24:25] +; GFX8-NEXT: v_cndmask_b32_e64 v25, v25, v32, s[26:27] +; GFX8-NEXT: v_cndmask_b32_e64 v27, v27, v32, s[28:29] +; GFX8-NEXT: v_cndmask_b32_e64 v29, v29, v32, s[30:31] +; GFX8-NEXT: v_cndmask_b32_e64 v31, v31, v32, s[34:35] +; GFX8-NEXT: v_cndmask_b32_e64 v30, v30, 0, s[34:35] +; GFX8-NEXT: v_readlane_b32 s35, v34, 3 +; GFX8-NEXT: v_readlane_b32 s34, v34, 2 +; GFX8-NEXT: v_readlane_b32 s31, v34, 1 +; GFX8-NEXT: v_readlane_b32 s30, v34, 0 +; GFX8-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX8-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload +; GFX8-NEXT: s_mov_b64 exec, s[4:5] ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_cmp_gt_f64_e32 vcc, v[30:31], v[32:33] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[30:31], v[32:33] -; GFX8-NEXT: v_cndmask_b32_e64 v35, v39, v36, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v26, v34, v26, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v34, v50, v28, s[8:9] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[28:29], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[10:11] -; GFX8-NEXT: v_cmp_class_f64_e64 s[8:9], v[50:51], 64 -; GFX8-NEXT: v_cndmask_b32_e32 v36, v33, v31, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v37, v39, v36, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v36, v32, v30, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[30:31], 64 -; GFX8-NEXT: v_cndmask_b32_e64 v36, 0, v36, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[32:33], 64 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[34:35] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[36:37] -; GFX8-NEXT: v_cndmask_b32_e64 v28, v34, v28, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v29, v35, v29, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v28, v28, v50, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v30, v36, v30, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v31, v37, v31, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v30, v30, v32, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v29, v29, v51, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v31, v31, v33, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v28, v34, v28, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v29, v35, v29, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v30, v36, v30, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v31, v37, v31, s[12:13] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maximum_v16f64: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill +; GFX9-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 ; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4 -; GFX9-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:16 -; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12 -; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:24 -; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:20 -; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:32 -; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:28 -; GFX9-NEXT: v_mov_b32_e32 v39, 0x7ff80000 -; GFX9-NEXT: s_waitcnt vmcnt(6) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[31:32] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[31:32] -; GFX9-NEXT: s_waitcnt vmcnt(4) -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[2:3], v[33:34] -; GFX9-NEXT: v_cndmask_b32_e32 v48, v32, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v49, v39, v48, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v48, v31, v0, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[31:32], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v48, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v49, v1, vcc -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[33:34] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v31, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v32, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v50, v34, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v31, v33, v2, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v32, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v31, 0, v31, s[6:7] -; GFX9-NEXT: s_waitcnt vmcnt(2) -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[4:5], v[35:36] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v31, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v32, v3, vcc -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[35:36] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v33, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v34, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[35:36], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v50, v36, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v33, v35, v4, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v34, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v33, 0, v33, s[6:7] +; GFX9-NEXT: v_writelane_b32 v34, s30, 0 +; GFX9-NEXT: v_writelane_b32 v34, s31, 1 +; GFX9-NEXT: v_writelane_b32 v34, s34, 2 +; GFX9-NEXT: v_writelane_b32 v34, s35, 3 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[31:32] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:12 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[6:7], v[37:38] -; GFX9-NEXT: v_cndmask_b32_e32 v4, v33, v4, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v5, v34, v5, vcc -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[37:38] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v35, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v36, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v50, v38, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v35, v37, v6, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v36, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v35, v6, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v7, v36, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v37, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v38, s[4:5] -; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36 -; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:40 -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v48, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v49, v1, vcc -; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:48 -; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:44 -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[31:32] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v31, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v32, v3, vcc +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[31:32] +; GFX9-NEXT: v_max_f64 v[2:3], v[2:3], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:20 +; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[4:5] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[31:32] +; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:28 +; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, 0, s[6:7] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[31:32] +; GFX9-NEXT: v_max_f64 v[6:7], v[6:7], v[31:32] +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:36 +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 +; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, 0, s[8:9] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[10:11], v[8:9], v[31:32] +; GFX9-NEXT: v_max_f64 v[8:9], v[8:9], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:44 +; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, 0, s[10:11] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[12:13], v[10:11], v[31:32] +; GFX9-NEXT: v_max_f64 v[10:11], v[10:11], v[31:32] ; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 ; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:52 -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[33:34] -; GFX9-NEXT: v_cndmask_b32_e32 v4, v33, v4, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v5, v34, v5, vcc -; GFX9-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 -; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[35:36] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v35, v6, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v7, v36, v7, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[8:9], 64 -; GFX9-NEXT: s_waitcnt vmcnt(6) -; GFX9-NEXT: v_cmp_gt_f64_e64 s[4:5], v[8:9], v[37:38] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[8:9], v[37:38] -; GFX9-NEXT: v_cndmask_b32_e64 v50, v38, v9, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v35, v37, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v36, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v35, v8, vcc -; GFX9-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[35:36] -; GFX9-NEXT: v_cndmask_b32_e32 v9, v36, v9, vcc -; GFX9-NEXT: s_waitcnt vmcnt(4) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[10:11], v[48:49] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[10:11], v[48:49] -; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v37, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v38, s[4:5] -; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:68 -; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:72 -; GFX9-NEXT: v_cndmask_b32_e64 v8, v35, v8, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v50, v49, v11, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v35, v48, v10, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 64 -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[48:49], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v51, v39, v50, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v50, 0, v35, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v36, v9, s[6:7] -; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:80 -; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:76 -; GFX9-NEXT: s_waitcnt vmcnt(6) -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[12:13], v[31:32] -; GFX9-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v48, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v49, s[4:5] -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[50:51] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[4:5], v[12:13], v[31:32] -; GFX9-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v48, v32, v13, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[12:13], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v49, v39, v48, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v48, v31, v12, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[31:32], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX9-NEXT: s_waitcnt vmcnt(4) -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[14:15], v[33:34] -; GFX9-NEXT: v_cndmask_b32_e32 v12, v48, v12, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v13, v49, v13, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, v31, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v13, v13, v32, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, 0, s[12:13] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[14:15], v[12:13], v[31:32] +; GFX9-NEXT: v_max_f64 v[12:13], v[12:13], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:60 +; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, 0, s[14:15] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[16:17], v[14:15], v[31:32] +; GFX9-NEXT: v_max_f64 v[14:15], v[14:15], v[31:32] +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:68 +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 +; GFX9-NEXT: v_cndmask_b32_e64 v14, v14, 0, s[16:17] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[18:19], v[16:17], v[31:32] +; GFX9-NEXT: v_max_f64 v[16:17], v[16:17], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:76 +; GFX9-NEXT: v_cndmask_b32_e64 v16, v16, 0, s[18:19] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[20:21], v[18:19], v[31:32] +; GFX9-NEXT: v_max_f64 v[18:19], v[18:19], v[31:32] ; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:84 -; GFX9-NEXT: v_cmp_gt_f64_e64 s[4:5], v[14:15], v[33:34] -; GFX9-NEXT: v_cndmask_b32_e32 v12, v48, v12, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v13, v49, v13, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v50, v34, v15, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v48, v33, v14, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v49, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v14, v48, v14, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v15, v49, v15, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX9-NEXT: v_cndmask_b32_e64 v14, v14, v33, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v15, v15, v34, s[4:5] -; GFX9-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:96 -; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:92 -; GFX9-NEXT: v_cndmask_b32_e32 v14, v48, v14, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v15, v49, v15, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 64 -; GFX9-NEXT: s_waitcnt vmcnt(6) -; GFX9-NEXT: v_cmp_gt_f64_e64 s[4:5], v[16:17], v[37:38] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[16:17], v[37:38] -; GFX9-NEXT: v_cndmask_b32_e64 v50, v38, v17, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v48, v37, v16, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v49, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v48, v16, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v17, v49, v17, vcc -; GFX9-NEXT: s_waitcnt vmcnt(4) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[18:19], v[35:36] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[18:19], v[35:36] -; GFX9-NEXT: v_cndmask_b32_e64 v16, v16, v37, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v17, v17, v38, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v50, v36, v19, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v37, v35, v18, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[18:19], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v51, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v50, 0, v37, s[6:7] -; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:100 -; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:104 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v48, v16, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v17, v49, v17, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[35:36], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v18, v50, v18, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v19, v51, v19, s[4:5] -; GFX9-NEXT: s_waitcnt vmcnt(4) -; GFX9-NEXT: v_cmp_gt_f64_e64 s[4:5], v[20:21], v[31:32] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[20:21], v[31:32] -; GFX9-NEXT: v_cndmask_b32_e32 v18, v18, v35, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v19, v19, v36, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[50:51] -; GFX9-NEXT: v_cndmask_b32_e64 v48, v32, v21, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v35, v31, v20, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[20:21], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v36, v39, v48, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v18, v50, v18, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v19, v51, v19, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[31:32], 64 -; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:112 -; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:108 -; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:120 -; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:116 -; GFX9-NEXT: v_cndmask_b32_e64 v20, v35, v20, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v21, v36, v21, s[4:5] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[35:36] -; GFX9-NEXT: s_waitcnt vmcnt(6) -; GFX9-NEXT: v_cmp_gt_f64_e64 s[6:7], v[22:23], v[33:34] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[22:23], v[33:34] -; GFX9-NEXT: v_cndmask_b32_e32 v20, v20, v31, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v21, v21, v32, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[22:23], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v20, v35, v20, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v21, v36, v21, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v31, v34, v23, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v36, v39, v31, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v31, v33, v22, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v35, 0, v31, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[35:36] -; GFX9-NEXT: v_cndmask_b32_e64 v22, v22, v33, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v18, v18, 0, s[20:21] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[22:23], v[20:21], v[31:32] +; GFX9-NEXT: v_max_f64 v[20:21], v[20:21], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:92 +; GFX9-NEXT: v_cndmask_b32_e64 v20, v20, 0, s[22:23] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[24:25], v[22:23], v[31:32] +; GFX9-NEXT: v_max_f64 v[22:23], v[22:23], v[31:32] +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:100 +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 +; GFX9-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[24:25] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[26:27], v[24:25], v[31:32] +; GFX9-NEXT: v_max_f64 v[24:25], v[24:25], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:108 +; GFX9-NEXT: v_cndmask_b32_e64 v24, v24, 0, s[26:27] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[28:29], v[26:27], v[31:32] +; GFX9-NEXT: v_max_f64 v[26:27], v[26:27], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:116 +; GFX9-NEXT: v_cndmask_b32_e64 v26, v26, 0, s[28:29] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32] +; GFX9-NEXT: v_max_f64 v[28:29], v[28:29], v[31:32] ; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:128 ; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 -; GFX9-NEXT: v_cndmask_b32_e64 v23, v23, v34, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[24:25], 64 -; GFX9-NEXT: s_waitcnt vmcnt(7) -; GFX9-NEXT: v_cmp_gt_f64_e64 s[4:5], v[24:25], v[37:38] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[24:25], v[37:38] -; GFX9-NEXT: v_cndmask_b32_e64 v34, v38, v25, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v35, v39, v34, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v34, v37, v24, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v24, v34, v24, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v25, v35, v25, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[37:38], 64 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[34:35] -; GFX9-NEXT: v_cndmask_b32_e32 v24, v24, v37, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v25, v25, v38, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[26:27], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v24, v34, v24, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v25, v35, v25, s[4:5] -; GFX9-NEXT: s_waitcnt vmcnt(5) -; GFX9-NEXT: v_cmp_gt_f64_e64 s[6:7], v[26:27], v[48:49] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[26:27], v[48:49] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[48:49], 64 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: v_cmp_o_f64_e64 s[10:11], v[28:29], v[50:51] -; GFX9-NEXT: v_cndmask_b32_e64 v36, v49, v27, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v34, v48, v26, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v35, v39, v36, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[8:9] -; GFX9-NEXT: v_cmp_gt_f64_e64 s[8:9], v[28:29], v[50:51] -; GFX9-NEXT: v_cndmask_b32_e32 v26, v34, v26, vcc -; GFX9-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[34:35] -; GFX9-NEXT: v_cndmask_b32_e32 v27, v35, v27, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v26, v26, v48, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v27, v27, v49, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v36, v51, v29, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v27, v35, v27, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v28, v28, 0, s[30:31] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33] +; GFX9-NEXT: v_max_f64 v[30:31], v[30:31], v[32:33] +; GFX9-NEXT: v_mov_b32_e32 v32, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v32, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v32, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v32, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v32, s[8:9] +; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v32, s[10:11] +; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v32, s[12:13] +; GFX9-NEXT: v_cndmask_b32_e64 v13, v13, v32, s[14:15] +; GFX9-NEXT: v_cndmask_b32_e64 v15, v15, v32, s[16:17] +; GFX9-NEXT: v_cndmask_b32_e64 v17, v17, v32, s[18:19] +; GFX9-NEXT: v_cndmask_b32_e64 v19, v19, v32, s[20:21] +; GFX9-NEXT: v_cndmask_b32_e64 v21, v21, v32, s[22:23] +; GFX9-NEXT: v_cndmask_b32_e64 v23, v23, v32, s[24:25] +; GFX9-NEXT: v_cndmask_b32_e64 v25, v25, v32, s[26:27] +; GFX9-NEXT: v_cndmask_b32_e64 v27, v27, v32, s[28:29] +; GFX9-NEXT: v_cndmask_b32_e64 v29, v29, v32, s[30:31] +; GFX9-NEXT: v_cndmask_b32_e64 v31, v31, v32, s[34:35] +; GFX9-NEXT: v_cndmask_b32_e64 v30, v30, 0, s[34:35] +; GFX9-NEXT: v_readlane_b32 s35, v34, 3 +; GFX9-NEXT: v_readlane_b32 s34, v34, 2 +; GFX9-NEXT: v_readlane_b32 s31, v34, 1 +; GFX9-NEXT: v_readlane_b32 s30, v34, 0 +; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload +; GFX9-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_cmp_gt_f64_e32 vcc, v[30:31], v[32:33] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[30:31], v[32:33] -; GFX9-NEXT: v_cndmask_b32_e64 v35, v39, v36, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v26, v34, v26, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v34, v50, v28, s[8:9] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[28:29], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[10:11] -; GFX9-NEXT: v_cmp_class_f64_e64 s[8:9], v[50:51], 64 -; GFX9-NEXT: v_cndmask_b32_e32 v36, v33, v31, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v37, v39, v36, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v36, v32, v30, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[30:31], 64 -; GFX9-NEXT: v_cndmask_b32_e64 v36, 0, v36, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[32:33], 64 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[34:35] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[36:37] -; GFX9-NEXT: v_cndmask_b32_e64 v28, v34, v28, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v29, v35, v29, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v28, v28, v50, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v30, v36, v30, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v31, v37, v31, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v30, v30, v32, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v29, v29, v51, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v31, v31, v33, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v28, v34, v28, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v29, v35, v29, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v30, v36, v30, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v31, v37, v31, s[12:13] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_maximum_v16f64: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a1, v41 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a2, v42 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a3, v43 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a4, v44 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a5, v45 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a6, v46 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a7, v47 ; Reload Reuse -; GFX940-NEXT: scratch_load_dword v41, off, s32 offset:8 -; GFX940-NEXT: scratch_load_dword v40, off, s32 offset:4 -; GFX940-NEXT: scratch_load_dword v51, off, s32 offset:16 -; GFX940-NEXT: scratch_load_dword v50, off, s32 offset:12 -; GFX940-NEXT: scratch_load_dword v45, off, s32 offset:24 -; GFX940-NEXT: scratch_load_dword v44, off, s32 offset:20 -; GFX940-NEXT: scratch_load_dword v47, off, s32 offset:32 -; GFX940-NEXT: scratch_load_dword v46, off, s32 offset:28 +; GFX940-NEXT: v_accvgpr_write_b32 a1, v40 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a2, v41 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a3, v42 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a4, v43 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a5, v44 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a6, v45 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a7, v46 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a8, v47 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a9, v56 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a10, v57 ; Reload Reuse +; GFX940-NEXT: scratch_load_dword v37, off, s32 offset:16 +; GFX940-NEXT: scratch_load_dword v36, off, s32 offset:12 +; GFX940-NEXT: scratch_load_dword v39, off, s32 offset:24 +; GFX940-NEXT: scratch_load_dword v38, off, s32 offset:20 +; GFX940-NEXT: scratch_load_dword v49, off, s32 offset:32 +; GFX940-NEXT: scratch_load_dword v48, off, s32 offset:28 +; GFX940-NEXT: scratch_load_dword v57, off, s32 offset:8 +; GFX940-NEXT: scratch_load_dword v56, off, s32 offset:4 +; GFX940-NEXT: scratch_load_dword v47, off, s32 offset:40 +; GFX940-NEXT: scratch_load_dword v46, off, s32 offset:36 +; GFX940-NEXT: scratch_load_dword v45, off, s32 offset:48 +; GFX940-NEXT: scratch_load_dword v44, off, s32 offset:44 +; GFX940-NEXT: scratch_load_dword v43, off, s32 offset:56 +; GFX940-NEXT: scratch_load_dword v42, off, s32 offset:52 +; GFX940-NEXT: scratch_load_dword v41, off, s32 offset:64 +; GFX940-NEXT: scratch_load_dword v40, off, s32 offset:60 +; GFX940-NEXT: scratch_load_dword v55, off, s32 offset:72 +; GFX940-NEXT: scratch_load_dword v54, off, s32 offset:68 +; GFX940-NEXT: scratch_load_dword v53, off, s32 offset:80 +; GFX940-NEXT: scratch_load_dword v52, off, s32 offset:76 +; GFX940-NEXT: scratch_load_dword v51, off, s32 offset:88 +; GFX940-NEXT: scratch_load_dword v50, off, s32 offset:84 +; GFX940-NEXT: scratch_load_dword v35, off, s32 offset:96 +; GFX940-NEXT: scratch_load_dword v34, off, s32 offset:92 ; GFX940-NEXT: scratch_load_dword v31, off, s32 -; GFX940-NEXT: scratch_load_dword v33, off, s32 offset:128 -; GFX940-NEXT: scratch_load_dword v32, off, s32 offset:124 -; GFX940-NEXT: scratch_load_dword v35, off, s32 offset:120 -; GFX940-NEXT: scratch_load_dword v34, off, s32 offset:116 -; GFX940-NEXT: scratch_load_dword v43, off, s32 offset:40 -; GFX940-NEXT: scratch_load_dword v42, off, s32 offset:36 +; GFX940-NEXT: scratch_load_dword v33, off, s32 offset:104 +; GFX940-NEXT: scratch_load_dword v32, off, s32 offset:100 +; GFX940-NEXT: v_accvgpr_write_b32 a11, v58 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a12, v59 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a13, v60 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a14, v61 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a15, v62 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a16, v63 ; Reload Reuse +; GFX940-NEXT: s_waitcnt vmcnt(25) +; GFX940-NEXT: v_max_f64 v[58:59], v[2:3], v[36:37] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[36:37] ; GFX940-NEXT: scratch_load_dword v37, off, s32 offset:112 ; GFX940-NEXT: scratch_load_dword v36, off, s32 offset:108 -; GFX940-NEXT: scratch_load_dword v39, off, s32 offset:104 -; GFX940-NEXT: scratch_load_dword v38, off, s32 offset:100 -; GFX940-NEXT: scratch_load_dword v49, off, s32 offset:96 -; GFX940-NEXT: scratch_load_dword v48, off, s32 offset:92 -; GFX940-NEXT: scratch_load_dword v53, off, s32 offset:56 -; GFX940-NEXT: scratch_load_dword v52, off, s32 offset:52 -; GFX940-NEXT: scratch_load_dword v55, off, s32 offset:48 -; GFX940-NEXT: scratch_load_dword v54, off, s32 offset:44 -; GFX940-NEXT: v_accvgpr_write_b32 a8, v56 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a9, v57 ; Reload Reuse -; GFX940-NEXT: v_mov_b32_e32 v56, 0x7ff80000 -; GFX940-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a10, v58 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a12, v60 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a13, v61 ; Reload Reuse +; GFX940-NEXT: s_waitcnt vmcnt(25) +; GFX940-NEXT: v_max_f64 v[60:61], v[4:5], v[38:39] +; GFX940-NEXT: v_cmp_u_f64_e64 s[0:1], v[4:5], v[38:39] +; GFX940-NEXT: scratch_load_dword v39, off, s32 offset:120 +; GFX940-NEXT: scratch_load_dword v38, off, s32 offset:116 +; GFX940-NEXT: s_waitcnt vmcnt(25) +; GFX940-NEXT: v_max_f64 v[62:63], v[6:7], v[48:49] +; GFX940-NEXT: v_cmp_u_f64_e64 s[2:3], v[6:7], v[48:49] +; GFX940-NEXT: scratch_load_dword v49, off, s32 offset:128 +; GFX940-NEXT: scratch_load_dword v48, off, s32 offset:124 +; GFX940-NEXT: s_waitcnt vmcnt(25) +; GFX940-NEXT: v_max_f64 v[2:3], v[0:1], v[56:57] +; GFX940-NEXT: v_cmp_u_f64_e64 s[4:5], v[0:1], v[56:57] +; GFX940-NEXT: v_mov_b32_e32 v0, 0x7ff80000 ; GFX940-NEXT: s_waitcnt vmcnt(23) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[0:1], v[40:41] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v57, v41, v1, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[40:41] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e64 v59, v56, v57, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v57, v40, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v58, 0, v57, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[40:41], 64 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v58, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v59, v1, vcc +; GFX940-NEXT: v_max_f64 v[56:57], v[8:9], v[46:47] +; GFX940-NEXT: v_cndmask_b32_e64 v1, v2, 0, s[4:5] +; GFX940-NEXT: v_accvgpr_write_b32 a0, v1 +; GFX940-NEXT: v_cndmask_b32_e64 v1, v3, v0, s[4:5] +; GFX940-NEXT: v_cndmask_b32_e64 v2, v58, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v59, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[8:9], v[46:47] ; GFX940-NEXT: s_waitcnt vmcnt(21) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[2:3], v[50:51] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v40, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v41, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v57, v51, v3, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[50:51] -; GFX940-NEXT: v_cndmask_b32_e32 v40, v50, v2, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 64 -; GFX940-NEXT: v_cndmask_b32_e64 v61, v56, v57, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v60, 0, v40, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v60, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v61, v3, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[50:51], 64 -; GFX940-NEXT: scratch_load_dword v41, off, s32 offset:64 -; GFX940-NEXT: scratch_load_dword v40, off, s32 offset:60 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v50, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v51, vcc -; GFX940-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[58:59] -; GFX940-NEXT: s_waitcnt vmcnt(21) -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[4:5], v[44:45] -; GFX940-NEXT: scratch_load_dword v51, off, s32 offset:88 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v58, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v59, v1, vcc -; GFX940-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[60:61] +; GFX940-NEXT: v_max_f64 v[46:47], v[10:11], v[44:45] +; GFX940-NEXT: v_cndmask_b32_e64 v4, v60, 0, s[0:1] +; GFX940-NEXT: v_cndmask_b32_e64 v8, v56, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v9, v57, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[10:11], v[44:45] +; GFX940-NEXT: s_waitcnt vmcnt(19) +; GFX940-NEXT: v_max_f64 v[44:45], v[12:13], v[42:43] +; GFX940-NEXT: v_cndmask_b32_e64 v5, v61, v0, s[0:1] +; GFX940-NEXT: v_cndmask_b32_e64 v10, v46, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v11, v47, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[12:13], v[42:43] +; GFX940-NEXT: s_waitcnt vmcnt(17) +; GFX940-NEXT: v_max_f64 v[42:43], v[14:15], v[40:41] +; GFX940-NEXT: v_cndmask_b32_e64 v6, v62, 0, s[2:3] +; GFX940-NEXT: v_cndmask_b32_e64 v12, v44, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v13, v45, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[14:15], v[40:41] +; GFX940-NEXT: s_waitcnt vmcnt(15) +; GFX940-NEXT: v_max_f64 v[40:41], v[16:17], v[54:55] +; GFX940-NEXT: v_cndmask_b32_e64 v7, v63, v0, s[2:3] +; GFX940-NEXT: v_cndmask_b32_e64 v14, v42, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v15, v43, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[16:17], v[54:55] +; GFX940-NEXT: s_waitcnt vmcnt(13) +; GFX940-NEXT: v_max_f64 v[54:55], v[18:19], v[52:53] +; GFX940-NEXT: v_accvgpr_read_b32 v63, a16 ; Reload Reuse +; GFX940-NEXT: v_cndmask_b32_e64 v16, v40, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v17, v41, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[18:19], v[52:53] +; GFX940-NEXT: s_waitcnt vmcnt(11) +; GFX940-NEXT: v_max_f64 v[52:53], v[20:21], v[50:51] +; GFX940-NEXT: v_accvgpr_read_b32 v62, a15 ; Reload Reuse +; GFX940-NEXT: v_cndmask_b32_e64 v18, v54, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v19, v55, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[20:21], v[50:51] +; GFX940-NEXT: s_waitcnt vmcnt(9) +; GFX940-NEXT: v_max_f64 v[50:51], v[22:23], v[34:35] +; GFX940-NEXT: v_accvgpr_read_b32 v61, a14 ; Reload Reuse +; GFX940-NEXT: v_cndmask_b32_e64 v20, v52, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v21, v53, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[22:23], v[34:35] +; GFX940-NEXT: s_waitcnt vmcnt(6) +; GFX940-NEXT: v_max_f64 v[34:35], v[24:25], v[32:33] +; GFX940-NEXT: v_accvgpr_read_b32 v60, a13 ; Reload Reuse +; GFX940-NEXT: v_cndmask_b32_e64 v22, v50, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v23, v51, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[24:25], v[32:33] +; GFX940-NEXT: v_accvgpr_read_b32 v59, a12 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v58, a11 ; Reload Reuse +; GFX940-NEXT: v_cndmask_b32_e64 v24, v34, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v25, v35, v0, vcc +; GFX940-NEXT: v_accvgpr_read_b32 v57, a10 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v56, a9 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v47, a8 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v46, a7 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v45, a6 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v44, a5 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v43, a4 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v42, a3 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v41, a2 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v40, a1 ; Reload Reuse +; GFX940-NEXT: s_waitcnt vmcnt(4) +; GFX940-NEXT: v_max_f64 v[32:33], v[26:27], v[36:37] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[26:27], v[36:37] ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v60, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v61, v3, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[4:5], v[44:45] -; GFX940-NEXT: v_accvgpr_read_b32 v61, a13 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v60, a12 ; Reload Reuse -; GFX940-NEXT: v_cndmask_b32_e32 v50, v45, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v59, v56, v50, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v50, v44, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v58, 0, v50, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[44:45], 64 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v58, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v59, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, v4, v44, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v5, v45, s[0:1] -; GFX940-NEXT: scratch_load_dword v45, off, s32 offset:72 -; GFX940-NEXT: scratch_load_dword v44, off, s32 offset:68 -; GFX940-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[58:59] -; GFX940-NEXT: s_waitcnt vmcnt(22) -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[6:7], v[46:47] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v58, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v59, v5, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[6:7], v[46:47] +; GFX940-NEXT: v_cndmask_b32_e64 v26, v32, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v27, v33, v0, vcc +; GFX940-NEXT: s_waitcnt vmcnt(2) +; GFX940-NEXT: v_max_f64 v[32:33], v[28:29], v[38:39] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[28:29], v[38:39] ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v50, v47, v7, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v59, v56, v50, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v50, v46, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v58, 0, v50, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[46:47], 64 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v58, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v7, v59, v7, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v6, v6, v46, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v7, v47, s[0:1] -; GFX940-NEXT: scratch_load_dword v47, off, s32 offset:80 -; GFX940-NEXT: scratch_load_dword v46, off, s32 offset:76 -; GFX940-NEXT: scratch_load_dword v50, off, s32 offset:84 -; GFX940-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[58:59] -; GFX940-NEXT: s_waitcnt vmcnt(18) -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[8:9], v[42:43] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v58, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v7, v59, v7, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[8:9], v[42:43] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v57, v43, v9, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v59, v56, v57, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v57, v42, v8, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v58, 0, v57, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[8:9], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[42:43], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[58:59] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v58, v8, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v9, v59, v9, vcc -; GFX940-NEXT: s_waitcnt vmcnt(8) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[10:11], v[54:55] -; GFX940-NEXT: v_cndmask_b32_e64 v8, v8, v42, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v9, v9, v43, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v42, v55, v11, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[10:11], v[54:55] -; GFX940-NEXT: v_cndmask_b32_e64 v8, v58, v8, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v9, v59, v9, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v43, v56, v42, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v42, v54, v10, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v42, 0, v42, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[54:55], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[42:43] -; GFX940-NEXT: v_cndmask_b32_e32 v10, v42, v10, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v11, v43, v11, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[12:13], v[52:53] -; GFX940-NEXT: v_cndmask_b32_e64 v10, v10, v54, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v11, v11, v55, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v54, v53, v13, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[12:13], v[52:53] -; GFX940-NEXT: v_cndmask_b32_e64 v10, v42, v10, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v11, v43, v11, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v55, v56, v54, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v54, v52, v12, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v54, 0, v54, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[12:13], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[52:53], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[54:55] -; GFX940-NEXT: v_cndmask_b32_e32 v12, v54, v12, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v13, v55, v13, vcc -; GFX940-NEXT: s_waitcnt vmcnt(6) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[14:15], v[40:41] -; GFX940-NEXT: v_cndmask_b32_e64 v12, v12, v52, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v13, v13, v53, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v41, v15, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[14:15], v[40:41] -; GFX940-NEXT: v_cndmask_b32_e64 v12, v54, v12, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v13, v55, v13, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v53, v56, v52, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v40, v14, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v52, 0, v52, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[40:41], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[52:53] -; GFX940-NEXT: v_cndmask_b32_e32 v14, v52, v14, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v14, v14, v40, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v15, v53, v15, vcc -; GFX940-NEXT: s_waitcnt vmcnt(3) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[16:17], v[44:45] -; GFX940-NEXT: v_cndmask_b32_e64 v14, v52, v14, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v15, v15, v41, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v45, v17, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[16:17], v[44:45] -; GFX940-NEXT: v_cndmask_b32_e64 v15, v53, v15, s[2:3] -; GFX940-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse -; GFX940-NEXT: v_cndmask_b32_e64 v53, v56, v52, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v44, v16, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v52, 0, v52, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[44:45], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[52:53] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v52, v16, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, v16, v44, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v17, v53, v17, vcc -; GFX940-NEXT: s_waitcnt vmcnt(1) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[18:19], v[46:47] -; GFX940-NEXT: v_cndmask_b32_e64 v16, v52, v16, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v17, v17, v45, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v47, v19, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[18:19], v[46:47] -; GFX940-NEXT: v_cndmask_b32_e64 v17, v53, v17, s[2:3] -; GFX940-NEXT: v_accvgpr_read_b32 v58, a10 ; Reload Reuse -; GFX940-NEXT: v_cndmask_b32_e64 v53, v56, v52, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v46, v18, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v52, 0, v52, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[18:19], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[46:47], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[52:53] -; GFX940-NEXT: v_cndmask_b32_e32 v18, v52, v18, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v18, v18, v46, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v19, v53, v19, vcc +; GFX940-NEXT: v_cndmask_b32_e64 v28, v32, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v29, v33, v0, vcc ; GFX940-NEXT: s_waitcnt vmcnt(0) -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[20:21], v[50:51] -; GFX940-NEXT: v_cndmask_b32_e64 v18, v52, v18, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v19, v19, v47, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v51, v21, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[20:21], v[50:51] -; GFX940-NEXT: v_cndmask_b32_e64 v19, v53, v19, s[2:3] -; GFX940-NEXT: v_accvgpr_read_b32 v57, a9 ; Reload Reuse -; GFX940-NEXT: v_cndmask_b32_e64 v53, v56, v52, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v50, v20, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v52, 0, v52, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[20:21], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[50:51], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[52:53] -; GFX940-NEXT: v_cndmask_b32_e32 v20, v52, v20, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v21, v53, v21, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[22:23], v[48:49] -; GFX940-NEXT: v_cndmask_b32_e64 v20, v20, v50, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v21, v21, v51, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v50, v49, v23, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[22:23], v[48:49] -; GFX940-NEXT: v_cndmask_b32_e64 v20, v52, v20, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v21, v53, v21, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v51, v56, v50, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v50, v48, v22, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v50, 0, v50, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[22:23], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[48:49], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[50:51] -; GFX940-NEXT: v_cndmask_b32_e32 v22, v50, v22, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v23, v51, v23, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[24:25], v[38:39] -; GFX940-NEXT: v_cndmask_b32_e64 v22, v22, v48, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v23, v23, v49, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v48, v39, v25, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[24:25], v[38:39] -; GFX940-NEXT: v_cndmask_b32_e64 v22, v50, v22, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v23, v51, v23, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v49, v56, v48, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v48, v38, v24, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[24:25], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[38:39], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[48:49] -; GFX940-NEXT: v_cndmask_b32_e32 v24, v48, v24, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v25, v49, v25, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[26:27], v[36:37] -; GFX940-NEXT: v_cndmask_b32_e64 v24, v24, v38, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v25, v25, v39, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v38, v37, v27, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[26:27], v[36:37] -; GFX940-NEXT: v_cndmask_b32_e64 v24, v48, v24, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v25, v49, v25, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v39, v56, v38, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v38, v36, v26, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v38, 0, v38, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[26:27], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[36:37], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[38:39] -; GFX940-NEXT: v_cndmask_b32_e32 v26, v38, v26, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v27, v39, v27, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[28:29], v[34:35] -; GFX940-NEXT: v_cndmask_b32_e64 v26, v26, v36, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v27, v27, v37, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v36, v35, v29, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[28:29], v[34:35] -; GFX940-NEXT: v_cndmask_b32_e64 v26, v38, v26, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v27, v39, v27, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v37, v56, v36, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v36, v34, v28, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v36, 0, v36, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[28:29], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[34:35], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[36:37] -; GFX940-NEXT: v_cndmask_b32_e32 v28, v36, v28, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v29, v37, v29, vcc -; GFX940-NEXT: v_cmp_gt_f64_e32 vcc, v[30:31], v[32:33] -; GFX940-NEXT: v_cndmask_b32_e64 v28, v28, v34, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v29, v29, v35, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v34, v33, v31, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[30:31], v[32:33] -; GFX940-NEXT: v_cndmask_b32_e64 v28, v36, v28, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v29, v37, v29, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v35, v56, v34, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v34, v32, v30, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[30:31], 64 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[32:33], 64 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[34:35] -; GFX940-NEXT: v_cndmask_b32_e32 v30, v34, v30, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v31, v35, v31, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v30, v30, v32, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v31, v31, v33, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v30, v34, v30, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v31, v35, v31, s[2:3] -; GFX940-NEXT: v_accvgpr_read_b32 v56, a8 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v47, a7 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v46, a6 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v45, a5 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v44, a4 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v43, a3 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse +; GFX940-NEXT: v_max_f64 v[32:33], v[30:31], v[48:49] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[30:31], v[48:49] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v30, v32, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v31, v33, v0, vcc +; GFX940-NEXT: v_accvgpr_read_b32 v0, a0 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maximum_v16f64: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: s_clause 0x20 +; GFX10-NEXT: s_clause 0x19 +; GFX10-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:12 +; GFX10-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:24 +; GFX10-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:20 +; GFX10-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:32 +; GFX10-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:28 +; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36 +; GFX10-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:68 +; GFX10-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:64 +; GFX10-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:60 +; GFX10-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:56 +; GFX10-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:52 +; GFX10-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:48 +; GFX10-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:44 +; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:40 ; GFX10-NEXT: buffer_load_dword v65, off, s[0:3], s32 offset:8 ; GFX10-NEXT: buffer_load_dword v64, off, s[0:3], s32 offset:4 -; GFX10-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:16 -; GFX10-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:12 -; GFX10-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:24 -; GFX10-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:20 -; GFX10-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:32 -; GFX10-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:28 -; GFX10-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:36 -; GFX10-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:40 -; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:56 -; GFX10-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:52 -; GFX10-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:48 -; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:44 -; GFX10-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:64 -; GFX10-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:60 -; GFX10-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 -; GFX10-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:72 -; GFX10-NEXT: buffer_load_dword v83, off, s[0:3], s32 offset:80 -; GFX10-NEXT: buffer_load_dword v82, off, s[0:3], s32 offset:76 -; GFX10-NEXT: buffer_load_dword v67, off, s[0:3], s32 offset:88 -; GFX10-NEXT: buffer_load_dword v66, off, s[0:3], s32 offset:84 +; GFX10-NEXT: buffer_load_dword v66, off, s[0:3], s32 offset:100 ; GFX10-NEXT: buffer_load_dword v69, off, s[0:3], s32 offset:96 ; GFX10-NEXT: buffer_load_dword v68, off, s[0:3], s32 offset:92 -; GFX10-NEXT: buffer_load_dword v70, off, s[0:3], s32 offset:100 -; GFX10-NEXT: buffer_load_dword v71, off, s[0:3], s32 offset:104 -; GFX10-NEXT: buffer_load_dword v81, off, s[0:3], s32 offset:112 -; GFX10-NEXT: buffer_load_dword v80, off, s[0:3], s32 offset:108 -; GFX10-NEXT: buffer_load_dword v85, off, s[0:3], s32 offset:120 -; GFX10-NEXT: buffer_load_dword v84, off, s[0:3], s32 offset:116 +; GFX10-NEXT: buffer_load_dword v71, off, s[0:3], s32 offset:88 +; GFX10-NEXT: buffer_load_dword v70, off, s[0:3], s32 offset:84 +; GFX10-NEXT: buffer_load_dword v81, off, s[0:3], s32 offset:80 +; GFX10-NEXT: buffer_load_dword v80, off, s[0:3], s32 offset:76 +; GFX10-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:72 +; GFX10-NEXT: buffer_load_dword v67, off, s[0:3], s32 offset:104 +; GFX10-NEXT: s_waitcnt vmcnt(24) +; GFX10-NEXT: v_max_f64 v[82:83], v[2:3], v[31:32] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[2:3], v[31:32] +; GFX10-NEXT: s_waitcnt vmcnt(22) +; GFX10-NEXT: v_max_f64 v[84:85], v[4:5], v[33:34] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[4:5], v[33:34] +; GFX10-NEXT: s_clause 0x3 +; GFX10-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 +; GFX10-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 +; GFX10-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:112 +; GFX10-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:108 +; GFX10-NEXT: s_waitcnt vmcnt(24) +; GFX10-NEXT: v_max_f64 v[32:33], v[6:7], v[35:36] +; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[6:7], v[35:36] +; GFX10-NEXT: s_clause 0x2 ; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX10-NEXT: buffer_load_dword v87, off, s[0:3], s32 offset:128 -; GFX10-NEXT: buffer_load_dword v86, off, s[0:3], s32 offset:124 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[0:1], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s12, v[2:3], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s17, v[10:11], 64 -; GFX10-NEXT: s_waitcnt vmcnt(31) -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[0:1], v[64:65] -; GFX10-NEXT: v_cmp_o_f64_e64 s4, v[0:1], v[64:65] -; GFX10-NEXT: s_waitcnt vmcnt(29) -; GFX10-NEXT: v_cmp_gt_f64_e64 s5, v[2:3], v[54:55] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[2:3], v[54:55] -; GFX10-NEXT: s_waitcnt vmcnt(27) -; GFX10-NEXT: v_cmp_gt_f64_e64 s7, v[4:5], v[52:53] -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[4:5], v[52:53] -; GFX10-NEXT: s_waitcnt vmcnt(25) -; GFX10-NEXT: v_cmp_gt_f64_e64 s9, v[6:7], v[50:51] -; GFX10-NEXT: v_cmp_o_f64_e64 s11, v[6:7], v[50:51] +; GFX10-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:128 +; GFX10-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:124 ; GFX10-NEXT: s_waitcnt vmcnt(23) -; GFX10-NEXT: v_cmp_gt_f64_e64 s13, v[8:9], v[48:49] -; GFX10-NEXT: v_cmp_class_f64_e64 s14, v[64:65], 64 +; GFX10-NEXT: v_cmp_u_f64_e64 s10, v[14:15], v[50:51] ; GFX10-NEXT: s_waitcnt vmcnt(21) -; GFX10-NEXT: v_cmp_gt_f64_e64 s15, v[12:13], v[36:37] -; GFX10-NEXT: s_waitcnt vmcnt(17) -; GFX10-NEXT: v_cmp_o_f64_e64 s16, v[14:15], v[34:35] -; GFX10-NEXT: v_cndmask_b32_e32 v96, v64, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v97, v54, v2, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v99, v55, v3, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v100, v52, v4, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v96, 0, v96, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v101, v50, v6, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v98, 0, v97, s6 -; GFX10-NEXT: v_cndmask_b32_e32 v97, v65, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[54:55], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v96, v0, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v99, 0x7ff80000, v99, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v98, v2, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v97, 0x7ff80000, v97, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v100, 0, v100, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v102, 0, v101, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v99, v3, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 s12, v[6:7], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v97, v1, s10 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[4:5], 64 -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[10:11], v[38:39] -; GFX10-NEXT: v_cndmask_b32_e64 v112, v48, v8, s13 -; GFX10-NEXT: v_cmp_o_f64_e64 s5, v[12:13], v[36:37] -; GFX10-NEXT: v_cmp_gt_f64_e64 s6, v[14:15], v[34:35] -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v64, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v65, s14 -; GFX10-NEXT: v_cmp_class_f64_e64 s14, v[52:53], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v113, v36, v12, s15 +; GFX10-NEXT: v_cmp_u_f64_e64 s9, v[12:13], v[52:53] +; GFX10-NEXT: s_waitcnt vmcnt(19) +; GFX10-NEXT: v_cmp_u_f64_e64 s7, v[10:11], v[54:55] +; GFX10-NEXT: s_waitcnt vmcnt(18) +; GFX10-NEXT: v_max_f64 v[34:35], v[8:9], v[37:38] +; GFX10-NEXT: v_cmp_u_f64_e64 s6, v[8:9], v[37:38] +; GFX10-NEXT: s_waitcnt vmcnt(16) +; GFX10-NEXT: v_max_f64 v[8:9], v[0:1], v[64:65] +; GFX10-NEXT: v_max_f64 v[36:37], v[10:11], v[54:55] +; GFX10-NEXT: v_cmp_u_f64_e64 s8, v[0:1], v[64:65] +; GFX10-NEXT: v_max_f64 v[38:39], v[12:13], v[52:53] +; GFX10-NEXT: v_max_f64 v[52:53], v[14:15], v[50:51] +; GFX10-NEXT: s_waitcnt vmcnt(11) +; GFX10-NEXT: v_max_f64 v[54:55], v[20:21], v[70:71] +; GFX10-NEXT: v_cmp_u_f64_e64 s13, v[20:21], v[70:71] +; GFX10-NEXT: s_waitcnt vmcnt(9) +; GFX10-NEXT: v_cmp_u_f64_e64 s12, v[18:19], v[80:81] +; GFX10-NEXT: s_waitcnt vmcnt(8) +; GFX10-NEXT: v_max_f64 v[50:51], v[16:17], v[48:49] +; GFX10-NEXT: v_cmp_u_f64_e64 s11, v[16:17], v[48:49] +; GFX10-NEXT: v_max_f64 v[48:49], v[18:19], v[80:81] +; GFX10-NEXT: v_max_f64 v[64:65], v[22:23], v[68:69] +; GFX10-NEXT: v_cmp_u_f64_e64 s14, v[22:23], v[68:69] +; GFX10-NEXT: s_waitcnt vmcnt(7) +; GFX10-NEXT: v_max_f64 v[68:69], v[24:25], v[66:67] +; GFX10-NEXT: v_cmp_u_f64_e64 s15, v[24:25], v[66:67] +; GFX10-NEXT: v_cndmask_b32_e64 v10, v36, 0, s7 +; GFX10-NEXT: v_cndmask_b32_e64 v0, v8, 0, s8 +; GFX10-NEXT: v_cndmask_b32_e64 v1, v9, 0x7ff80000, s8 +; GFX10-NEXT: v_cndmask_b32_e64 v8, v34, 0, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v9, v35, 0x7ff80000, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v11, v37, 0x7ff80000, s7 +; GFX10-NEXT: v_cndmask_b32_e64 v12, v38, 0, s9 +; GFX10-NEXT: v_cndmask_b32_e64 v13, v39, 0x7ff80000, s9 +; GFX10-NEXT: v_cndmask_b32_e64 v14, v52, 0, s10 +; GFX10-NEXT: v_cndmask_b32_e64 v15, v53, 0x7ff80000, s10 +; GFX10-NEXT: v_cndmask_b32_e64 v16, v50, 0, s11 +; GFX10-NEXT: v_cndmask_b32_e64 v17, v51, 0x7ff80000, s11 +; GFX10-NEXT: v_cndmask_b32_e64 v18, v48, 0, s12 +; GFX10-NEXT: v_cndmask_b32_e64 v19, v49, 0x7ff80000, s12 +; GFX10-NEXT: v_cndmask_b32_e64 v20, v54, 0, s13 +; GFX10-NEXT: v_cndmask_b32_e64 v21, v55, 0x7ff80000, s13 +; GFX10-NEXT: v_cndmask_b32_e64 v22, v64, 0, s14 +; GFX10-NEXT: v_cndmask_b32_e64 v23, v65, 0x7ff80000, s14 +; GFX10-NEXT: v_cndmask_b32_e64 v24, v68, 0, s15 +; GFX10-NEXT: v_cndmask_b32_e64 v25, v69, 0x7ff80000, s15 +; GFX10-NEXT: s_waitcnt vmcnt(5) +; GFX10-NEXT: v_max_f64 v[70:71], v[28:29], v[2:3] +; GFX10-NEXT: v_cmp_u_f64_e64 s17, v[28:29], v[2:3] +; GFX10-NEXT: s_waitcnt vmcnt(3) +; GFX10-NEXT: v_max_f64 v[66:67], v[26:27], v[4:5] +; GFX10-NEXT: v_cmp_u_f64_e64 s16, v[26:27], v[4:5] +; GFX10-NEXT: v_cndmask_b32_e64 v2, v82, 0, vcc_lo ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_cmp_o_f64_e64 s18, v[30:31], v[86:87] -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v54, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v54, v53, v5, s7 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v55, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[50:51], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v55, v51, v7, s9 -; GFX10-NEXT: v_cmp_o_f64_e64 s9, v[8:9], v[48:49] -; GFX10-NEXT: v_cndmask_b32_e64 v101, 0x7ff80000, v54, s8 -; GFX10-NEXT: v_cmp_gt_f64_e64 s7, v[16:17], v[32:33] -; GFX10-NEXT: v_cndmask_b32_e64 v6, v102, v6, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v103, 0x7ff80000, v55, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v100, v4, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v101, v5, s10 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[8:9], 64 -; GFX10-NEXT: v_cmp_o_f64_e64 s11, v[10:11], v[38:39] -; GFX10-NEXT: v_cndmask_b32_e64 v7, v103, v7, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 s12, v[48:49], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v114, v38, v10, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v115, v34, v14, s6 -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[16:17], v[32:33] -; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v52, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v53, s14 -; GFX10-NEXT: v_cmp_gt_f64_e64 s14, v[18:19], v[82:83] -; GFX10-NEXT: v_cndmask_b32_e64 v52, 0, v115, s16 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v50, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v50, v49, v9, s13 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v51, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[38:39], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v54, 0, v112, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v51, v39, v11, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v55, 0x7ff80000, v50, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v50, 0, v113, s5 -; GFX10-NEXT: v_cmp_o_f64_e64 s4, v[18:19], v[82:83] -; GFX10-NEXT: v_cndmask_b32_e64 v8, v54, v8, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v64, 0, v114, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v9, v55, v9, s10 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[12:13], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v65, 0x7ff80000, v51, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, v48, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v10, v64, v10, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v49, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 s12, v[14:15], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v65, v11, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v48, v37, v13, s15 -; GFX10-NEXT: v_cmp_class_f64_e64 s17, v[34:35], 64 -; GFX10-NEXT: v_cmp_gt_f64_e64 s9, v[20:21], v[66:67] -; GFX10-NEXT: v_cmp_o_f64_e64 s11, v[20:21], v[66:67] -; GFX10-NEXT: v_cndmask_b32_e64 v116, v32, v16, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v51, 0x7ff80000, v48, s5 -; GFX10-NEXT: v_cndmask_b32_e32 v10, v10, v38, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v11, v11, v39, vcc_lo -; GFX10-NEXT: v_cmp_gt_f64_e32 vcc_lo, v[22:23], v[68:69] -; GFX10-NEXT: v_cndmask_b32_e64 v38, v35, v15, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v49, v82, v18, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v48, 0, v116, s8 -; GFX10-NEXT: v_cmp_class_f64_e64 s13, v[36:37], 64 -; GFX10-NEXT: v_cmp_o_f64_e64 s5, v[22:23], v[68:69] -; GFX10-NEXT: v_cndmask_b32_e64 v53, 0x7ff80000, v38, s16 -; GFX10-NEXT: v_cndmask_b32_e64 v12, v50, v12, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v51, v13, s10 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[16:17], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v38, 0, v49, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v112, v83, v19, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v14, v52, v14, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v15, v53, v15, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 s12, v[32:33], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s14, v[18:19], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v114, v67, v21, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v14, v14, v34, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v15, v15, v35, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v34, v33, v17, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v39, 0x7ff80000, v112, s4 -; GFX10-NEXT: v_cmp_gt_f64_e64 s4, v[24:25], v[70:71] -; GFX10-NEXT: v_cndmask_b32_e32 v113, v69, v23, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v35, v68, v22, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[20:21], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v49, 0x7ff80000, v34, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v34, 0x7ff80000, v114, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v12, v12, v36, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v13, v37, s13 -; GFX10-NEXT: v_cmp_class_f64_e64 s13, v[82:83], 64 -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[24:25], v[70:71] -; GFX10-NEXT: v_cndmask_b32_e64 v16, v48, v16, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v17, v49, v17, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v36, 0x7ff80000, v113, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v35, 0, v35, s5 -; GFX10-NEXT: v_cmp_gt_f64_e64 s7, v[26:27], v[80:81] -; GFX10-NEXT: v_cndmask_b32_e64 v16, v16, v32, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v32, v66, v20, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v17, v17, v33, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v18, v38, v18, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v19, v39, v19, s14 -; GFX10-NEXT: v_cmp_o_f64_e64 s15, v[26:27], v[80:81] -; GFX10-NEXT: v_cndmask_b32_e64 v33, 0, v32, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v112, v71, v25, s4 -; GFX10-NEXT: v_cmp_gt_f64_e64 s16, v[28:29], v[84:85] -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[28:29], v[84:85] -; GFX10-NEXT: v_cndmask_b32_e32 v21, v34, v21, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v20, v33, v20, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[22:23], 64 -; GFX10-NEXT: v_cmp_gt_f64_e64 s17, v[30:31], v[86:87] -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[70:71], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v18, v18, v82, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v82, v70, v24, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v19, v19, v83, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v83, 0x7ff80000, v112, s6 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[68:69], 64 -; GFX10-NEXT: v_cmp_eq_f64_e64 s9, 0, v[96:97] -; GFX10-NEXT: v_cndmask_b32_e64 v82, 0, v82, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v37, v81, v27, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v32, v80, v26, s7 -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[80:81], 64 -; GFX10-NEXT: v_cmp_class_f64_e64 s7, v[84:85], 64 -; GFX10-NEXT: v_cmp_eq_f64_e64 s10, 0, v[98:99] -; GFX10-NEXT: v_cndmask_b32_e64 v113, 0x7ff80000, v37, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v112, 0, v32, s15 -; GFX10-NEXT: v_cmp_eq_f64_e64 s11, 0, v[100:101] -; GFX10-NEXT: v_cndmask_b32_e64 v115, v85, v29, s16 -; GFX10-NEXT: v_cndmask_b32_e64 v114, v84, v28, s16 -; GFX10-NEXT: v_cmp_eq_f64_e64 s12, 0, v[102:103] -; GFX10-NEXT: v_cmp_eq_f64_e64 s13, 0, v[54:55] -; GFX10-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[24:25], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v115, 0x7ff80000, v115, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v114, 0, v114, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v116, v87, v31, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v32, v86, v30, s17 -; GFX10-NEXT: v_cmp_class_f64_e64 s8, v[86:87], 64 -; GFX10-NEXT: v_cmp_eq_f64_e64 s14, 0, v[64:65] -; GFX10-NEXT: v_cmp_eq_f64_e64 s15, 0, v[50:51] -; GFX10-NEXT: v_cndmask_b32_e64 v117, 0x7ff80000, v116, s18 -; GFX10-NEXT: v_cndmask_b32_e64 v116, 0, v32, s18 -; GFX10-NEXT: v_cmp_eq_f64_e64 s16, 0, v[52:53] -; GFX10-NEXT: v_cmp_eq_f64_e64 s17, 0, v[48:49] -; GFX10-NEXT: v_cmp_eq_f64_e64 s18, 0, v[38:39] -; GFX10-NEXT: v_cmp_eq_f64_e64 s19, 0, v[33:34] -; GFX10-NEXT: v_cmp_eq_f64_e64 s20, 0, v[35:36] -; GFX10-NEXT: v_cmp_eq_f64_e64 s21, 0, v[82:83] -; GFX10-NEXT: v_cmp_eq_f64_e64 s22, 0, v[112:113] -; GFX10-NEXT: v_cmp_eq_f64_e64 s23, 0, v[114:115] -; GFX10-NEXT: v_cmp_eq_f64_e64 s24, 0, v[116:117] -; GFX10-NEXT: v_cndmask_b32_e64 v22, v22, v68, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v23, v23, v69, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v96, v0, s9 -; GFX10-NEXT: v_cndmask_b32_e32 v24, v82, v24, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v25, v83, v25, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[26:27], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v98, v2, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v100, v4, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v24, v24, v70, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v25, v25, v71, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v102, v6, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v8, v54, v8, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v10, v64, v10, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v12, v50, v12, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v14, v52, v14, s16 -; GFX10-NEXT: v_cndmask_b32_e64 v16, v48, v16, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v18, v38, v18, s18 -; GFX10-NEXT: v_cndmask_b32_e64 v22, v35, v22, s20 -; GFX10-NEXT: v_cndmask_b32_e64 v24, v82, v24, s21 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v97, v1, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v99, v3, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v101, v5, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v103, v7, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v9, v55, v9, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v65, v11, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v51, v13, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v15, v53, v15, s16 -; GFX10-NEXT: v_cndmask_b32_e32 v26, v112, v26, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v27, v113, v27, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[28:29], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v17, v49, v17, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v19, v39, v19, s18 -; GFX10-NEXT: v_cndmask_b32_e64 v26, v26, v80, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v27, v27, v81, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v23, v36, v23, s20 -; GFX10-NEXT: v_cndmask_b32_e64 v25, v83, v25, s21 -; GFX10-NEXT: v_cndmask_b32_e64 v26, v112, v26, s22 -; GFX10-NEXT: v_cndmask_b32_e64 v27, v113, v27, s22 -; GFX10-NEXT: v_cndmask_b32_e32 v28, v114, v28, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v29, v115, v29, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[30:31], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v28, v28, v84, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v29, v29, v85, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v28, v114, v28, s23 -; GFX10-NEXT: v_cndmask_b32_e64 v29, v115, v29, s23 -; GFX10-NEXT: v_cndmask_b32_e32 v30, v116, v30, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v31, v117, v31, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[66:67], 64 -; GFX10-NEXT: v_cndmask_b32_e64 v30, v30, v86, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v31, v31, v87, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v30, v116, v30, s24 -; GFX10-NEXT: v_cndmask_b32_e64 v31, v117, v31, s24 -; GFX10-NEXT: v_cndmask_b32_e32 v20, v20, v66, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v21, v21, v67, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v20, v33, v20, s19 -; GFX10-NEXT: v_cndmask_b32_e64 v21, v34, v21, s19 +; GFX10-NEXT: v_max_f64 v[80:81], v[30:31], v[6:7] +; GFX10-NEXT: v_cmp_u_f64_e64 s18, v[30:31], v[6:7] +; GFX10-NEXT: v_cndmask_b32_e64 v3, v83, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v4, v84, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v85, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v6, v32, 0, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v7, v33, 0x7ff80000, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v28, v70, 0, s17 +; GFX10-NEXT: v_cndmask_b32_e64 v29, v71, 0x7ff80000, s17 +; GFX10-NEXT: v_cndmask_b32_e64 v26, v66, 0, s16 +; GFX10-NEXT: v_cndmask_b32_e64 v27, v67, 0x7ff80000, s16 +; GFX10-NEXT: v_cndmask_b32_e64 v30, v80, 0, s18 +; GFX10-NEXT: v_cndmask_b32_e64 v31, v81, 0x7ff80000, s18 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maximum_v16f64: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_clause 0x1f -; GFX11-NEXT: scratch_load_b32 v87, off, s32 offset:8 -; GFX11-NEXT: scratch_load_b32 v86, off, s32 offset:4 -; GFX11-NEXT: scratch_load_b32 v85, off, s32 offset:16 -; GFX11-NEXT: scratch_load_b32 v84, off, s32 offset:12 -; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:24 -; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:20 -; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:32 -; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:28 -; GFX11-NEXT: scratch_load_b32 v37, off, s32 offset:40 -; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:36 -; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:48 -; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:44 -; GFX11-NEXT: scratch_load_b32 v49, off, s32 offset:56 -; GFX11-NEXT: scratch_load_b32 v48, off, s32 offset:52 -; GFX11-NEXT: scratch_load_b32 v51, off, s32 offset:64 -; GFX11-NEXT: scratch_load_b32 v50, off, s32 offset:60 -; GFX11-NEXT: scratch_load_b32 v53, off, s32 offset:72 -; GFX11-NEXT: scratch_load_b32 v52, off, s32 offset:68 -; GFX11-NEXT: scratch_load_b32 v55, off, s32 offset:80 -; GFX11-NEXT: scratch_load_b32 v54, off, s32 offset:76 -; GFX11-NEXT: scratch_load_b32 v65, off, s32 offset:88 -; GFX11-NEXT: scratch_load_b32 v64, off, s32 offset:84 -; GFX11-NEXT: scratch_load_b32 v67, off, s32 offset:96 -; GFX11-NEXT: scratch_load_b32 v66, off, s32 offset:92 -; GFX11-NEXT: scratch_load_b32 v69, off, s32 offset:104 -; GFX11-NEXT: scratch_load_b32 v68, off, s32 offset:100 -; GFX11-NEXT: scratch_load_b32 v71, off, s32 offset:112 -; GFX11-NEXT: scratch_load_b32 v70, off, s32 offset:108 -; GFX11-NEXT: scratch_load_b32 v81, off, s32 offset:120 -; GFX11-NEXT: scratch_load_b32 v80, off, s32 offset:116 ; GFX11-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-NEXT: scratch_load_b32 v83, off, s32 offset:128 -; GFX11-NEXT: scratch_load_b32 v82, off, s32 offset:124 -; GFX11-NEXT: s_waitcnt vmcnt(31) -; GFX11-NEXT: v_cmp_gt_f64_e64 s9, v[0:1], v[86:87] -; GFX11-NEXT: v_cmp_o_f64_e64 s11, v[0:1], v[86:87] -; GFX11-NEXT: s_waitcnt vmcnt(29) -; GFX11-NEXT: v_cmp_gt_f64_e64 s10, v[2:3], v[84:85] -; GFX11-NEXT: v_cmp_class_f64_e64 s14, v[86:87], 64 -; GFX11-NEXT: s_waitcnt vmcnt(27) -; GFX11-NEXT: v_cmp_gt_f64_e64 s0, v[4:5], v[32:33] -; GFX11-NEXT: v_cmp_o_f64_e32 vcc_lo, v[4:5], v[32:33] -; GFX11-NEXT: s_waitcnt vmcnt(25) -; GFX11-NEXT: v_cmp_gt_f64_e64 s2, v[6:7], v[34:35] -; GFX11-NEXT: v_cmp_o_f64_e64 s12, v[2:3], v[84:85] -; GFX11-NEXT: v_cmp_o_f64_e64 s1, v[6:7], v[34:35] -; GFX11-NEXT: s_waitcnt vmcnt(23) -; GFX11-NEXT: v_cmp_gt_f64_e64 s4, v[8:9], v[36:37] -; GFX11-NEXT: v_cmp_o_f64_e64 s3, v[8:9], v[36:37] -; GFX11-NEXT: v_cmp_class_f64_e64 s16, v[84:85], 64 -; GFX11-NEXT: s_waitcnt vmcnt(21) -; GFX11-NEXT: v_cmp_gt_f64_e64 s6, v[10:11], v[38:39] -; GFX11-NEXT: v_cmp_o_f64_e64 s5, v[10:11], v[38:39] -; GFX11-NEXT: s_waitcnt vmcnt(19) -; GFX11-NEXT: v_cmp_gt_f64_e64 s8, v[12:13], v[48:49] -; GFX11-NEXT: v_cmp_o_f64_e64 s7, v[12:13], v[48:49] -; GFX11-NEXT: s_waitcnt vmcnt(17) -; GFX11-NEXT: v_cmp_gt_f64_e64 s13, v[14:15], v[50:51] -; GFX11-NEXT: s_waitcnt vmcnt(15) -; GFX11-NEXT: v_cmp_o_f64_e64 s15, v[16:17], v[52:53] -; GFX11-NEXT: s_waitcnt vmcnt(13) -; GFX11-NEXT: v_cmp_gt_f64_e64 s17, v[18:19], v[54:55] -; GFX11-NEXT: v_cmp_o_f64_e64 s18, v[18:19], v[54:55] -; GFX11-NEXT: s_waitcnt vmcnt(11) -; GFX11-NEXT: v_cmp_gt_f64_e64 s19, v[20:21], v[64:65] -; GFX11-NEXT: v_cmp_o_f64_e64 s20, v[20:21], v[64:65] -; GFX11-NEXT: s_waitcnt vmcnt(9) -; GFX11-NEXT: v_cmp_gt_f64_e64 s21, v[22:23], v[66:67] -; GFX11-NEXT: v_cmp_o_f64_e64 s22, v[22:23], v[66:67] -; GFX11-NEXT: s_waitcnt vmcnt(7) -; GFX11-NEXT: v_cmp_gt_f64_e64 s23, v[24:25], v[68:69] -; GFX11-NEXT: v_cmp_o_f64_e64 s24, v[24:25], v[68:69] -; GFX11-NEXT: s_waitcnt vmcnt(5) -; GFX11-NEXT: v_cmp_gt_f64_e64 s25, v[26:27], v[70:71] -; GFX11-NEXT: v_cmp_o_f64_e64 s26, v[26:27], v[70:71] -; GFX11-NEXT: s_waitcnt vmcnt(3) -; GFX11-NEXT: v_cmp_gt_f64_e64 s27, v[28:29], v[80:81] -; GFX11-NEXT: v_cmp_o_f64_e64 s28, v[28:29], v[80:81] +; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:8 +; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:4 +; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:16 +; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:12 +; GFX11-NEXT: scratch_load_b32 v37, off, s32 offset:24 +; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:20 +; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:32 +; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:28 +; GFX11-NEXT: scratch_load_b32 v49, off, s32 offset:40 +; GFX11-NEXT: scratch_load_b32 v48, off, s32 offset:36 +; GFX11-NEXT: scratch_load_b32 v51, off, s32 offset:48 +; GFX11-NEXT: scratch_load_b32 v50, off, s32 offset:44 +; GFX11-NEXT: scratch_load_b32 v53, off, s32 offset:56 +; GFX11-NEXT: scratch_load_b32 v52, off, s32 offset:52 +; GFX11-NEXT: scratch_load_b32 v55, off, s32 offset:64 +; GFX11-NEXT: scratch_load_b32 v54, off, s32 offset:60 +; GFX11-NEXT: scratch_load_b32 v65, off, s32 offset:72 +; GFX11-NEXT: scratch_load_b32 v64, off, s32 offset:68 +; GFX11-NEXT: scratch_load_b32 v67, off, s32 offset:80 +; GFX11-NEXT: scratch_load_b32 v66, off, s32 offset:76 +; GFX11-NEXT: scratch_load_b32 v69, off, s32 offset:88 +; GFX11-NEXT: scratch_load_b32 v68, off, s32 offset:84 +; GFX11-NEXT: scratch_load_b32 v71, off, s32 offset:96 +; GFX11-NEXT: scratch_load_b32 v70, off, s32 offset:92 +; GFX11-NEXT: scratch_load_b32 v81, off, s32 offset:104 +; GFX11-NEXT: scratch_load_b32 v80, off, s32 offset:100 +; GFX11-NEXT: scratch_load_b32 v83, off, s32 offset:112 +; GFX11-NEXT: scratch_load_b32 v82, off, s32 offset:108 +; GFX11-NEXT: scratch_load_b32 v85, off, s32 offset:120 +; GFX11-NEXT: scratch_load_b32 v84, off, s32 offset:116 +; GFX11-NEXT: scratch_load_b32 v87, off, s32 offset:128 +; GFX11-NEXT: scratch_load_b32 v86, off, s32 offset:124 +; GFX11-NEXT: s_waitcnt vmcnt(30) +; GFX11-NEXT: v_max_f64 v[96:97], v[0:1], v[32:33] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[32:33] +; GFX11-NEXT: s_waitcnt vmcnt(28) +; GFX11-NEXT: v_max_f64 v[32:33], v[2:3], v[34:35] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[34:35] +; GFX11-NEXT: s_waitcnt vmcnt(26) +; GFX11-NEXT: v_max_f64 v[34:35], v[4:5], v[36:37] +; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[36:37] +; GFX11-NEXT: s_waitcnt vmcnt(24) +; GFX11-NEXT: v_max_f64 v[36:37], v[6:7], v[38:39] +; GFX11-NEXT: v_cmp_u_f64_e64 s2, v[6:7], v[38:39] +; GFX11-NEXT: s_waitcnt vmcnt(22) +; GFX11-NEXT: v_max_f64 v[38:39], v[8:9], v[48:49] +; GFX11-NEXT: v_cmp_u_f64_e64 s3, v[8:9], v[48:49] +; GFX11-NEXT: s_waitcnt vmcnt(20) +; GFX11-NEXT: v_max_f64 v[48:49], v[10:11], v[50:51] +; GFX11-NEXT: v_cmp_u_f64_e64 s4, v[10:11], v[50:51] +; GFX11-NEXT: s_waitcnt vmcnt(18) +; GFX11-NEXT: v_max_f64 v[50:51], v[12:13], v[52:53] +; GFX11-NEXT: v_cmp_u_f64_e64 s5, v[12:13], v[52:53] +; GFX11-NEXT: s_waitcnt vmcnt(16) +; GFX11-NEXT: v_max_f64 v[52:53], v[14:15], v[54:55] +; GFX11-NEXT: v_cmp_u_f64_e64 s6, v[14:15], v[54:55] +; GFX11-NEXT: s_waitcnt vmcnt(14) +; GFX11-NEXT: v_max_f64 v[54:55], v[16:17], v[64:65] +; GFX11-NEXT: v_cmp_u_f64_e64 s7, v[16:17], v[64:65] +; GFX11-NEXT: s_waitcnt vmcnt(12) +; GFX11-NEXT: v_max_f64 v[64:65], v[18:19], v[66:67] +; GFX11-NEXT: v_cmp_u_f64_e64 s8, v[18:19], v[66:67] +; GFX11-NEXT: s_waitcnt vmcnt(10) +; GFX11-NEXT: v_max_f64 v[66:67], v[20:21], v[68:69] +; GFX11-NEXT: v_cmp_u_f64_e64 s9, v[20:21], v[68:69] +; GFX11-NEXT: s_waitcnt vmcnt(8) +; GFX11-NEXT: v_max_f64 v[68:69], v[22:23], v[70:71] +; GFX11-NEXT: v_cmp_u_f64_e64 s10, v[22:23], v[70:71] +; GFX11-NEXT: s_waitcnt vmcnt(6) +; GFX11-NEXT: v_max_f64 v[70:71], v[24:25], v[80:81] +; GFX11-NEXT: v_cmp_u_f64_e64 s11, v[24:25], v[80:81] +; GFX11-NEXT: s_waitcnt vmcnt(4) +; GFX11-NEXT: v_max_f64 v[80:81], v[26:27], v[82:83] +; GFX11-NEXT: v_cmp_u_f64_e64 s12, v[26:27], v[82:83] +; GFX11-NEXT: s_waitcnt vmcnt(2) +; GFX11-NEXT: v_max_f64 v[82:83], v[28:29], v[84:85] +; GFX11-NEXT: v_cmp_u_f64_e64 s13, v[28:29], v[84:85] ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_cmp_gt_f64_e64 s29, v[30:31], v[82:83] -; GFX11-NEXT: v_cmp_o_f64_e64 vcc_hi, v[30:31], v[82:83] -; GFX11-NEXT: v_cndmask_b32_e64 v96, v87, v1, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v101, v86, v0, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v98, v85, v3, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v103, v84, v2, s10 -; GFX11-NEXT: v_cmp_class_f64_e64 s10, v[0:1], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v97, 0x7ff80000, v96, s11 -; GFX11-NEXT: v_cndmask_b32_e64 v96, 0, v101, s11 -; GFX11-NEXT: v_cndmask_b32_e64 v100, v33, v5, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v102, v35, v7, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v99, 0x7ff80000, v98, s12 -; GFX11-NEXT: v_cndmask_b32_e64 v98, 0, v103, s12 -; GFX11-NEXT: v_cmp_class_f64_e64 s11, v[2:3], 64 -; GFX11-NEXT: v_cndmask_b32_e32 v101, 0x7ff80000, v100, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v103, 0x7ff80000, v102, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v112, v37, v9, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v114, v39, v11, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v116, v49, v13, s8 -; GFX11-NEXT: v_cmp_o_f64_e64 s9, v[14:15], v[50:51] -; GFX11-NEXT: v_cndmask_b32_e64 v118, v51, v15, s13 -; GFX11-NEXT: v_cndmask_b32_e64 v113, 0x7ff80000, v112, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v115, 0x7ff80000, v114, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v117, 0x7ff80000, v116, s7 -; GFX11-NEXT: v_cmp_gt_f64_e64 s12, v[16:17], v[52:53] -; GFX11-NEXT: v_cndmask_b32_e64 v130, v55, v19, s17 -; GFX11-NEXT: v_cndmask_b32_e64 v132, v65, v21, s19 -; GFX11-NEXT: v_cndmask_b32_e64 v134, v67, v23, s21 -; GFX11-NEXT: v_cndmask_b32_e64 v144, v69, v25, s23 -; GFX11-NEXT: v_cndmask_b32_e64 v145, v71, v27, s25 -; GFX11-NEXT: v_cndmask_b32_e64 v131, 0x7ff80000, v130, s18 -; GFX11-NEXT: v_cndmask_b32_e64 v133, 0x7ff80000, v132, s20 -; GFX11-NEXT: v_cndmask_b32_e64 v135, 0x7ff80000, v134, s22 -; GFX11-NEXT: v_cndmask_b32_e64 v146, v81, v29, s27 -; GFX11-NEXT: v_cndmask_b32_e64 v148, v80, v28, s27 -; GFX11-NEXT: v_cndmask_b32_e64 v147, v83, v31, s29 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v147, 0x7ff80000, v147, vcc_hi -; GFX11-NEXT: v_cndmask_b32_e64 v0, v96, v0, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v97, v1, s10 -; GFX11-NEXT: v_cmp_class_f64_e64 s10, v[36:37], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v86, s14 -; GFX11-NEXT: v_cndmask_b32_e64 v86, v32, v4, s0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v87, s14 -; GFX11-NEXT: v_cndmask_b32_e64 v87, v34, v6, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v98, v2, s11 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v99, v3, s11 -; GFX11-NEXT: v_cndmask_b32_e32 v100, 0, v86, vcc_lo -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[4:5], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v102, 0, v87, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v84, s16 -; GFX11-NEXT: v_cndmask_b32_e64 v84, v36, v8, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v86, v38, v10, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v87, v48, v12, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v119, 0x7ff80000, v118, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v128, v53, v17, s12 -; GFX11-NEXT: v_cndmask_b32_e64 v112, 0, v84, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v114, 0, v86, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v116, 0, v87, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v84, v50, v14, s13 -; GFX11-NEXT: v_cndmask_b32_e64 v129, 0x7ff80000, v128, s15 -; GFX11-NEXT: v_cndmask_b32_e64 v86, v52, v16, s12 -; GFX11-NEXT: v_cndmask_b32_e64 v87, v54, v18, s17 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v85, s16 -; GFX11-NEXT: v_cndmask_b32_e64 v118, 0, v84, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v84, v64, v20, s19 -; GFX11-NEXT: v_cndmask_b32_e64 v128, 0, v86, s15 -; GFX11-NEXT: v_cndmask_b32_e64 v130, 0, v87, s18 -; GFX11-NEXT: v_cndmask_b32_e64 v86, v66, v22, s21 -; GFX11-NEXT: v_cndmask_b32_e64 v85, 0x7ff80000, v144, s24 -; GFX11-NEXT: v_cndmask_b32_e64 v132, 0, v84, s20 -; GFX11-NEXT: v_cndmask_b32_e64 v87, v68, v24, s23 -; GFX11-NEXT: v_cndmask_b32_e64 v144, v70, v26, s25 -; GFX11-NEXT: v_cndmask_b32_e64 v134, 0, v86, s22 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[68:69], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[70:71], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v84, 0, v87, s24 -; GFX11-NEXT: v_cndmask_b32_e64 v87, 0x7ff80000, v145, s26 -; GFX11-NEXT: v_cndmask_b32_e64 v86, 0, v144, s26 -; GFX11-NEXT: v_cndmask_b32_e64 v145, 0x7ff80000, v146, s28 -; GFX11-NEXT: v_cndmask_b32_e64 v144, 0, v148, s28 -; GFX11-NEXT: v_cndmask_b32_e64 v146, v82, v30, s29 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[80:81], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s3, v[82:83], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s6, v[32:33], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s8, v[34:35], 64 -; GFX11-NEXT: v_dual_cndmask_b32 v5, v101, v5 :: v_dual_cndmask_b32 v4, v100, v4 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[6:7], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v146, 0, v146, vcc_hi -; GFX11-NEXT: v_cmp_class_f64_e64 s12, v[38:39], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s14, v[48:49], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s16, v[50:51], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s18, v[52:53], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s20, v[54:55], 64 -; GFX11-NEXT: v_cmp_class_f64_e64 s21, v[64:65], 64 -; GFX11-NEXT: v_cmp_eq_f64_e64 s4, 0, v[96:97] -; GFX11-NEXT: v_cmp_eq_f64_e64 s5, 0, v[98:99] -; GFX11-NEXT: v_cmp_eq_f64_e64 s7, 0, v[100:101] -; GFX11-NEXT: v_cmp_eq_f64_e64 s9, 0, v[102:103] -; GFX11-NEXT: v_cmp_eq_f64_e64 s11, 0, v[112:113] -; GFX11-NEXT: v_cmp_eq_f64_e64 s13, 0, v[114:115] -; GFX11-NEXT: v_cmp_eq_f64_e64 s15, 0, v[116:117] -; GFX11-NEXT: v_cmp_eq_f64_e64 s17, 0, v[118:119] -; GFX11-NEXT: v_cmp_eq_f64_e64 s19, 0, v[128:129] -; GFX11-NEXT: v_cmp_eq_f64_e64 s22, 0, v[130:131] -; GFX11-NEXT: v_cmp_eq_f64_e64 s23, 0, v[132:133] -; GFX11-NEXT: v_cmp_eq_f64_e64 s24, 0, v[134:135] -; GFX11-NEXT: v_cmp_eq_f64_e64 s25, 0, v[84:85] -; GFX11-NEXT: v_cmp_eq_f64_e64 s26, 0, v[86:87] -; GFX11-NEXT: v_cmp_eq_f64_e64 s27, 0, v[144:145] -; GFX11-NEXT: v_cmp_eq_f64_e64 s28, 0, v[146:147] -; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v33, s6 -; GFX11-NEXT: v_dual_cndmask_b32 v7, v103, v7 :: v_dual_cndmask_b32 v6, v102, v6 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[8:9], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v32, s6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v7, v7, v35, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v96, v0, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v98, v2, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v100, v4, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v97, v1, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v99, v3, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v101, v5, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v103, v7, s9 -; GFX11-NEXT: v_dual_cndmask_b32 v9, v113, v9 :: v_dual_cndmask_b32 v8, v112, v8 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[10:11], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v34, s8 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v37, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v102, v6, s9 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v9, v113, v9, s11 -; GFX11-NEXT: v_dual_cndmask_b32 v11, v115, v11 :: v_dual_cndmask_b32 v10, v114, v10 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[12:13], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v8, v8, v36, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v11, v11, v39, s12 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v8, v112, v8, s11 -; GFX11-NEXT: v_cndmask_b32_e64 v11, v115, v11, s13 -; GFX11-NEXT: v_dual_cndmask_b32 v13, v117, v13 :: v_dual_cndmask_b32 v12, v116, v12 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[14:15], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v38, s12 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v13, v13, v49, s14 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v114, v10, s13 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v13, v117, v13, s15 -; GFX11-NEXT: v_dual_cndmask_b32 v15, v119, v15 :: v_dual_cndmask_b32 v14, v118, v14 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[16:17], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v12, v12, v48, s14 -; GFX11-NEXT: v_cndmask_b32_e64 v15, v15, v51, s16 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v12, v116, v12, s15 -; GFX11-NEXT: v_cndmask_b32_e64 v15, v119, v15, s17 -; GFX11-NEXT: v_dual_cndmask_b32 v17, v129, v17 :: v_dual_cndmask_b32 v16, v128, v16 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[18:19], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v14, v14, v50, s16 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v17, v17, v53, s18 -; GFX11-NEXT: v_cndmask_b32_e64 v14, v118, v14, s17 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v17, v129, v17, s19 -; GFX11-NEXT: v_dual_cndmask_b32 v19, v131, v19 :: v_dual_cndmask_b32 v18, v130, v18 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[20:21], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v16, v16, v52, s18 -; GFX11-NEXT: v_cndmask_b32_e64 v19, v19, v55, s20 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v16, v128, v16, s19 -; GFX11-NEXT: v_cndmask_b32_e64 v19, v131, v19, s22 -; GFX11-NEXT: v_dual_cndmask_b32 v21, v133, v21 :: v_dual_cndmask_b32 v20, v132, v20 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[22:23], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v18, v18, v54, s20 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v21, v21, v65, s21 -; GFX11-NEXT: v_cndmask_b32_e64 v18, v130, v18, s22 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_cndmask_b32_e64 v21, v133, v21, s23 -; GFX11-NEXT: v_dual_cndmask_b32 v23, v135, v23 :: v_dual_cndmask_b32 v22, v134, v22 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[24:25], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v20, v20, v64, s21 -; GFX11-NEXT: v_cndmask_b32_e64 v20, v132, v20, s23 -; GFX11-NEXT: v_dual_cndmask_b32 v25, v85, v25 :: v_dual_cndmask_b32 v24, v84, v24 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[26:27], 64 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_cndmask_b32_e64 v25, v25, v69, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v25, v85, v25, s25 -; GFX11-NEXT: v_dual_cndmask_b32 v27, v87, v27 :: v_dual_cndmask_b32 v26, v86, v26 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[28:29], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v24, v24, v68, s0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v27, v27, v71, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v24, v84, v24, s25 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v27, v87, v27, s26 -; GFX11-NEXT: v_dual_cndmask_b32 v29, v145, v29 :: v_dual_cndmask_b32 v28, v144, v28 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[30:31], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v26, v26, v70, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v29, v29, v81, s2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v26, v86, v26, s26 -; GFX11-NEXT: v_cndmask_b32_e64 v29, v145, v29, s27 -; GFX11-NEXT: v_dual_cndmask_b32 v31, v147, v31 :: v_dual_cndmask_b32 v30, v146, v30 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[66:67], 64 -; GFX11-NEXT: v_cndmask_b32_e64 v28, v28, v80, s2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v31, v31, v83, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v28, v144, v28, s27 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v31, v147, v31, s28 -; GFX11-NEXT: v_dual_cndmask_b32 v23, v23, v67 :: v_dual_cndmask_b32 v22, v22, v66 -; GFX11-NEXT: v_cndmask_b32_e64 v30, v30, v82, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v23, v135, v23, s24 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v22, v134, v22, s24 -; GFX11-NEXT: v_cndmask_b32_e64 v30, v146, v30, s28 +; GFX11-NEXT: v_max_f64 v[84:85], v[30:31], v[86:87] +; GFX11-NEXT: v_cmp_u_f64_e64 s14, v[30:31], v[86:87] +; GFX11-NEXT: v_cndmask_b32_e64 v0, v96, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v97, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v32, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v33, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v4, v34, 0, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v5, v35, 0x7ff80000, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v6, v36, 0, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v7, v37, 0x7ff80000, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v8, v38, 0, s3 +; GFX11-NEXT: v_cndmask_b32_e64 v9, v39, 0x7ff80000, s3 +; GFX11-NEXT: v_cndmask_b32_e64 v10, v48, 0, s4 +; GFX11-NEXT: v_cndmask_b32_e64 v11, v49, 0x7ff80000, s4 +; GFX11-NEXT: v_cndmask_b32_e64 v12, v50, 0, s5 +; GFX11-NEXT: v_cndmask_b32_e64 v13, v51, 0x7ff80000, s5 +; GFX11-NEXT: v_cndmask_b32_e64 v14, v52, 0, s6 +; GFX11-NEXT: v_cndmask_b32_e64 v15, v53, 0x7ff80000, s6 +; GFX11-NEXT: v_cndmask_b32_e64 v16, v54, 0, s7 +; GFX11-NEXT: v_cndmask_b32_e64 v17, v55, 0x7ff80000, s7 +; GFX11-NEXT: v_cndmask_b32_e64 v18, v64, 0, s8 +; GFX11-NEXT: v_cndmask_b32_e64 v19, v65, 0x7ff80000, s8 +; GFX11-NEXT: v_cndmask_b32_e64 v20, v66, 0, s9 +; GFX11-NEXT: v_cndmask_b32_e64 v21, v67, 0x7ff80000, s9 +; GFX11-NEXT: v_cndmask_b32_e64 v22, v68, 0, s10 +; GFX11-NEXT: v_cndmask_b32_e64 v23, v69, 0x7ff80000, s10 +; GFX11-NEXT: v_cndmask_b32_e64 v24, v70, 0, s11 +; GFX11-NEXT: v_cndmask_b32_e64 v25, v71, 0x7ff80000, s11 +; GFX11-NEXT: v_cndmask_b32_e64 v26, v80, 0, s12 +; GFX11-NEXT: v_cndmask_b32_e64 v27, v81, 0x7ff80000, s12 +; GFX11-NEXT: v_cndmask_b32_e64 v28, v82, 0, s13 +; GFX11-NEXT: v_cndmask_b32_e64 v29, v83, 0x7ff80000, s13 +; GFX11-NEXT: v_cndmask_b32_e64 v30, v84, 0, s14 +; GFX11-NEXT: v_cndmask_b32_e64 v31, v85, 0x7ff80000, s14 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_maximum_v16f64: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll index 95d351e8f1fa..e00ebff751c7 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll @@ -443,28 +443,14 @@ define <2 x half> @v_minimum_v2f16(<2 x half> %src0, <2 x half> %src1) { ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX8-NEXT: v_cmp_lt_f16_e32 vcc, v3, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v3, vcc +; GFX8-NEXT: v_min_f16_e32 v4, v3, v2 ; GFX8-NEXT: v_mov_b32_e32 v5, 0x7e00 ; GFX8-NEXT: v_cmp_o_f16_e32 vcc, v3, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v3, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v2, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_eq_f16_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_cmp_lt_f16_e32 vcc, v0, v1 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v1, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc +; GFX8-NEXT: v_min_f16_e32 v3, v0, v1 ; GFX8-NEXT: v_cmp_o_f16_e32 vcc, v0, v1 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v0, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v1, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GFX8-NEXT: v_cmp_eq_f16_e32 vcc, 0, v3 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v3, vcc ; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: s_setpc_b64 s[30:31] ; @@ -543,26 +529,9 @@ define <2 x half> @v_minimum_v2f16__nnan(<2 x half> %src0, <2 x half> %src1) { ; GFX8-LABEL: v_minimum_v2f16__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX8-NEXT: v_cmp_lt_f16_e32 vcc, v3, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v3, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v3, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v2, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_eq_f16_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_cmp_lt_f16_e32 vcc, v0, v1 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v1, v0, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v0, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, v1, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GFX8-NEXT: v_cmp_eq_f16_e32 vcc, 0, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_min_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 +; GFX8-NEXT: v_min_f16_e32 v0, v0, v1 +; GFX8-NEXT: v_or_b32_e32 v0, v0, v2 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v2f16__nnan: @@ -608,13 +577,11 @@ define <2 x half> @v_minimum_v2f16__nsz(<2 x half> %src0, <2 x half> %src1) { ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX8-NEXT: v_cmp_lt_f16_e32 vcc, v3, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v3, vcc +; GFX8-NEXT: v_min_f16_e32 v4, v3, v2 ; GFX8-NEXT: v_mov_b32_e32 v5, 0x7e00 ; GFX8-NEXT: v_cmp_o_f16_e32 vcc, v3, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc -; GFX8-NEXT: v_cmp_lt_f16_e32 vcc, v0, v1 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v1, v0, vcc +; GFX8-NEXT: v_min_f16_e32 v3, v0, v1 ; GFX8-NEXT: v_cmp_o_f16_e32 vcc, v0, v1 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v3, vcc @@ -696,14 +663,9 @@ define <2 x half> @v_minimum_v2f16__nnan_nsz(<2 x half> %src0, <2 x half> %src1) ; GFX8-LABEL: v_minimum_v2f16__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX8-NEXT: v_cmp_lt_f16_e32 vcc, v3, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc -; GFX8-NEXT: v_cmp_lt_f16_e32 vcc, v0, v1 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_min_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 +; GFX8-NEXT: v_min_f16_e32 v0, v0, v1 +; GFX8-NEXT: v_or_b32_e32 v0, v0, v2 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v2f16__nnan_nsz: @@ -750,31 +712,15 @@ define void @s_minimum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) { ; GFX8-NEXT: s_lshr_b32 s6, s5, 16 ; GFX8-NEXT: s_lshr_b32 s7, s4, 16 ; GFX8-NEXT: v_mov_b32_e32 v0, s6 -; GFX8-NEXT: v_mov_b32_e32 v1, s7 -; GFX8-NEXT: v_cmp_lt_f16_e32 vcc, s7, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc -; GFX8-NEXT: v_mov_b32_e32 v3, 0x7e00 +; GFX8-NEXT: v_min_f16_e32 v1, s7, v0 +; GFX8-NEXT: v_mov_b32_e32 v2, 0x7e00 ; GFX8-NEXT: v_cmp_o_f16_e32 vcc, s7, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, s7, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, s6, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc -; GFX8-NEXT: v_cmp_eq_f16_e32 vcc, 0, v2 +; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc ; GFX8-NEXT: v_mov_b32_e32 v1, s5 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX8-NEXT: v_mov_b32_e32 v2, s4 -; GFX8-NEXT: v_cmp_lt_f16_e32 vcc, s4, v1 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc +; GFX8-NEXT: v_min_f16_e32 v3, s4, v1 ; GFX8-NEXT: v_cmp_o_f16_e32 vcc, s4, v1 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, s4, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_class_f16_e64 vcc, s5, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX8-NEXT: v_cmp_eq_f16_e32 vcc, 0, v3 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc ; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: ;;#ASMSTART ; GFX8-NEXT: ; use v0 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll index 1da2647fbd60..e056682051aa 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll @@ -495,167 +495,73 @@ define <2 x float> @v_minimum_v2f32(<2 x float> %src0, <2 x float> %src1) { ; GFX7-LABEL: v_minimum_v2f32: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v4, v0, v2 +; GFX7-NEXT: v_min_f32_e32 v4, v0, v2 ; GFX7-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v2, v1, v3 +; GFX7-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc +; GFX7-NEXT: v_min_f32_e32 v2, v1, v3 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v2f32: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc +; GFX8-NEXT: v_min_f32_e32 v4, v0, v2 ; GFX8-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc +; GFX8-NEXT: v_min_f32_e32 v2, v1, v3 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v2f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc +; GFX9-NEXT: v_min_f32_e32 v4, v0, v2 ; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc +; GFX9-NEXT: v_min_f32_e32 v2, v1, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v2f32: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 +; GFX940-NEXT: v_min_f32_e32 v4, v0, v2 ; GFX940-NEXT: v_mov_b32_e32 v5, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc +; GFX940-NEXT: v_min_f32_e32 v2, v1, v3 +; GFX940-NEXT: s_nop 0 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v2f32: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v4, v0, v2 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v5, v1, v3 +; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v5, 0x7fc00000, v5, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v2f32: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v4, v0, v2 :: v_dual_min_f32 v5, v1, v3 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v5, 0x7fc00000, v5, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v2f32: @@ -676,136 +582,42 @@ define <2 x float> @v_minimum_v2f32__nnan(<2 x float> %src0, <2 x float> %src1) ; GFX7-LABEL: v_minimum_v2f32__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v4, v0, v2 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v2, v1, v3 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX7-NEXT: v_min_f32_e32 v0, v0, v2 +; GFX7-NEXT: v_min_f32_e32 v1, v1, v3 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v2f32__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX8-NEXT: v_min_f32_e32 v0, v0, v2 +; GFX8-NEXT: v_min_f32_e32 v1, v1, v3 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v2f32__nnan: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_min_f32_e32 v0, v0, v2 +; GFX9-NEXT: v_min_f32_e32 v1, v1, v3 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v2f32__nnan: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX940-NEXT: v_min_f32_e32 v0, v0, v2 +; GFX940-NEXT: v_min_f32_e32 v1, v1, v3 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v2f32__nnan: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v0, v0, v2 +; GFX10-NEXT: v_min_f32_e32 v1, v1, v3 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v2f32__nnan: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v0, v0, v2 :: v_dual_min_f32 v1, v1, v3 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v2f32__nnan: @@ -826,11 +638,11 @@ define <2 x float> @v_minimum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) { ; GFX7-LABEL: v_minimum_v2f32__nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v4, v0, v2 +; GFX7-NEXT: v_min_f32_e32 v4, v0, v2 ; GFX7-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v2, v1, v3 +; GFX7-NEXT: v_min_f32_e32 v2, v1, v3 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] @@ -838,13 +650,11 @@ define <2 x float> @v_minimum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) { ; GFX8-LABEL: v_minimum_v2f32__nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc +; GFX8-NEXT: v_min_f32_e32 v4, v0, v2 ; GFX8-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc +; GFX8-NEXT: v_min_f32_e32 v2, v1, v3 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] @@ -852,13 +662,11 @@ define <2 x float> @v_minimum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) { ; GFX9-LABEL: v_minimum_v2f32__nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc +; GFX9-NEXT: v_min_f32_e32 v4, v0, v2 ; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc +; GFX9-NEXT: v_min_f32_e32 v2, v1, v3 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] @@ -866,16 +674,12 @@ define <2 x float> @v_minimum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) { ; GFX940-LABEL: v_minimum_v2f32__nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 +; GFX940-NEXT: v_min_f32_e32 v4, v0, v2 ; GFX940-NEXT: v_mov_b32_e32 v5, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v2 -; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_min_f32_e32 v2, v1, v3 +; GFX940-NEXT: s_nop 0 ; GFX940-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v1, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v3 ; GFX940-NEXT: s_nop 1 ; GFX940-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc @@ -884,11 +688,9 @@ define <2 x float> @v_minimum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) { ; GFX10-LABEL: v_minimum_v2f32__nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v4, v0, v2 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v2 +; GFX10-NEXT: v_min_f32_e32 v5, v1, v3 ; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v3 ; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo @@ -897,12 +699,9 @@ define <2 x float> @v_minimum_v2f32__nsz(<2 x float> %src0, <2 x float> %src1) { ; GFX11-LABEL: v_minimum_v2f32__nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v4, v0, v2 :: v_dual_min_f32 v5, v1, v3 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v4, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v3 ; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v5, vcc_lo @@ -926,55 +725,42 @@ define <2 x float> @v_minimum_v2f32__nnan_nsz(<2 x float> %src0, <2 x float> %sr ; GFX7-LABEL: v_minimum_v2f32__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v0, v0, v2 -; GFX7-NEXT: v_min_legacy_f32_e32 v1, v1, v3 +; GFX7-NEXT: v_min_f32_e32 v0, v0, v2 +; GFX7-NEXT: v_min_f32_e32 v1, v1, v3 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v2f32__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX8-NEXT: v_min_f32_e32 v0, v0, v2 +; GFX8-NEXT: v_min_f32_e32 v1, v1, v3 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v2f32__nnan_nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX9-NEXT: v_min_f32_e32 v0, v0, v2 +; GFX9-NEXT: v_min_f32_e32 v1, v1, v3 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v2f32__nnan_nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX940-NEXT: v_min_f32_e32 v0, v0, v2 +; GFX940-NEXT: v_min_f32_e32 v1, v1, v3 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v2f32__nnan_nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v0, v0, v2 +; GFX10-NEXT: v_min_f32_e32 v1, v1, v3 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v2f32__nnan_nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v0, v0, v2 :: v_dual_min_f32 v1, v1, v3 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v2f32__nnan_nsz: @@ -996,28 +782,14 @@ define void @s_minimum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) { ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_mov_b32_e32 v0, s7 -; GFX7-NEXT: v_min_legacy_f32_e32 v1, s5, v0 +; GFX7-NEXT: v_min_f32_e32 v1, s5, v0 ; GFX7-NEXT: v_mov_b32_e32 v2, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, s5, v0 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX7-NEXT: v_mov_b32_e32 v3, s5 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, s5, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, s7, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v1 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc ; GFX7-NEXT: v_mov_b32_e32 v0, s6 -; GFX7-NEXT: v_min_legacy_f32_e32 v3, s4, v0 +; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX7-NEXT: v_min_f32_e32 v3, s4, v0 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, s4, v0 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc -; GFX7-NEXT: v_mov_b32_e32 v3, s4 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, s4, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v2, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, s6, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ; GFX7-NEXT: ;;#ASMSTART ; GFX7-NEXT: ; use v[0:1] ; GFX7-NEXT: ;;#ASMEND @@ -1027,30 +799,14 @@ define void @s_minimum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) { ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v0, s7 -; GFX8-NEXT: v_mov_b32_e32 v1, s5 -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, s5, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc -; GFX8-NEXT: v_mov_b32_e32 v3, 0x7fc00000 +; GFX8-NEXT: v_min_f32_e32 v1, s5, v0 +; GFX8-NEXT: v_mov_b32_e32 v2, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, s5, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, s5, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, s7, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v0, vcc ; GFX8-NEXT: v_mov_b32_e32 v0, s6 -; GFX8-NEXT: v_mov_b32_e32 v2, s4 -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, s4, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX8-NEXT: v_min_f32_e32 v3, s4, v0 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, s4, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, s4, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, s6, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ; GFX8-NEXT: ;;#ASMSTART ; GFX8-NEXT: ; use v[0:1] ; GFX8-NEXT: ;;#ASMEND @@ -1060,30 +816,14 @@ define void @s_minimum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) { ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v0, s7 -; GFX9-NEXT: v_mov_b32_e32 v1, s5 -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, s5, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc -; GFX9-NEXT: v_mov_b32_e32 v3, 0x7fc00000 +; GFX9-NEXT: v_min_f32_e32 v1, s5, v0 +; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, s5, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, s5, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, s7, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v0, vcc ; GFX9-NEXT: v_mov_b32_e32 v0, s6 -; GFX9-NEXT: v_mov_b32_e32 v2, s4 -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, s4, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_min_f32_e32 v3, s4, v0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, s4, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, s4, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, s6, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ; GFX9-NEXT: ;;#ASMSTART ; GFX9-NEXT: ; use v[0:1] ; GFX9-NEXT: ;;#ASMEND @@ -1093,40 +833,15 @@ define void @s_minimum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) { ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v0, s3 -; GFX940-NEXT: v_mov_b32_e32 v1, s1 -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0 -; GFX940-NEXT: v_mov_b32_e32 v3, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc +; GFX940-NEXT: v_min_f32_e32 v1, s1, v0 +; GFX940-NEXT: v_mov_b32_e32 v2, 0x7fc00000 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, s1, v0 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, s1, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, s3, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v2 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v0, vcc ; GFX940-NEXT: v_mov_b32_e32 v0, s2 -; GFX940-NEXT: v_mov_b32_e32 v2, s0 -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc +; GFX940-NEXT: v_min_f32_e32 v3, s0, v0 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, s0, v0 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, s0, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, s2, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ; GFX940-NEXT: ;;#ASMSTART ; GFX940-NEXT: ; use v[0:1] ; GFX940-NEXT: ;;#ASMEND @@ -1135,28 +850,12 @@ define void @s_minimum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) { ; GFX10-LABEL: s_minimum_v2f32: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v0, s5 -; GFX10-NEXT: v_cmp_lt_f32_e64 vcc_lo, s5, s7 -; GFX10-NEXT: v_mov_b32_e32 v1, s4 -; GFX10-NEXT: v_cmp_class_f32_e64 s8, s5, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, s7, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e64 vcc_lo, s4, s6 -; GFX10-NEXT: v_cndmask_b32_e32 v1, s6, v1, vcc_lo +; GFX10-NEXT: v_min_f32_e64 v0, s5, s7 ; GFX10-NEXT: v_cmp_o_f32_e64 vcc_lo, s5, s7 -; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo +; GFX10-NEXT: v_min_f32_e64 v2, s4, s6 +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v0, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e64 vcc_lo, s4, s6 -; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v1, v0, s5, s8 -; GFX10-NEXT: v_cmp_class_f32_e64 s5, s4, 32 -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v2, s4, s5 -; GFX10-NEXT: v_cmp_class_f32_e64 s4, s7, 32 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s7, s4 -; GFX10-NEXT: v_cmp_class_f32_e64 s4, s6, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s6, s4 -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo ; GFX10-NEXT: ;;#ASMSTART ; GFX10-NEXT: ; use v[0:1] ; GFX10-NEXT: ;;#ASMEND @@ -1165,32 +864,13 @@ define void @s_minimum_v2f32(<2 x float> inreg %src0, <2 x float> inreg %src1) { ; GFX11-LABEL: s_minimum_v2f32: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0 -; GFX11-NEXT: v_cmp_lt_f32_e64 vcc_lo, s1, s3 -; GFX11-NEXT: v_cmp_class_f32_e64 s4, s1, 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, s3, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e64 vcc_lo, s0, s2 -; GFX11-NEXT: v_cndmask_b32_e32 v1, s2, v1, vcc_lo +; GFX11-NEXT: v_min_f32_e64 v0, s1, s3 ; GFX11-NEXT: v_cmp_o_f32_e64 vcc_lo, s1, s3 -; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo +; GFX11-NEXT: v_min_f32_e64 v2, s0, s2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v0, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e64 vcc_lo, s0, s2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v1, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v1, v0, s1, s4 -; GFX11-NEXT: v_cmp_class_f32_e64 s1, s0, 32 -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_cndmask_b32_e64 v3, v2, s0, s1 -; GFX11-NEXT: v_cmp_class_f32_e64 s0, s3, 32 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s3, s0 -; GFX11-NEXT: v_cmp_class_f32_e64 s0, s2, 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s2, s0 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo ; GFX11-NEXT: ;;#ASMSTART ; GFX11-NEXT: ; use v[0:1] ; GFX11-NEXT: ;;#ASMEND @@ -1218,227 +898,92 @@ define <3 x float> @v_minimum_v3f32(<3 x float> %src0, <3 x float> %src1) { ; GFX7-LABEL: v_minimum_v3f32: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v6, v0, v3 +; GFX7-NEXT: v_min_f32_e32 v6, v0, v3 ; GFX7-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v3, v1, v4 +; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc +; GFX7-NEXT: v_min_f32_e32 v3, v1, v4 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v3, v2, v5 +; GFX7-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc +; GFX7-NEXT: v_min_f32_e32 v3, v2, v5 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v3f32: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc +; GFX8-NEXT: v_min_f32_e32 v6, v0, v3 ; GFX8-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc +; GFX8-NEXT: v_min_f32_e32 v3, v1, v4 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc +; GFX8-NEXT: v_min_f32_e32 v3, v2, v5 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v3f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc +; GFX9-NEXT: v_min_f32_e32 v6, v0, v3 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v1, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v2, v5 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v3f32: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 +; GFX940-NEXT: v_min_f32_e32 v6, v0, v3 ; GFX940-NEXT: v_mov_b32_e32 v7, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc +; GFX940-NEXT: v_min_f32_e32 v3, v1, v4 +; GFX940-NEXT: s_nop 0 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc +; GFX940-NEXT: v_min_f32_e32 v3, v2, v5 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v3f32: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v4, v1, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v6, v0, v3 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v6, 0x7fc00000, v6, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v7, v1, v4 +; GFX10-NEXT: v_min_f32_e32 v8, v2, v5 +; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v7, 0x7fc00000, v7, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v7, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v3f32: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v4, v1, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v6, v0, v3 :: v_dual_min_f32 v7, v1, v4 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v6, 0x7fc00000, v6, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v7, 0x7fc00000, v7, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v8, v2, v5 :: v_dual_cndmask_b32 v1, 0x7fc00000, v7 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v3f32: @@ -1460,184 +1005,48 @@ define <3 x float> @v_minimum_v3f32__nnan(<3 x float> %src0, <3 x float> %src1) ; GFX7-LABEL: v_minimum_v3f32__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v6, v0, v3 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v3, v1, v4 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v3, v2, v5 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX7-NEXT: v_min_f32_e32 v0, v0, v3 +; GFX7-NEXT: v_min_f32_e32 v1, v1, v4 +; GFX7-NEXT: v_min_f32_e32 v2, v2, v5 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v3f32__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX8-NEXT: v_min_f32_e32 v0, v0, v3 +; GFX8-NEXT: v_min_f32_e32 v1, v1, v4 +; GFX8-NEXT: v_min_f32_e32 v2, v2, v5 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v3f32__nnan: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_min_f32_e32 v0, v0, v3 +; GFX9-NEXT: v_min_f32_e32 v1, v1, v4 +; GFX9-NEXT: v_min_f32_e32 v2, v2, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v3f32__nnan: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v6 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX940-NEXT: v_min_f32_e32 v0, v0, v3 +; GFX940-NEXT: v_min_f32_e32 v1, v1, v4 +; GFX940-NEXT: v_min_f32_e32 v2, v2, v5 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v3f32__nnan: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v4, v1, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v0, v0, v3 +; GFX10-NEXT: v_min_f32_e32 v1, v1, v4 +; GFX10-NEXT: v_min_f32_e32 v2, v2, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v3f32__nnan: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v4, v1, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v0, v0, v3 :: v_dual_min_f32 v1, v1, v4 +; GFX11-NEXT: v_min_f32_e32 v2, v2, v5 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v3f32__nnan: @@ -1659,14 +1068,14 @@ define <3 x float> @v_minimum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) { ; GFX7-LABEL: v_minimum_v3f32__nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v6, v0, v3 +; GFX7-NEXT: v_min_f32_e32 v6, v0, v3 ; GFX7-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v3, v1, v4 +; GFX7-NEXT: v_min_f32_e32 v3, v1, v4 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v3, v2, v5 +; GFX7-NEXT: v_min_f32_e32 v3, v2, v5 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 ; GFX7-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] @@ -1674,17 +1083,14 @@ define <3 x float> @v_minimum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) { ; GFX8-LABEL: v_minimum_v3f32__nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc +; GFX8-NEXT: v_min_f32_e32 v6, v0, v3 ; GFX8-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc +; GFX8-NEXT: v_min_f32_e32 v3, v1, v4 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX8-NEXT: v_min_f32_e32 v3, v2, v5 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] @@ -1692,17 +1098,14 @@ define <3 x float> @v_minimum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) { ; GFX9-LABEL: v_minimum_v3f32__nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc +; GFX9-NEXT: v_min_f32_e32 v6, v0, v3 ; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v1, v4 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX9-NEXT: v_min_f32_e32 v3, v2, v5 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] @@ -1710,22 +1113,16 @@ define <3 x float> @v_minimum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) { ; GFX940-LABEL: v_minimum_v3f32__nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 +; GFX940-NEXT: v_min_f32_e32 v6, v0, v3 ; GFX940-NEXT: v_mov_b32_e32 v7, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v3 -; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_min_f32_e32 v3, v1, v4 +; GFX940-NEXT: s_nop 0 ; GFX940-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v1, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v4 ; GFX940-NEXT: s_nop 1 ; GFX940-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v2, vcc +; GFX940-NEXT: v_min_f32_e32 v3, v2, v5 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v2, v5 ; GFX940-NEXT: s_nop 1 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc @@ -1734,13 +1131,10 @@ define <3 x float> @v_minimum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) { ; GFX10-LABEL: v_minimum_v3f32__nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v4, v1, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v6, v0, v3 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v3 +; GFX10-NEXT: v_min_f32_e32 v7, v1, v4 +; GFX10-NEXT: v_min_f32_e32 v8, v2, v5 ; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v4 ; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v7, vcc_lo @@ -1751,17 +1145,14 @@ define <3 x float> @v_minimum_v3f32__nsz(<3 x float> %src0, <3 x float> %src1) { ; GFX11-LABEL: v_minimum_v3f32__nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v3, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v4, v1, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v6, v0, v3 :: v_dual_min_f32 v7, v1, v4 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v6, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v7, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v8, v2, v5 :: v_dual_cndmask_b32 v1, 0x7fc00000, v7 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; @@ -1784,67 +1175,48 @@ define <3 x float> @v_minimum_v3f32__nnan_nsz(<3 x float> %src0, <3 x float> %sr ; GFX7-LABEL: v_minimum_v3f32__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v0, v0, v3 -; GFX7-NEXT: v_min_legacy_f32_e32 v1, v1, v4 -; GFX7-NEXT: v_min_legacy_f32_e32 v2, v2, v5 +; GFX7-NEXT: v_min_f32_e32 v0, v0, v3 +; GFX7-NEXT: v_min_f32_e32 v1, v1, v4 +; GFX7-NEXT: v_min_f32_e32 v2, v2, v5 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v3f32__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc +; GFX8-NEXT: v_min_f32_e32 v0, v0, v3 +; GFX8-NEXT: v_min_f32_e32 v1, v1, v4 +; GFX8-NEXT: v_min_f32_e32 v2, v2, v5 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v3f32__nnan_nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc +; GFX9-NEXT: v_min_f32_e32 v0, v0, v3 +; GFX9-NEXT: v_min_f32_e32 v1, v1, v4 +; GFX9-NEXT: v_min_f32_e32 v2, v2, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v3f32__nnan_nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v2, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc +; GFX940-NEXT: v_min_f32_e32 v0, v0, v3 +; GFX940-NEXT: v_min_f32_e32 v1, v1, v4 +; GFX940-NEXT: v_min_f32_e32 v2, v2, v5 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v3f32__nnan_nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v0, v0, v3 +; GFX10-NEXT: v_min_f32_e32 v1, v1, v4 +; GFX10-NEXT: v_min_f32_e32 v2, v2, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v3f32__nnan_nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v0, v0, v3 :: v_dual_min_f32 v1, v1, v4 +; GFX11-NEXT: v_min_f32_e32 v2, v2, v5 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v3f32__nnan_nsz: @@ -1866,292 +1238,111 @@ define <4 x float> @v_minimum_v4f32(<4 x float> %src0, <4 x float> %src1) { ; GFX7-LABEL: v_minimum_v4f32: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v8, v0, v4 +; GFX7-NEXT: v_min_f32_e32 v8, v0, v4 ; GFX7-NEXT: v_mov_b32_e32 v9, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v4, v1, v5 +; GFX7-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc +; GFX7-NEXT: v_min_f32_e32 v4, v1, v5 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v4, v2, v6 +; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc +; GFX7-NEXT: v_min_f32_e32 v4, v2, v6 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v4, v3, v7 +; GFX7-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc +; GFX7-NEXT: v_min_f32_e32 v4, v3, v7 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v4f32: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc +; GFX8-NEXT: v_min_f32_e32 v8, v0, v4 ; GFX8-NEXT: v_mov_b32_e32 v9, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc +; GFX8-NEXT: v_min_f32_e32 v4, v1, v5 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v2, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc +; GFX8-NEXT: v_min_f32_e32 v4, v2, v6 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v3, v7 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc +; GFX8-NEXT: v_min_f32_e32 v4, v3, v7 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v4f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc +; GFX9-NEXT: v_min_f32_e32 v8, v0, v4 ; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc +; GFX9-NEXT: v_min_f32_e32 v4, v1, v5 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc +; GFX9-NEXT: v_min_f32_e32 v4, v2, v6 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v3, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc +; GFX9-NEXT: v_min_f32_e32 v4, v3, v7 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v4f32: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4 +; GFX940-NEXT: v_min_f32_e32 v8, v0, v4 ; GFX940-NEXT: v_mov_b32_e32 v9, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc +; GFX940-NEXT: v_min_f32_e32 v4, v1, v5 +; GFX940-NEXT: s_nop 0 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v2, v6 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc +; GFX940-NEXT: v_min_f32_e32 v4, v2, v6 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v3, v7 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc +; GFX940-NEXT: v_min_f32_e32 v4, v3, v7 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v4f32: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v8, v0, v4 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v10, v6, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v9, v1, v5 +; GFX10-NEXT: v_min_f32_e32 v4, v2, v6 +; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v8, v3, v7 +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v9, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v10, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v4f32: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v8, v0, v4 :: v_dual_min_f32 v9, v1, v5 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v10, v6, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo +; GFX11-NEXT: v_min_f32_e32 v4, v2, v6 +; GFX11-NEXT: v_dual_min_f32 v8, v3, v7 :: v_dual_cndmask_b32 v1, 0x7fc00000, v9 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v10, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v4, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v4f32: @@ -2174,236 +1365,53 @@ define <4 x float> @v_minimum_v4f32__nnan(<4 x float> %src0, <4 x float> %src1) ; GFX7-LABEL: v_minimum_v4f32__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v8, v0, v4 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v4, v1, v5 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v4, v2, v6 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v4, v3, v7 -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX7-NEXT: v_min_f32_e32 v0, v0, v4 +; GFX7-NEXT: v_min_f32_e32 v1, v1, v5 +; GFX7-NEXT: v_min_f32_e32 v2, v2, v6 +; GFX7-NEXT: v_min_f32_e32 v3, v3, v7 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v4f32__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v2, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v3, v7 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX8-NEXT: v_min_f32_e32 v0, v0, v4 +; GFX8-NEXT: v_min_f32_e32 v1, v1, v5 +; GFX8-NEXT: v_min_f32_e32 v2, v2, v6 +; GFX8-NEXT: v_min_f32_e32 v3, v3, v7 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v4f32__nnan: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v3, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX9-NEXT: v_min_f32_e32 v0, v0, v4 +; GFX9-NEXT: v_min_f32_e32 v1, v1, v5 +; GFX9-NEXT: v_min_f32_e32 v2, v2, v6 +; GFX9-NEXT: v_min_f32_e32 v3, v3, v7 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v4f32__nnan: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v2, v6 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v3, v7 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX940-NEXT: v_min_f32_e32 v0, v0, v4 +; GFX940-NEXT: v_min_f32_e32 v1, v1, v5 +; GFX940-NEXT: v_min_f32_e32 v2, v2, v6 +; GFX940-NEXT: v_min_f32_e32 v3, v3, v7 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v4f32__nnan: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v6, v2, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v7, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v0, v0, v4 +; GFX10-NEXT: v_min_f32_e32 v1, v1, v5 +; GFX10-NEXT: v_min_f32_e32 v2, v2, v6 +; GFX10-NEXT: v_min_f32_e32 v3, v3, v7 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v4f32__nnan: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v6, v2, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v7, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v0, v0, v4 :: v_dual_min_f32 v1, v1, v5 +; GFX11-NEXT: v_dual_min_f32 v2, v2, v6 :: v_dual_min_f32 v3, v3, v7 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v4f32__nnan: @@ -2426,17 +1434,17 @@ define <4 x float> @v_minimum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) { ; GFX7-LABEL: v_minimum_v4f32__nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v8, v0, v4 +; GFX7-NEXT: v_min_f32_e32 v8, v0, v4 ; GFX7-NEXT: v_mov_b32_e32 v9, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v4, v1, v5 +; GFX7-NEXT: v_min_f32_e32 v4, v1, v5 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v4, v2, v6 +; GFX7-NEXT: v_min_f32_e32 v4, v2, v6 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 ; GFX7-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v4, v3, v7 +; GFX7-NEXT: v_min_f32_e32 v4, v3, v7 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 ; GFX7-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] @@ -2444,21 +1452,17 @@ define <4 x float> @v_minimum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) { ; GFX8-LABEL: v_minimum_v4f32__nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc +; GFX8-NEXT: v_min_f32_e32 v8, v0, v4 ; GFX8-NEXT: v_mov_b32_e32 v9, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc +; GFX8-NEXT: v_min_f32_e32 v4, v1, v5 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v2, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc +; GFX8-NEXT: v_min_f32_e32 v4, v2, v6 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v3, v7 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc +; GFX8-NEXT: v_min_f32_e32 v4, v3, v7 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] @@ -2466,21 +1470,17 @@ define <4 x float> @v_minimum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) { ; GFX9-LABEL: v_minimum_v4f32__nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc +; GFX9-NEXT: v_min_f32_e32 v8, v0, v4 ; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc +; GFX9-NEXT: v_min_f32_e32 v4, v1, v5 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc +; GFX9-NEXT: v_min_f32_e32 v4, v2, v6 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v3, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc +; GFX9-NEXT: v_min_f32_e32 v4, v3, v7 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] @@ -2488,28 +1488,20 @@ define <4 x float> @v_minimum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) { ; GFX940-LABEL: v_minimum_v4f32__nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4 +; GFX940-NEXT: v_min_f32_e32 v8, v0, v4 ; GFX940-NEXT: v_mov_b32_e32 v9, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v4 -; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_min_f32_e32 v4, v1, v5 +; GFX940-NEXT: s_nop 0 ; GFX940-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v5 ; GFX940-NEXT: s_nop 1 ; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v2, v6 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc +; GFX940-NEXT: v_min_f32_e32 v4, v2, v6 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v2, v6 ; GFX940-NEXT: s_nop 1 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v9, v4, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v3, v7 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc +; GFX940-NEXT: v_min_f32_e32 v4, v3, v7 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v3, v7 ; GFX940-NEXT: s_nop 1 ; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v4, vcc @@ -2518,44 +1510,35 @@ define <4 x float> @v_minimum_v4f32__nsz(<4 x float> %src0, <4 x float> %src1) { ; GFX10-LABEL: v_minimum_v4f32__nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v8, v0, v4 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v4 +; GFX10-NEXT: v_min_f32_e32 v9, v1, v5 +; GFX10-NEXT: v_min_f32_e32 v4, v2, v6 ; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v6, v2, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v7, v3, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v4, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v8, v3, v7 +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v9, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v9, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v4f32__nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v8, v0, v4 :: v_dual_min_f32 v9, v1, v5 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3) ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v8, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v1, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v6, v2, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v7, v3, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v4, vcc_lo +; GFX11-NEXT: v_min_f32_e32 v4, v2, v6 +; GFX11-NEXT: v_dual_min_f32 v8, v3, v7 :: v_dual_cndmask_b32 v1, 0x7fc00000, v9 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v4, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v9, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v4f32__nsz: @@ -2578,79 +1561,53 @@ define <4 x float> @v_minimum_v4f32__nnan_nsz(<4 x float> %src0, <4 x float> %sr ; GFX7-LABEL: v_minimum_v4f32__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v0, v0, v4 -; GFX7-NEXT: v_min_legacy_f32_e32 v1, v1, v5 -; GFX7-NEXT: v_min_legacy_f32_e32 v2, v2, v6 -; GFX7-NEXT: v_min_legacy_f32_e32 v3, v3, v7 +; GFX7-NEXT: v_min_f32_e32 v0, v0, v4 +; GFX7-NEXT: v_min_f32_e32 v1, v1, v5 +; GFX7-NEXT: v_min_f32_e32 v2, v2, v6 +; GFX7-NEXT: v_min_f32_e32 v3, v3, v7 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v4f32__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v2, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v3, v7 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX8-NEXT: v_min_f32_e32 v0, v0, v4 +; GFX8-NEXT: v_min_f32_e32 v1, v1, v5 +; GFX8-NEXT: v_min_f32_e32 v2, v2, v6 +; GFX8-NEXT: v_min_f32_e32 v3, v3, v7 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v4f32__nnan_nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v3, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX9-NEXT: v_min_f32_e32 v0, v0, v4 +; GFX9-NEXT: v_min_f32_e32 v1, v1, v5 +; GFX9-NEXT: v_min_f32_e32 v2, v2, v6 +; GFX9-NEXT: v_min_f32_e32 v3, v3, v7 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v4f32__nnan_nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v5 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v2, v6 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v3, v7 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX940-NEXT: v_min_f32_e32 v0, v0, v4 +; GFX940-NEXT: v_min_f32_e32 v1, v1, v5 +; GFX940-NEXT: v_min_f32_e32 v2, v2, v6 +; GFX940-NEXT: v_min_f32_e32 v3, v3, v7 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v4f32__nnan_nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v4 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v0, v0, v4 +; GFX10-NEXT: v_min_f32_e32 v1, v1, v5 +; GFX10-NEXT: v_min_f32_e32 v2, v2, v6 +; GFX10-NEXT: v_min_f32_e32 v3, v3, v7 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v4f32__nnan_nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v4 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v6 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v0, v0, v4 :: v_dual_min_f32 v1, v1, v5 +; GFX11-NEXT: v_dual_min_f32 v2, v2, v6 :: v_dual_min_f32 v3, v3, v7 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v4f32__nnan_nsz: @@ -2673,551 +1630,185 @@ define <8 x float> @v_minimum_v8f32(<8 x float> %src0, <8 x float> %src1) { ; GFX7-LABEL: v_minimum_v8f32: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v0, v8 +; GFX7-NEXT: v_min_f32_e32 v16, v0, v8 ; GFX7-NEXT: v_mov_b32_e32 v17, 0x7fc00000 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v8, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v8, v1, v9 +; GFX7-NEXT: v_cndmask_b32_e32 v0, v17, v16, vcc +; GFX7-NEXT: v_min_f32_e32 v8, v1, v9 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v9 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v9, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v8, v2, v10 +; GFX7-NEXT: v_cndmask_b32_e32 v1, v17, v8, vcc +; GFX7-NEXT: v_min_f32_e32 v8, v2, v10 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v10 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v10, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v8, v3, v11 +; GFX7-NEXT: v_cndmask_b32_e32 v2, v17, v8, vcc +; GFX7-NEXT: v_min_f32_e32 v8, v3, v11 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v3, v11 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v11, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v8, v4, v12 +; GFX7-NEXT: v_cndmask_b32_e32 v3, v17, v8, vcc +; GFX7-NEXT: v_min_f32_e32 v8, v4, v12 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v4, v12 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v12, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v12, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v8, v5, v13 +; GFX7-NEXT: v_cndmask_b32_e32 v4, v17, v8, vcc +; GFX7-NEXT: v_min_f32_e32 v8, v5, v13 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v5, v13 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v13, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v8, v6, v14 +; GFX7-NEXT: v_cndmask_b32_e32 v5, v17, v8, vcc +; GFX7-NEXT: v_min_f32_e32 v8, v6, v14 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v6, v14 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v14, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v8, v7, v15 +; GFX7-NEXT: v_cndmask_b32_e32 v6, v17, v8, vcc +; GFX7-NEXT: v_min_f32_e32 v8, v7, v15 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v7, v15 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v15, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX7-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v7, v17, v8, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v8f32: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc +; GFX8-NEXT: v_min_f32_e32 v16, v0, v8 ; GFX8-NEXT: v_mov_b32_e32 v17, 0x7fc00000 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v8, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v9 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v9, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v0, v17, v16, vcc +; GFX8-NEXT: v_min_f32_e32 v8, v1, v9 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v9 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v9, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v2, v10 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v8, vcc +; GFX8-NEXT: v_min_f32_e32 v8, v2, v10 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v2, v10 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v10, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v3, v11 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v11, v3, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v2, v17, v8, vcc +; GFX8-NEXT: v_min_f32_e32 v8, v3, v11 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v3, v11 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v11, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v4, v12 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v12, v4, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v3, v17, v8, vcc +; GFX8-NEXT: v_min_f32_e32 v8, v4, v12 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v4, v12 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v12, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v12, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v5, v13 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v13, v5, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v4, v17, v8, vcc +; GFX8-NEXT: v_min_f32_e32 v8, v5, v13 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v5, v13 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v13, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v6, v14 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v14, v6, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v5, v17, v8, vcc +; GFX8-NEXT: v_min_f32_e32 v8, v6, v14 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v6, v14 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v14, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v7, v15 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v15, v7, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v6, v17, v8, vcc +; GFX8-NEXT: v_min_f32_e32 v8, v7, v15 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v7, v15 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v15, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v7, v17, v8, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v8f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc +; GFX9-NEXT: v_min_f32_e32 v16, v0, v8 ; GFX9-NEXT: v_mov_b32_e32 v17, 0x7fc00000 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v8, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v17, v16, vcc +; GFX9-NEXT: v_min_f32_e32 v8, v1, v9 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v9, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v10 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v8, vcc +; GFX9-NEXT: v_min_f32_e32 v8, v2, v10 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v10 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v10, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v3, v11 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v11, v3, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v17, v8, vcc +; GFX9-NEXT: v_min_f32_e32 v8, v3, v11 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v3, v11 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v11, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v4, v12 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v12, v4, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v3, v17, v8, vcc +; GFX9-NEXT: v_min_f32_e32 v8, v4, v12 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v4, v12 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v12, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v12, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v5, v13 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v13, v5, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v4, v17, v8, vcc +; GFX9-NEXT: v_min_f32_e32 v8, v5, v13 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v5, v13 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v13, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v6, v14 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v14, v6, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v5, v17, v8, vcc +; GFX9-NEXT: v_min_f32_e32 v8, v6, v14 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v6, v14 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v14, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v7, v15 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v15, v7, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v6, v17, v8, vcc +; GFX9-NEXT: v_min_f32_e32 v8, v7, v15 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v7, v15 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v15, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v7, v17, v8, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v8f32: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v8 +; GFX940-NEXT: v_min_f32_e32 v16, v0, v8 ; GFX940-NEXT: v_mov_b32_e32 v17, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v8, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v9 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v9, v1, vcc +; GFX940-NEXT: v_min_f32_e32 v8, v1, v9 +; GFX940-NEXT: s_nop 0 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v17, v16, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v9 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v9, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v2, v10 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v1, v17, v8, vcc +; GFX940-NEXT: v_min_f32_e32 v8, v2, v10 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v2, v10 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v10, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v3, v11 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v11, v3, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v2, v17, v8, vcc +; GFX940-NEXT: v_min_f32_e32 v8, v3, v11 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v3, v11 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v11, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v4, v12 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v12, v4, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v17, v8, vcc +; GFX940-NEXT: v_min_f32_e32 v8, v4, v12 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v4, v12 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v12, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v4, v12, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v5, v13 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v13, v5, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v4, v17, v8, vcc +; GFX940-NEXT: v_min_f32_e32 v8, v5, v13 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v5, v13 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v13, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v6, v14 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v14, v6, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v5, v17, v8, vcc +; GFX940-NEXT: v_min_f32_e32 v8, v6, v14 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v6, v14 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v14, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v7, v15 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v15, v7, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v6, v17, v8, vcc +; GFX940-NEXT: v_min_f32_e32 v8, v7, v15 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v7, v15 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v17, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v15, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v8 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v7, v17, v8, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v8f32: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v9 -; GFX10-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v16, v0, v8 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v17, v1, v9 +; GFX10-NEXT: v_min_f32_e32 v8, v2, v10 +; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v16, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v9 -; GFX10-NEXT: v_cndmask_b32_e32 v17, 0x7fc00000, v17, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v8, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v9, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v10 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v11 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v11, v3, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v12 -; GFX10-NEXT: v_cndmask_b32_e32 v16, v12, v4, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v9, v3, v11 +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v17, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v10 -; GFX10-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v10, v7, v15 +; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v11 -; GFX10-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v8, v4, v12 +; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v9, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v4, v12 -; GFX10-NEXT: v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v10, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v11, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v12, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v12, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v13 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v13, v5, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v14 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v14, v6, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v15 -; GFX10-NEXT: v_cndmask_b32_e32 v10, v15, v7, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v9, v5, v13 +; GFX10-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v5, v13 -; GFX10-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v8, v6, v14 +; GFX10-NEXT: v_cndmask_b32_e32 v5, 0x7fc00000, v9, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v6, v14 -; GFX10-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v6, 0x7fc00000, v8, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v7, v15 -; GFX10-NEXT: v_cndmask_b32_e32 v10, 0x7fc00000, v10, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v13, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v14, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v15, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v7, 0x7fc00000, v10, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v8f32: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v9 -; GFX11-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v16, v0, v8 :: v_dual_min_f32 v17, v1, v9 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v8 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v16, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v9 -; GFX11-NEXT: v_cndmask_b32_e32 v17, 0x7fc00000, v17, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v8, 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v9, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v10 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v11 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v11, v3, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v12 -; GFX11-NEXT: v_cndmask_b32_e32 v16, v12, v4, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v9, v3, v11 :: v_dual_min_f32 v8, v2, v10 +; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v17, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v10 -; GFX11-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo +; GFX11-NEXT: v_min_f32_e32 v10, v7, v15 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v11 -; GFX11-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v8, v4, v12 :: v_dual_cndmask_b32 v3, 0x7fc00000, v9 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v4, v12 -; GFX11-NEXT: v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v10, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v11, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v12, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v12, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v13 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v13, v5, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v14 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v14, v6, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v15 -; GFX11-NEXT: v_cndmask_b32_e32 v10, v15, v7, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v9, v5, v13 :: v_dual_cndmask_b32 v4, 0x7fc00000, v8 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v5, v13 -; GFX11-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_dual_min_f32 v8, v6, v14 :: v_dual_cndmask_b32 v5, 0x7fc00000, v9 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v6, v14 -; GFX11-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v9, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v6, 0x7fc00000, v8, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v7, v15 -; GFX11-NEXT: v_cndmask_b32_e32 v10, 0x7fc00000, v10, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v13, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v14, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v15, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v7, 0x7fc00000, v10, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v8f32: @@ -3244,1071 +1835,371 @@ define <16 x float> @v_minimum_v16f32(<16 x float> %src0, <16 x float> %src1) { ; GFX7-LABEL: v_minimum_v16f32: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v32, v0, v16 -; GFX7-NEXT: v_mov_b32_e32 v31, 0x7fc00000 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v32, v31, v32, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v16, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v1, v17 +; GFX7-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX7-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX7-NEXT: s_mov_b64 exec, s[4:5] +; GFX7-NEXT: v_cmp_o_f32_e64 s[16:17], v0, v16 +; GFX7-NEXT: v_min_f32_e32 v0, v0, v16 +; GFX7-NEXT: buffer_load_dword v16, off, s[0:3], s32 +; GFX7-NEXT: v_writelane_b32 v31, s30, 0 ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v1, v17 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v17, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX7-NEXT: buffer_load_dword v17, off, s[0:3], s32 -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v2, v18 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v2, v18 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v18, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v3, v19 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v3, v19 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v19, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v4, v20 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v4, v20 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v20, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v20, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v5, v21 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v5, v21 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v21, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v21, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v6, v22 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v6, v22 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v22, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v22, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v7, v23 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v7, v23 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v23, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v8, v24 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v8, v24 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v8, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v24, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v9, v25 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v9, v25 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v9, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v25, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v9, v9, v25, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v10, v26 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v10, v26 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v10, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v26, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v10, v10, v26, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v11, v27 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v11, v27 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v11, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v27, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v11, v11, v27, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v12, v28 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v12, v28 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v12, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v28, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v12, v12, v28, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v13, v29 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v13, v29 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v13, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v29, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v14, v30 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v14, v30 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v14, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v30, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc +; GFX7-NEXT: v_min_f32_e32 v1, v1, v17 +; GFX7-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v18 +; GFX7-NEXT: v_min_f32_e32 v2, v2, v18 +; GFX7-NEXT: v_mov_b32_e32 v17, 0x7fc00000 +; GFX7-NEXT: v_min_f32_e32 v18, v13, v29 +; GFX7-NEXT: v_cmp_o_f32_e64 s[28:29], v13, v29 +; GFX7-NEXT: v_writelane_b32 v31, s31, 1 +; GFX7-NEXT: v_cmp_o_f32_e64 s[6:7], v3, v19 +; GFX7-NEXT: v_min_f32_e32 v3, v3, v19 +; GFX7-NEXT: v_cmp_o_f32_e64 s[8:9], v4, v20 +; GFX7-NEXT: v_min_f32_e32 v4, v4, v20 +; GFX7-NEXT: v_cmp_o_f32_e64 s[10:11], v5, v21 +; GFX7-NEXT: v_min_f32_e32 v5, v5, v21 +; GFX7-NEXT: v_cmp_o_f32_e64 s[12:13], v6, v22 +; GFX7-NEXT: v_min_f32_e32 v6, v6, v22 +; GFX7-NEXT: v_cmp_o_f32_e64 s[14:15], v7, v23 +; GFX7-NEXT: v_min_f32_e32 v7, v7, v23 +; GFX7-NEXT: v_cmp_o_f32_e64 s[18:19], v8, v24 +; GFX7-NEXT: v_min_f32_e32 v8, v8, v24 +; GFX7-NEXT: v_cmp_o_f32_e64 s[20:21], v9, v25 +; GFX7-NEXT: v_min_f32_e32 v9, v9, v25 +; GFX7-NEXT: v_cmp_o_f32_e64 s[22:23], v10, v26 +; GFX7-NEXT: v_min_f32_e32 v10, v10, v26 +; GFX7-NEXT: v_cmp_o_f32_e64 s[24:25], v11, v27 +; GFX7-NEXT: v_min_f32_e32 v11, v11, v27 +; GFX7-NEXT: v_cmp_o_f32_e64 s[26:27], v12, v28 +; GFX7-NEXT: v_min_f32_e32 v12, v12, v28 +; GFX7-NEXT: v_min_f32_e32 v19, v14, v30 +; GFX7-NEXT: v_cmp_o_f32_e64 s[30:31], v14, v30 +; GFX7-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v13, v17, v18, s[28:29] +; GFX7-NEXT: v_cndmask_b32_e64 v0, v17, v0, s[16:17] +; GFX7-NEXT: v_cndmask_b32_e64 v2, v17, v2, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v17, v3, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v4, v17, v4, s[8:9] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[10:11] +; GFX7-NEXT: v_cndmask_b32_e64 v6, v17, v6, s[12:13] +; GFX7-NEXT: v_cndmask_b32_e64 v7, v17, v7, s[14:15] +; GFX7-NEXT: v_cndmask_b32_e64 v8, v17, v8, s[18:19] +; GFX7-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[20:21] +; GFX7-NEXT: v_cndmask_b32_e64 v10, v17, v10, s[22:23] +; GFX7-NEXT: v_cndmask_b32_e64 v11, v17, v11, s[24:25] +; GFX7-NEXT: v_cndmask_b32_e64 v12, v17, v12, s[26:27] +; GFX7-NEXT: v_cndmask_b32_e64 v14, v17, v19, s[30:31] +; GFX7-NEXT: v_readlane_b32 s31, v31, 1 +; GFX7-NEXT: v_readlane_b32 s30, v31, 0 +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_min_f32_e32 v18, v15, v16 +; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v15, v16 +; GFX7-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc +; GFX7-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload +; GFX7-NEXT: s_mov_b64 exec, s[4:5] ; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_min_legacy_f32_e32 v16, v15, v17 -; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v15, v17 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v15, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc -; GFX7-NEXT: v_cmp_class_f32_e64 vcc, v17, 32 -; GFX7-NEXT: v_cndmask_b32_e32 v15, v15, v17, vcc -; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX7-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v16f32: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v32, v16, v0, vcc -; GFX8-NEXT: v_mov_b32_e32 v31, 0x7fc00000 -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v32, v31, v32, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v16, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v1, v17 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v17, v1, vcc +; GFX8-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX8-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX8-NEXT: s_mov_b64 exec, s[4:5] +; GFX8-NEXT: v_cmp_o_f32_e64 s[16:17], v0, v16 +; GFX8-NEXT: v_min_f32_e32 v0, v0, v16 +; GFX8-NEXT: buffer_load_dword v16, off, s[0:3], s32 +; GFX8-NEXT: v_writelane_b32 v31, s30, 0 ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v1, v17 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v17, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX8-NEXT: buffer_load_dword v17, off, s[0:3], s32 -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v2, v18 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v2, v18 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v18, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v3, v19 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v3, v19 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v19, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v4, v20 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v20, v4, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v4, v20 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v20, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v20, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v5, v21 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v21, v5, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v5, v21 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v21, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v21, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v6, v22 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v6, v22 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v22, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v22, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v7, v23 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v23, v7, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v7, v23 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v23, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v8, v24 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v24, v8, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v8, v24 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v8, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v24, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v9, v25 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v25, v9, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v9, v25 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v9, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v25, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v9, v9, v25, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v10, v26 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v10, v26 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v10, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v26, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v10, v10, v26, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v11, v27 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v27, v11, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v11, v27 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v11, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v27, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v11, v11, v27, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v12, v28 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v28, v12, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v12, v28 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v12, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v28, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v12, v12, v28, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v13, v29 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v29, v13, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v13, v29 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v13, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v29, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v14, v30 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v30, v14, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v14, v30 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v14, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v30, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc +; GFX8-NEXT: v_min_f32_e32 v1, v1, v17 +; GFX8-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v18 +; GFX8-NEXT: v_min_f32_e32 v2, v2, v18 +; GFX8-NEXT: v_mov_b32_e32 v17, 0x7fc00000 +; GFX8-NEXT: v_min_f32_e32 v18, v13, v29 +; GFX8-NEXT: v_cmp_o_f32_e64 s[28:29], v13, v29 +; GFX8-NEXT: v_writelane_b32 v31, s31, 1 +; GFX8-NEXT: v_cmp_o_f32_e64 s[6:7], v3, v19 +; GFX8-NEXT: v_min_f32_e32 v3, v3, v19 +; GFX8-NEXT: v_cmp_o_f32_e64 s[8:9], v4, v20 +; GFX8-NEXT: v_min_f32_e32 v4, v4, v20 +; GFX8-NEXT: v_cmp_o_f32_e64 s[10:11], v5, v21 +; GFX8-NEXT: v_min_f32_e32 v5, v5, v21 +; GFX8-NEXT: v_cmp_o_f32_e64 s[12:13], v6, v22 +; GFX8-NEXT: v_min_f32_e32 v6, v6, v22 +; GFX8-NEXT: v_cmp_o_f32_e64 s[14:15], v7, v23 +; GFX8-NEXT: v_min_f32_e32 v7, v7, v23 +; GFX8-NEXT: v_cmp_o_f32_e64 s[18:19], v8, v24 +; GFX8-NEXT: v_min_f32_e32 v8, v8, v24 +; GFX8-NEXT: v_cmp_o_f32_e64 s[20:21], v9, v25 +; GFX8-NEXT: v_min_f32_e32 v9, v9, v25 +; GFX8-NEXT: v_cmp_o_f32_e64 s[22:23], v10, v26 +; GFX8-NEXT: v_min_f32_e32 v10, v10, v26 +; GFX8-NEXT: v_cmp_o_f32_e64 s[24:25], v11, v27 +; GFX8-NEXT: v_min_f32_e32 v11, v11, v27 +; GFX8-NEXT: v_cmp_o_f32_e64 s[26:27], v12, v28 +; GFX8-NEXT: v_min_f32_e32 v12, v12, v28 +; GFX8-NEXT: v_min_f32_e32 v19, v14, v30 +; GFX8-NEXT: v_cmp_o_f32_e64 s[30:31], v14, v30 +; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v13, v17, v18, s[28:29] +; GFX8-NEXT: v_cndmask_b32_e64 v0, v17, v0, s[16:17] +; GFX8-NEXT: v_cndmask_b32_e64 v2, v17, v2, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v17, v3, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v4, v17, v4, s[8:9] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[10:11] +; GFX8-NEXT: v_cndmask_b32_e64 v6, v17, v6, s[12:13] +; GFX8-NEXT: v_cndmask_b32_e64 v7, v17, v7, s[14:15] +; GFX8-NEXT: v_cndmask_b32_e64 v8, v17, v8, s[18:19] +; GFX8-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[20:21] +; GFX8-NEXT: v_cndmask_b32_e64 v10, v17, v10, s[22:23] +; GFX8-NEXT: v_cndmask_b32_e64 v11, v17, v11, s[24:25] +; GFX8-NEXT: v_cndmask_b32_e64 v12, v17, v12, s[26:27] +; GFX8-NEXT: v_cndmask_b32_e64 v14, v17, v19, s[30:31] +; GFX8-NEXT: v_readlane_b32 s31, v31, 1 +; GFX8-NEXT: v_readlane_b32 s30, v31, 0 +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_min_f32_e32 v18, v15, v16 +; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v15, v16 +; GFX8-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc +; GFX8-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload +; GFX8-NEXT: s_mov_b64 exec, s[4:5] ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v15, v17 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v17, v15, vcc -; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v15, v17 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v15, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc -; GFX8-NEXT: v_cmp_class_f32_e64 vcc, v17, 32 -; GFX8-NEXT: v_cndmask_b32_e32 v15, v15, v17, vcc -; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v16f32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v32, v16, v0, vcc -; GFX9-NEXT: v_mov_b32_e32 v31, 0x7fc00000 -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v32, v31, v32, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v16, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v1, v17 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v1, vcc +; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX9-NEXT: s_mov_b64 exec, s[4:5] +; GFX9-NEXT: v_cmp_o_f32_e64 s[16:17], v0, v16 +; GFX9-NEXT: v_min_f32_e32 v0, v0, v16 +; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 +; GFX9-NEXT: v_writelane_b32 v31, s30, 0 ; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v17 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v17, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX9-NEXT: buffer_load_dword v17, off, s[0:3], s32 -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v2, v18 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v2, v18 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v18, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v3, v19 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v3, v19 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v19, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v4, v20 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v20, v4, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v4, v20 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v20, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v20, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v5, v21 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v21, v5, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v5, v21 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v21, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v21, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v6, v22 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v6, v22 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v22, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v22, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v7, v23 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v23, v7, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v7, v23 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v23, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v8, v24 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v24, v8, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v8, v24 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v8, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v24, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v9, v25 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v25, v9, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v9, v25 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v9, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v25, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v9, v25, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v10, v26 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v10, v26 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v10, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v26, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v10, v10, v26, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v11, v27 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v27, v11, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v11, v27 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v11, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v27, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v11, v11, v27, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v12, v28 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v28, v12, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v12, v28 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v12, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v28, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v12, v12, v28, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v13, v29 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v29, v13, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v13, v29 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v13, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v29, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v14, v30 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v30, v14, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v14, v30 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v14, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v30, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc +; GFX9-NEXT: v_min_f32_e32 v1, v1, v17 +; GFX9-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v18 +; GFX9-NEXT: v_min_f32_e32 v2, v2, v18 +; GFX9-NEXT: v_mov_b32_e32 v17, 0x7fc00000 +; GFX9-NEXT: v_min_f32_e32 v18, v13, v29 +; GFX9-NEXT: v_cmp_o_f32_e64 s[28:29], v13, v29 +; GFX9-NEXT: v_writelane_b32 v31, s31, 1 +; GFX9-NEXT: v_cmp_o_f32_e64 s[6:7], v3, v19 +; GFX9-NEXT: v_min_f32_e32 v3, v3, v19 +; GFX9-NEXT: v_cmp_o_f32_e64 s[8:9], v4, v20 +; GFX9-NEXT: v_min_f32_e32 v4, v4, v20 +; GFX9-NEXT: v_cmp_o_f32_e64 s[10:11], v5, v21 +; GFX9-NEXT: v_min_f32_e32 v5, v5, v21 +; GFX9-NEXT: v_cmp_o_f32_e64 s[12:13], v6, v22 +; GFX9-NEXT: v_min_f32_e32 v6, v6, v22 +; GFX9-NEXT: v_cmp_o_f32_e64 s[14:15], v7, v23 +; GFX9-NEXT: v_min_f32_e32 v7, v7, v23 +; GFX9-NEXT: v_cmp_o_f32_e64 s[18:19], v8, v24 +; GFX9-NEXT: v_min_f32_e32 v8, v8, v24 +; GFX9-NEXT: v_cmp_o_f32_e64 s[20:21], v9, v25 +; GFX9-NEXT: v_min_f32_e32 v9, v9, v25 +; GFX9-NEXT: v_cmp_o_f32_e64 s[22:23], v10, v26 +; GFX9-NEXT: v_min_f32_e32 v10, v10, v26 +; GFX9-NEXT: v_cmp_o_f32_e64 s[24:25], v11, v27 +; GFX9-NEXT: v_min_f32_e32 v11, v11, v27 +; GFX9-NEXT: v_cmp_o_f32_e64 s[26:27], v12, v28 +; GFX9-NEXT: v_min_f32_e32 v12, v12, v28 +; GFX9-NEXT: v_min_f32_e32 v19, v14, v30 +; GFX9-NEXT: v_cmp_o_f32_e64 s[30:31], v14, v30 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v13, v17, v18, s[28:29] +; GFX9-NEXT: v_cndmask_b32_e64 v0, v17, v0, s[16:17] +; GFX9-NEXT: v_cndmask_b32_e64 v2, v17, v2, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v17, v3, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v4, v17, v4, s[8:9] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[10:11] +; GFX9-NEXT: v_cndmask_b32_e64 v6, v17, v6, s[12:13] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v17, v7, s[14:15] +; GFX9-NEXT: v_cndmask_b32_e64 v8, v17, v8, s[18:19] +; GFX9-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[20:21] +; GFX9-NEXT: v_cndmask_b32_e64 v10, v17, v10, s[22:23] +; GFX9-NEXT: v_cndmask_b32_e64 v11, v17, v11, s[24:25] +; GFX9-NEXT: v_cndmask_b32_e64 v12, v17, v12, s[26:27] +; GFX9-NEXT: v_cndmask_b32_e64 v14, v17, v19, s[30:31] +; GFX9-NEXT: v_readlane_b32 s31, v31, 1 +; GFX9-NEXT: v_readlane_b32 s30, v31, 0 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_min_f32_e32 v18, v15, v16 +; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v15, v16 +; GFX9-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc +; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload +; GFX9-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, v15, v17 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v15, vcc -; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v15, v17 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v31, v16, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v15, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc -; GFX9-NEXT: v_cmp_class_f32_e64 vcc, v17, 32 -; GFX9-NEXT: v_cndmask_b32_e32 v15, v15, v17, vcc -; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v16f32: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX940-NEXT: scratch_load_dword v31, off, s32 -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v0, v16 ; GFX940-NEXT: v_mov_b32_e32 v32, 0x7fc00000 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v33, v16, v0, vcc +; GFX940-NEXT: v_min_f32_e32 v33, v0, v16 ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v33, v32, v33, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v0, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v16, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v33 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v1, v17 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v17, v1, vcc +; GFX940-NEXT: v_min_f32_e32 v34, v1, v17 +; GFX940-NEXT: v_min_f32_e32 v35, v2, v18 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v32, v33, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v1, v17 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v1, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v17, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v1, v16, v1, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v2, v18 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc +; GFX940-NEXT: v_min_f32_e32 v36, v3, v19 +; GFX940-NEXT: v_min_f32_e32 v37, v4, v20 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v32, v34, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v2, v18 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v2, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v18, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v3, v19 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc +; GFX940-NEXT: v_min_f32_e32 v38, v5, v21 +; GFX940-NEXT: v_min_f32_e32 v39, v6, v22 +; GFX940-NEXT: v_cndmask_b32_e32 v2, v32, v35, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v3, v19 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v3, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v19, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v3, v16, v3, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v4, v20 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v20, v4, vcc +; GFX940-NEXT: v_min_f32_e32 v48, v7, v23 +; GFX940-NEXT: v_min_f32_e32 v49, v8, v24 +; GFX940-NEXT: v_cndmask_b32_e32 v3, v32, v36, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v4, v20 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v4, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v20, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v4, v20, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v5, v21 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v21, v5, vcc +; GFX940-NEXT: v_min_f32_e32 v50, v9, v25 +; GFX940-NEXT: v_min_f32_e32 v51, v10, v26 +; GFX940-NEXT: v_cndmask_b32_e32 v4, v32, v37, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v5, v21 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v5, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v21, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v5, v5, v21, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v5, v16, v5, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v6, v22 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc +; GFX940-NEXT: v_min_f32_e32 v52, v11, v27 +; GFX940-NEXT: v_min_f32_e32 v53, v12, v28 +; GFX940-NEXT: v_cndmask_b32_e32 v5, v32, v38, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v6, v22 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v6, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v22, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v6, v22, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v7, v23 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v23, v7, vcc +; GFX940-NEXT: v_min_f32_e32 v54, v13, v29 +; GFX940-NEXT: v_min_f32_e32 v55, v14, v30 +; GFX940-NEXT: v_cndmask_b32_e32 v6, v32, v39, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v7, v23 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v7, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v23, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v7, v16, v7, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v8, v24 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v24, v8, vcc +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_min_f32_e32 v16, v15, v31 +; GFX940-NEXT: v_cndmask_b32_e32 v7, v32, v48, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v8, v24 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v8, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v24, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v9, v25 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v25, v9, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v8, v32, v49, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v9, v25 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v9, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v25, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v9, v9, v25, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v9, v16, v9, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v10, v26 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v9, v32, v50, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v10, v26 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v10, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v26, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v10, v10, v26, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v11, v27 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v27, v11, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v10, v32, v51, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v11, v27 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v11, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v27, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v11, v11, v27, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v11, v16, v11, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v12, v28 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v28, v12, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v11, v32, v52, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v12, v28 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v12, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v28, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v12, v12, v28, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v13, v29 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v29, v13, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v12, v32, v53, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v13, v29 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v13, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v29, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v13, v16, v13, vcc -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v14, v30 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v30, v14, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v13, v32, v54, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v14, v30 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v14, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v30, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc -; GFX940-NEXT: s_waitcnt vmcnt(0) -; GFX940-NEXT: v_cmp_lt_f32_e32 vcc, v15, v31 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v31, v15, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v14, v32, v55, vcc ; GFX940-NEXT: v_cmp_o_f32_e32 vcc, v15, v31 ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v15, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc -; GFX940-NEXT: v_cmp_class_f32_e64 vcc, v31, 32 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v15, v15, v31, vcc -; GFX940-NEXT: v_cmp_eq_f32_e32 vcc, 0, v16 -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v15, v32, v16, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v16f32: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v16 ; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX10-NEXT: v_cndmask_b32_e32 v32, v16, v0, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v17 -; GFX10-NEXT: v_cndmask_b32_e32 v33, v17, v1, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v18 -; GFX10-NEXT: v_cndmask_b32_e32 v34, v18, v2, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v19 -; GFX10-NEXT: v_cndmask_b32_e32 v35, v19, v3, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v20 -; GFX10-NEXT: v_cndmask_b32_e32 v36, v20, v4, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v21 -; GFX10-NEXT: v_cndmask_b32_e32 v37, v21, v5, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v22 -; GFX10-NEXT: v_cndmask_b32_e32 v38, v22, v6, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v23 -; GFX10-NEXT: v_cndmask_b32_e32 v39, v23, v7, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v24 -; GFX10-NEXT: v_cndmask_b32_e32 v48, v24, v8, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v25 -; GFX10-NEXT: v_cndmask_b32_e32 v49, v25, v9, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v26 -; GFX10-NEXT: v_cndmask_b32_e32 v50, v26, v10, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v27 -; GFX10-NEXT: v_cndmask_b32_e32 v51, v27, v11, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v28 -; GFX10-NEXT: v_cndmask_b32_e32 v52, v28, v12, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v29 -; GFX10-NEXT: v_cndmask_b32_e32 v53, v29, v13, vcc_lo -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v30 -; GFX10-NEXT: v_cndmask_b32_e32 v54, v30, v14, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v32, v0, v16 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v16 -; GFX10-NEXT: v_cndmask_b32_e32 v32, 0x7fc00000, v32, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v33, v1, v17 +; GFX10-NEXT: v_min_f32_e32 v34, v2, v18 +; GFX10-NEXT: v_min_f32_e32 v35, v3, v19 +; GFX10-NEXT: v_min_f32_e32 v36, v4, v20 +; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v32, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v17 -; GFX10-NEXT: v_cndmask_b32_e32 v33, 0x7fc00000, v33, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v37, v5, v21 +; GFX10-NEXT: v_min_f32_e32 v38, v6, v22 +; GFX10-NEXT: v_min_f32_e32 v39, v7, v23 +; GFX10-NEXT: v_min_f32_e32 v48, v8, v24 +; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v33, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v18 -; GFX10-NEXT: v_cndmask_b32_e32 v34, 0x7fc00000, v34, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v49, v9, v25 +; GFX10-NEXT: v_min_f32_e32 v50, v10, v26 +; GFX10-NEXT: v_min_f32_e32 v51, v11, v27 +; GFX10-NEXT: v_min_f32_e32 v52, v12, v28 +; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v34, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v19 -; GFX10-NEXT: v_cndmask_b32_e32 v35, 0x7fc00000, v35, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v53, v13, v29 +; GFX10-NEXT: v_min_f32_e32 v54, v14, v30 +; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v35, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v4, v20 -; GFX10-NEXT: v_cndmask_b32_e32 v36, 0x7fc00000, v36, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v36, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v5, v21 -; GFX10-NEXT: v_cndmask_b32_e32 v37, 0x7fc00000, v37, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v5, 0x7fc00000, v37, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v6, v22 -; GFX10-NEXT: v_cndmask_b32_e32 v38, 0x7fc00000, v38, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v6, 0x7fc00000, v38, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v7, v23 -; GFX10-NEXT: v_cndmask_b32_e32 v39, 0x7fc00000, v39, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v7, 0x7fc00000, v39, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v8, v24 -; GFX10-NEXT: v_cndmask_b32_e32 v48, 0x7fc00000, v48, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v48, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v9, v25 -; GFX10-NEXT: v_cndmask_b32_e32 v49, 0x7fc00000, v49, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v49, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v10, v26 -; GFX10-NEXT: v_cndmask_b32_e32 v50, 0x7fc00000, v50, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v10, 0x7fc00000, v50, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v11, v27 -; GFX10-NEXT: v_cndmask_b32_e32 v51, 0x7fc00000, v51, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v11, 0x7fc00000, v51, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v12, v28 -; GFX10-NEXT: v_cndmask_b32_e32 v52, 0x7fc00000, v52, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v12, 0x7fc00000, v52, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v13, v29 -; GFX10-NEXT: v_cndmask_b32_e32 v53, 0x7fc00000, v53, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v13, 0x7fc00000, v53, vcc_lo ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v14, v30 -; GFX10-NEXT: v_cndmask_b32_e32 v54, 0x7fc00000, v54, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v33, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v34, v2, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v35, v3, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v36, v4, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v37, v5, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v38, v6, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v39, v7, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v8, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v48, v8, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v9, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v49, v9, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v10, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v11, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v12, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v12, v52, v12, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v13, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v13, v53, v13, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v14, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v14, v54, v14, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v16, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v17, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v18, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v19, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v20, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v20, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v21, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v21, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v22, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v22, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v23, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v24, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v25, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v9, v25, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v26, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v10, v10, v26, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v27, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v11, v11, v27, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v28, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v12, v12, v28, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v29, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v30, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v32 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v33 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v33, v1, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v34 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v34, v2, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v35 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v35, v3, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v36 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v36, v4, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v37, v5, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v38 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v38, v6, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v14, 0x7fc00000, v54, vcc_lo ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v31 -; GFX10-NEXT: v_cndmask_b32_e32 v16, v31, v15, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v39 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v39, v7, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v48, v8, vcc_lo +; GFX10-NEXT: v_min_f32_e32 v16, v15, v31 ; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v15, v31 -; GFX10-NEXT: v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v49 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v49, v9, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v50 -; GFX10-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v15, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v51 -; GFX10-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v52 -; GFX10-NEXT: v_cndmask_b32_e32 v12, v52, v12, vcc_lo -; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v31, 32 -; GFX10-NEXT: v_cndmask_b32_e32 v15, v15, v31, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v53 -; GFX10-NEXT: v_cndmask_b32_e32 v13, v53, v13, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v54 -; GFX10-NEXT: v_cndmask_b32_e32 v14, v54, v14, vcc_lo -; GFX10-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 -; GFX10-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v15, 0x7fc00000, v16, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v16f32: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v16 ; GFX11-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-NEXT: v_cndmask_b32_e32 v32, v16, v0, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v17 -; GFX11-NEXT: v_cndmask_b32_e32 v33, v17, v1, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v18 -; GFX11-NEXT: v_cndmask_b32_e32 v34, v18, v2, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v19 -; GFX11-NEXT: v_cndmask_b32_e32 v35, v19, v3, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v20 -; GFX11-NEXT: v_cndmask_b32_e32 v36, v20, v4, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v21 -; GFX11-NEXT: v_cndmask_b32_e32 v37, v21, v5, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v22 -; GFX11-NEXT: v_cndmask_b32_e32 v38, v22, v6, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v23 -; GFX11-NEXT: v_cndmask_b32_e32 v39, v23, v7, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v24 -; GFX11-NEXT: v_cndmask_b32_e32 v48, v24, v8, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v25 -; GFX11-NEXT: v_cndmask_b32_e32 v49, v25, v9, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v26 -; GFX11-NEXT: v_cndmask_b32_e32 v50, v26, v10, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v27 -; GFX11-NEXT: v_cndmask_b32_e32 v51, v27, v11, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v28 -; GFX11-NEXT: v_cndmask_b32_e32 v52, v28, v12, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v29 -; GFX11-NEXT: v_cndmask_b32_e32 v53, v29, v13, vcc_lo -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v30 -; GFX11-NEXT: v_cndmask_b32_e32 v54, v30, v14, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v32, v0, v16 :: v_dual_min_f32 v33, v1, v17 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v16 -; GFX11-NEXT: v_cndmask_b32_e32 v32, 0x7fc00000, v32, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v34, v2, v18 :: v_dual_min_f32 v35, v3, v19 +; GFX11-NEXT: v_dual_min_f32 v36, v4, v20 :: v_dual_min_f32 v37, v5, v21 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7fc00000, v32, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v17 -; GFX11-NEXT: v_cndmask_b32_e32 v33, 0x7fc00000, v33, vcc_lo +; GFX11-NEXT: v_min_f32_e32 v54, v14, v30 +; GFX11-NEXT: v_dual_min_f32 v38, v6, v22 :: v_dual_min_f32 v39, v7, v23 +; GFX11-NEXT: v_dual_min_f32 v48, v8, v24 :: v_dual_min_f32 v49, v9, v25 +; GFX11-NEXT: v_cndmask_b32_e32 v1, 0x7fc00000, v33, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v2, v18 -; GFX11-NEXT: v_cndmask_b32_e32 v34, 0x7fc00000, v34, vcc_lo +; GFX11-NEXT: v_dual_min_f32 v50, v10, v26 :: v_dual_min_f32 v51, v11, v27 +; GFX11-NEXT: v_dual_min_f32 v52, v12, v28 :: v_dual_min_f32 v53, v13, v29 +; GFX11-NEXT: v_cndmask_b32_e32 v2, 0x7fc00000, v34, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v19 -; GFX11-NEXT: v_cndmask_b32_e32 v35, 0x7fc00000, v35, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v3, 0x7fc00000, v35, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v4, v20 -; GFX11-NEXT: v_cndmask_b32_e32 v36, 0x7fc00000, v36, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v4, 0x7fc00000, v36, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v5, v21 -; GFX11-NEXT: v_cndmask_b32_e32 v37, 0x7fc00000, v37, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v5, 0x7fc00000, v37, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v6, v22 -; GFX11-NEXT: v_cndmask_b32_e32 v38, 0x7fc00000, v38, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v6, 0x7fc00000, v38, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v7, v23 -; GFX11-NEXT: v_cndmask_b32_e32 v39, 0x7fc00000, v39, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v7, 0x7fc00000, v39, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v8, v24 -; GFX11-NEXT: v_cndmask_b32_e32 v48, 0x7fc00000, v48, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v8, 0x7fc00000, v48, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v9, v25 -; GFX11-NEXT: v_cndmask_b32_e32 v49, 0x7fc00000, v49, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v9, 0x7fc00000, v49, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v10, v26 -; GFX11-NEXT: v_cndmask_b32_e32 v50, 0x7fc00000, v50, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v10, 0x7fc00000, v50, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v11, v27 -; GFX11-NEXT: v_cndmask_b32_e32 v51, 0x7fc00000, v51, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v11, 0x7fc00000, v51, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v12, v28 -; GFX11-NEXT: v_cndmask_b32_e32 v52, 0x7fc00000, v52, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v12, 0x7fc00000, v52, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v13, v29 -; GFX11-NEXT: v_cndmask_b32_e32 v53, 0x7fc00000, v53, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v13, 0x7fc00000, v53, vcc_lo ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v14, v30 -; GFX11-NEXT: v_cndmask_b32_e32 v54, 0x7fc00000, v54, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v1, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v33, v1, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v2, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v34, v2, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v3, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v35, v3, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v4, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v36, v4, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v5, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v37, v5, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v6, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v38, v6, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v7, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v39, v7, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v8, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v48, v8, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v9, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v49, v9, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v10, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v11, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v12, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v12, v52, v12, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v13, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v13, v53, v13, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v14, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v14, v54, v14, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v16, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v17, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v18, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v19, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v20, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v20, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v21, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v5, v21, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v22, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v6, v22, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v23, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v24, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v25, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v9, v25, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v26, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v10, v10, v26, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v27, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v11, v11, v27, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v28, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v12, v12, v28, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v29, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v30, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v32 -; GFX11-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v33 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v33, v1, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v34 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v34, v2, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v35 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v35, v3, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v36 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v36, v4, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v37, v5, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v38 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v38, v6, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v14, 0x7fc00000, v54, vcc_lo ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v31 -; GFX11-NEXT: v_cndmask_b32_e32 v16, v31, v15, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v39 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v39, v7, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v48, v8, vcc_lo +; GFX11-NEXT: v_min_f32_e32 v16, v15, v31 ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v15, v31 -; GFX11-NEXT: v_cndmask_b32_e32 v16, 0x7fc00000, v16, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v49 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v49, v9, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v50 -; GFX11-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v15, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v51 -; GFX11-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v52 -; GFX11-NEXT: v_cndmask_b32_e32 v12, v52, v12, vcc_lo -; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v31, 32 -; GFX11-NEXT: v_cndmask_b32_e32 v15, v15, v31, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v53 -; GFX11-NEXT: v_cndmask_b32_e32 v13, v53, v13, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v54 -; GFX11-NEXT: v_cndmask_b32_e32 v14, v54, v14, vcc_lo -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 -; GFX11-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-NEXT: v_cndmask_b32_e32 v15, 0x7fc00000, v16, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v16f32: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll index 7013c60bada5..37fe2e958e62 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll @@ -530,221 +530,86 @@ define <2 x double> @v_minimum_v2f64(<2 x double> %src0, <2 x double> %src1) { ; GFX7-LABEL: v_minimum_v2f64: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5] -; GFX7-NEXT: v_mov_b32_e32 v10, 0x7ff80000 -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[0:1], 32 -; GFX7-NEXT: v_cmp_class_f64_e64 s[8:9], v[4:5], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v9, v10, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[4:5] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[8:9] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v11, v7, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v11, v10, v11, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v10, 0, v4, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 32 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[12:13] +; GFX7-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX7-NEXT: v_min_f64 v[4:5], v[2:3], v[6:7] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7] +; GFX7-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[4:5] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v2f64: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5] -; GFX8-NEXT: v_mov_b32_e32 v10, 0x7ff80000 -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[0:1], 32 -; GFX8-NEXT: v_cmp_class_f64_e64 s[8:9], v[4:5], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v9, v10, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[4:5] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[8:9] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v11, v7, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v11, v10, v11, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v10, 0, v4, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 32 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[12:13] +; GFX8-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX8-NEXT: v_min_f64 v[4:5], v[2:3], v[6:7] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7] +; GFX8-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[4:5] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v2f64: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5] -; GFX9-NEXT: v_mov_b32_e32 v10, 0x7ff80000 -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[0:1], 32 -; GFX9-NEXT: v_cmp_class_f64_e64 s[8:9], v[4:5], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v9, v10, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[4:5] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[8:9] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v11, v7, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v11, v10, v11, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, v4, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 32 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[12:13] +; GFX9-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX9-NEXT: v_min_f64 v[4:5], v[2:3], v[6:7] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7] +; GFX9-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[4:5] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v2f64: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX940-NEXT: v_mov_b32_e32 v10, 0x7ff80000 -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[4:5] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc +; GFX940-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX940-NEXT: v_min_f64 v[4:5], v[2:3], v[6:7] ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v9, v10, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[4:5], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v7, v3, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[6:7] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v10, v4, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, 0, v4, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[6:7], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[4:5] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v4, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[2:3] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX940-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v8, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[6:7] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v2, v4, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v8, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v2f64: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[2:3], v[6:7] -; GFX10-NEXT: v_cmp_o_f64_e64 s5, v[0:1], v[4:5] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[2:3], v[6:7] -; GFX10-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v10, v7, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v12, v4, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v13, v6, v2, s4 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[2:3], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v9, 0x7ff80000, v8, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v10, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, v12, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, v13, s6 -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[4:5], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[6:7], 32 -; GFX10-NEXT: v_cmp_eq_f64_e64 s7, 0, v[8:9] -; GFX10-NEXT: v_cmp_eq_f64_e64 s8, 0, v[10:11] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v7, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v8, v0, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v10, v2, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v9, v1, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, v3, s8 +; GFX10-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5] +; GFX10-NEXT: v_min_f64 v[4:5], v[2:3], v[6:7] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[6:7] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v4, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, 0x7ff80000, s4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v2f64: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX11-NEXT: v_cmp_o_f64_e64 s1, v[0:1], v[4:5] -; GFX11-NEXT: v_cmp_lt_f64_e64 s0, v[2:3], v[6:7] -; GFX11-NEXT: v_cmp_o_f64_e64 s2, v[2:3], v[6:7] -; GFX11-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v10, v7, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v13, v6, v2, s0 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v9, 0x7ff80000, v8, s1 -; GFX11-NEXT: v_cndmask_b32_e32 v12, v4, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v10, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v10, 0, v13, s2 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[6:7], 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_cmp_eq_f64_e64 s4, 0, v[10:11] -; GFX11-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v8, 0, v12, s1 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[4:5], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v6, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v7, s2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cmp_eq_f64_e64 s3, 0, v[8:9] -; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v5, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v0, v8, v0, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v9, v1, s3 +; GFX11-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5] +; GFX11-NEXT: v_min_f64 v[4:5], v[2:3], v[6:7] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[6:7] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v5, 0x7ff80000, s0 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v2f64: @@ -765,182 +630,43 @@ define <2 x double> @v_minimum_v2f64__nnan(<2 x double> %src0, <2 x double> %src ; GFX7-LABEL: v_minimum_v2f64__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[4:5], 32 -; GFX7-NEXT: v_cmp_class_f64_e64 s[10:11], v[6:7], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v11, v7, v3, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v10, v6, v2, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[2:3], 32 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[8:9] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[12:13] +; GFX7-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX7-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v2f64__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[4:5], 32 -; GFX8-NEXT: v_cmp_class_f64_e64 s[10:11], v[6:7], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v11, v7, v3, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v10, v6, v2, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[2:3], 32 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[8:9] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[12:13] +; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v2f64__nnan: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[4:5], 32 -; GFX9-NEXT: v_cmp_class_f64_e64 s[10:11], v[6:7], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v11, v7, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v10, v6, v2, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[2:3], 32 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[8:9] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[12:13] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX9-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v2f64__nnan: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[4:5], 32 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v5, v7, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v2, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[6:7], 32 -; GFX940-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[0:1] -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[4:5] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[0:1] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v2, v4, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[2:3] +; GFX940-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX940-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7] ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v2f64__nnan: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[2:3], v[6:7] -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[4:5], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[6:7], 32 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v5, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v11, v7, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v10, v6, v2, s4 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[2:3], 32 -; GFX10-NEXT: v_cmp_eq_f64_e64 s7, 0, v[8:9] -; GFX10-NEXT: v_cmp_eq_f64_e64 s8, 0, v[10:11] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v7, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v8, v0, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v10, v2, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v9, v1, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, v3, s8 +; GFX10-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX10-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v2f64__nnan: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX11-NEXT: v_cmp_lt_f64_e64 s0, v[2:3], v[6:7] -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[4:5], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[6:7], 32 -; GFX11-NEXT: v_dual_cndmask_b32 v9, v5, v1 :: v_dual_cndmask_b32 v8, v4, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v11, v7, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v6, v2, s0 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 32 -; GFX11-NEXT: v_cmp_eq_f64_e64 s3, 0, v[8:9] -; GFX11-NEXT: v_cmp_eq_f64_e64 s4, 0, v[10:11] -; GFX11-NEXT: v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v6, s2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v7, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v8, v0, s3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v1, v9, v1, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 +; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v2f64__nnan: @@ -961,111 +687,86 @@ define <2 x double> @v_minimum_v2f64__nsz(<2 x double> %src0, <2 x double> %src1 ; GFX7-LABEL: v_minimum_v2f64__nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[6:7], v[2:3], v[6:7] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[2:3], v[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v4, v6, v2, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX7-NEXT: v_mov_b32_e32 v5, 0x7ff80000 -; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v0, 0, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v5, v1, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v2, 0, v4, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[8:9] +; GFX7-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX7-NEXT: v_min_f64 v[4:5], v[2:3], v[6:7] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7] +; GFX7-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[4:5] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v2f64__nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[6:7], v[2:3], v[6:7] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[2:3], v[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v2, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8-NEXT: v_mov_b32_e32 v5, 0x7ff80000 -; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v5, v1, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, v4, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[8:9] +; GFX8-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX8-NEXT: v_min_f64 v[4:5], v[2:3], v[6:7] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7] +; GFX8-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[4:5] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v2f64__nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[6:7], v[2:3], v[6:7] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[4:5] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[2:3], v[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v4, v6, v2, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000 -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v5, v1, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, v4, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[8:9] +; GFX9-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX9-NEXT: v_min_f64 v[4:5], v[2:3], v[6:7] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[6:7] +; GFX9-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[4:5] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v2f64__nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[4:5] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX940-NEXT: v_mov_b32_e32 v4, 0x7ff80000 -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7] -; GFX940-NEXT: v_cndmask_b32_e64 v0, 0, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v4, v1, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v2, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[6:7] -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX940-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5] +; GFX940-NEXT: v_min_f64 v[4:5], v[2:3], v[6:7] ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v2, 0, v5, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v4, v3, s[0:1] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc +; GFX940-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v8, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[6:7] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v2, v4, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v8, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v2f64__nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[2:3], v[6:7] -; GFX10-NEXT: v_cmp_o_f64_e64 s5, v[0:1], v[4:5] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[2:3], v[6:7] -; GFX10-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v9, v6, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, v8, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, v9, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v1, 0x7ff80000, v1, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v3, s6 +; GFX10-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5] +; GFX10-NEXT: v_min_f64 v[4:5], v[2:3], v[6:7] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[6:7] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v4, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, 0x7ff80000, s4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v2f64__nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX11-NEXT: v_cmp_lt_f64_e64 s0, v[2:3], v[6:7] -; GFX11-NEXT: v_cmp_o_f64_e64 s1, v[0:1], v[4:5] -; GFX11-NEXT: v_cmp_o_f64_e64 s2, v[2:3], v[6:7] -; GFX11-NEXT: v_dual_cndmask_b32 v8, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX11-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[4:5] +; GFX11-NEXT: v_min_f64 v[4:5], v[2:3], v[6:7] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[6:7] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v9, v6, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, v8, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v1, 0x7ff80000, v1, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, v9, s2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v3, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v0, v8, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v9, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v5, 0x7ff80000, s0 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v2f64__nsz: @@ -1086,69 +787,43 @@ define <2 x double> @v_minimum_v2f64__nnan_nsz(<2 x double> %src0, <2 x double> ; GFX7-LABEL: v_minimum_v2f64__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] +; GFX7-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX7-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v2f64__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] +; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v2f64__nnan_nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX9-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v2f64__nnan_nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[4:5] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[6:7] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX940-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX940-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7] ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v2f64__nnan_nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[2:3], v[6:7] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v3, s4 +; GFX10-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX10-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v2f64__nnan_nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[4:5] -; GFX11-NEXT: v_cmp_lt_f64_e64 s0, v[2:3], v[6:7] -; GFX11-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v6, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0 +; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v2f64__nnan_nsz: @@ -1170,61 +845,20 @@ define void @s_minimum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1) ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_mov_b32_e32 v0, s10 +; GFX7-NEXT: v_mov_b32_e32 v4, s8 ; GFX7-NEXT: v_mov_b32_e32 v1, s11 -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, s[6:7], v[0:1] -; GFX7-NEXT: v_cmp_o_f64_e64 s[12:13], s[6:7], v[0:1] -; GFX7-NEXT: v_cmp_class_f64_e64 s[18:19], s[10:11], 32 -; GFX7-NEXT: v_mov_b32_e32 v0, s8 -; GFX7-NEXT: v_mov_b32_e32 v1, s9 -; GFX7-NEXT: s_and_b64 s[14:15], vcc, exec -; GFX7-NEXT: s_cselect_b32 s16, s7, s11 -; GFX7-NEXT: s_and_b64 s[14:15], s[12:13], exec -; GFX7-NEXT: s_cselect_b32 s15, s16, 0x7ff80000 -; GFX7-NEXT: s_and_b64 s[16:17], vcc, exec -; GFX7-NEXT: s_cselect_b32 s14, s6, s10 -; GFX7-NEXT: v_cmp_class_f64_e64 s[16:17], s[6:7], 32 -; GFX7-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX7-NEXT: s_cselect_b32 s14, s14, 0 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[20:21], s[14:15], 0 -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, s[4:5], v[0:1] -; GFX7-NEXT: s_and_b64 s[12:13], s[16:17], exec -; GFX7-NEXT: s_cselect_b32 s7, s7, s15 -; GFX7-NEXT: s_and_b64 s[12:13], s[18:19], exec -; GFX7-NEXT: s_cselect_b32 s7, s11, s7 -; GFX7-NEXT: s_and_b64 s[12:13], s[20:21], exec -; GFX7-NEXT: s_cselect_b32 s7, s7, s15 -; GFX7-NEXT: s_and_b64 s[12:13], s[16:17], exec -; GFX7-NEXT: v_cmp_o_f64_e64 s[12:13], s[4:5], v[0:1] -; GFX7-NEXT: s_cselect_b32 s6, s6, s14 -; GFX7-NEXT: s_and_b64 s[16:17], s[18:19], exec -; GFX7-NEXT: s_cselect_b32 s6, s10, s6 -; GFX7-NEXT: s_and_b64 s[10:11], s[20:21], exec -; GFX7-NEXT: s_cselect_b32 s6, s6, s14 -; GFX7-NEXT: s_and_b64 s[10:11], vcc, exec -; GFX7-NEXT: s_cselect_b32 s14, s5, s9 -; GFX7-NEXT: s_and_b64 s[10:11], s[12:13], exec -; GFX7-NEXT: s_cselect_b32 s11, s14, 0x7ff80000 -; GFX7-NEXT: s_and_b64 s[14:15], vcc, exec -; GFX7-NEXT: s_cselect_b32 s10, s4, s8 -; GFX7-NEXT: v_cmp_class_f64_e64 s[14:15], s[4:5], 32 -; GFX7-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX7-NEXT: v_cmp_class_f64_e64 s[12:13], s[8:9], 32 -; GFX7-NEXT: s_cselect_b32 s10, s10, 0 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[16:17], s[10:11], 0 -; GFX7-NEXT: s_and_b64 s[18:19], s[14:15], exec -; GFX7-NEXT: s_cselect_b32 s5, s5, s11 -; GFX7-NEXT: s_and_b64 s[18:19], s[12:13], exec -; GFX7-NEXT: s_cselect_b32 s5, s9, s5 -; GFX7-NEXT: s_and_b64 s[18:19], s[16:17], exec -; GFX7-NEXT: s_cselect_b32 s5, s5, s11 -; GFX7-NEXT: s_and_b64 s[14:15], s[14:15], exec -; GFX7-NEXT: s_cselect_b32 s4, s4, s10 -; GFX7-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX7-NEXT: s_cselect_b32 s4, s8, s4 -; GFX7-NEXT: s_and_b64 s[8:9], s[16:17], exec -; GFX7-NEXT: s_cselect_b32 s4, s4, s10 +; GFX7-NEXT: v_mov_b32_e32 v5, s9 +; GFX7-NEXT: v_min_f64 v[2:3], s[6:7], v[0:1] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, s[6:7], v[0:1] +; GFX7-NEXT: v_min_f64 v[0:1], s[4:5], v[4:5] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], s[4:5], v[4:5] +; GFX7-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v6, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[4:5] ; GFX7-NEXT: ;;#ASMSTART -; GFX7-NEXT: ; use s[4:7] +; GFX7-NEXT: ; use v[0:3] ; GFX7-NEXT: ;;#ASMEND ; GFX7-NEXT: s_setpc_b64 s[30:31] ; @@ -1232,61 +866,20 @@ define void @s_minimum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1) ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v0, s10 +; GFX8-NEXT: v_mov_b32_e32 v4, s8 ; GFX8-NEXT: v_mov_b32_e32 v1, s11 -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, s[6:7], v[0:1] -; GFX8-NEXT: v_cmp_o_f64_e64 s[12:13], s[6:7], v[0:1] -; GFX8-NEXT: v_cmp_class_f64_e64 s[18:19], s[10:11], 32 -; GFX8-NEXT: v_mov_b32_e32 v0, s8 -; GFX8-NEXT: v_mov_b32_e32 v1, s9 -; GFX8-NEXT: s_and_b64 s[14:15], vcc, exec -; GFX8-NEXT: s_cselect_b32 s16, s7, s11 -; GFX8-NEXT: s_and_b64 s[14:15], s[12:13], exec -; GFX8-NEXT: s_cselect_b32 s15, s16, 0x7ff80000 -; GFX8-NEXT: s_and_b64 s[16:17], vcc, exec -; GFX8-NEXT: s_cselect_b32 s14, s6, s10 -; GFX8-NEXT: v_cmp_class_f64_e64 s[16:17], s[6:7], 32 -; GFX8-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX8-NEXT: s_cselect_b32 s14, s14, 0 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[20:21], s[14:15], 0 -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, s[4:5], v[0:1] -; GFX8-NEXT: s_and_b64 s[12:13], s[16:17], exec -; GFX8-NEXT: s_cselect_b32 s7, s7, s15 -; GFX8-NEXT: s_and_b64 s[12:13], s[18:19], exec -; GFX8-NEXT: s_cselect_b32 s7, s11, s7 -; GFX8-NEXT: s_and_b64 s[12:13], s[20:21], exec -; GFX8-NEXT: s_cselect_b32 s7, s7, s15 -; GFX8-NEXT: s_and_b64 s[12:13], s[16:17], exec -; GFX8-NEXT: v_cmp_o_f64_e64 s[12:13], s[4:5], v[0:1] -; GFX8-NEXT: s_cselect_b32 s6, s6, s14 -; GFX8-NEXT: s_and_b64 s[16:17], s[18:19], exec -; GFX8-NEXT: s_cselect_b32 s6, s10, s6 -; GFX8-NEXT: s_and_b64 s[10:11], s[20:21], exec -; GFX8-NEXT: s_cselect_b32 s6, s6, s14 -; GFX8-NEXT: s_and_b64 s[10:11], vcc, exec -; GFX8-NEXT: s_cselect_b32 s14, s5, s9 -; GFX8-NEXT: s_and_b64 s[10:11], s[12:13], exec -; GFX8-NEXT: s_cselect_b32 s11, s14, 0x7ff80000 -; GFX8-NEXT: s_and_b64 s[14:15], vcc, exec -; GFX8-NEXT: s_cselect_b32 s10, s4, s8 -; GFX8-NEXT: v_cmp_class_f64_e64 s[14:15], s[4:5], 32 -; GFX8-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX8-NEXT: v_cmp_class_f64_e64 s[12:13], s[8:9], 32 -; GFX8-NEXT: s_cselect_b32 s10, s10, 0 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[16:17], s[10:11], 0 -; GFX8-NEXT: s_and_b64 s[18:19], s[14:15], exec -; GFX8-NEXT: s_cselect_b32 s5, s5, s11 -; GFX8-NEXT: s_and_b64 s[18:19], s[12:13], exec -; GFX8-NEXT: s_cselect_b32 s5, s9, s5 -; GFX8-NEXT: s_and_b64 s[18:19], s[16:17], exec -; GFX8-NEXT: s_cselect_b32 s5, s5, s11 -; GFX8-NEXT: s_and_b64 s[14:15], s[14:15], exec -; GFX8-NEXT: s_cselect_b32 s4, s4, s10 -; GFX8-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX8-NEXT: s_cselect_b32 s4, s8, s4 -; GFX8-NEXT: s_and_b64 s[8:9], s[16:17], exec -; GFX8-NEXT: s_cselect_b32 s4, s4, s10 +; GFX8-NEXT: v_mov_b32_e32 v5, s9 +; GFX8-NEXT: v_min_f64 v[2:3], s[6:7], v[0:1] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, s[6:7], v[0:1] +; GFX8-NEXT: v_min_f64 v[0:1], s[4:5], v[4:5] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], s[4:5], v[4:5] +; GFX8-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v6, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[4:5] ; GFX8-NEXT: ;;#ASMSTART -; GFX8-NEXT: ; use s[4:7] +; GFX8-NEXT: ; use v[0:3] ; GFX8-NEXT: ;;#ASMEND ; GFX8-NEXT: s_setpc_b64 s[30:31] ; @@ -1294,61 +887,20 @@ define void @s_minimum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1) ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v0, s10 +; GFX9-NEXT: v_mov_b32_e32 v4, s8 ; GFX9-NEXT: v_mov_b32_e32 v1, s11 -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, s[6:7], v[0:1] -; GFX9-NEXT: v_cmp_o_f64_e64 s[12:13], s[6:7], v[0:1] -; GFX9-NEXT: v_cmp_class_f64_e64 s[18:19], s[10:11], 32 -; GFX9-NEXT: v_mov_b32_e32 v0, s8 -; GFX9-NEXT: v_mov_b32_e32 v1, s9 -; GFX9-NEXT: s_and_b64 s[14:15], vcc, exec -; GFX9-NEXT: s_cselect_b32 s16, s7, s11 -; GFX9-NEXT: s_and_b64 s[14:15], s[12:13], exec -; GFX9-NEXT: s_cselect_b32 s15, s16, 0x7ff80000 -; GFX9-NEXT: s_and_b64 s[16:17], vcc, exec -; GFX9-NEXT: s_cselect_b32 s14, s6, s10 -; GFX9-NEXT: v_cmp_class_f64_e64 s[16:17], s[6:7], 32 -; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX9-NEXT: s_cselect_b32 s14, s14, 0 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[20:21], s[14:15], 0 -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, s[4:5], v[0:1] -; GFX9-NEXT: s_and_b64 s[12:13], s[16:17], exec -; GFX9-NEXT: s_cselect_b32 s7, s7, s15 -; GFX9-NEXT: s_and_b64 s[12:13], s[18:19], exec -; GFX9-NEXT: s_cselect_b32 s7, s11, s7 -; GFX9-NEXT: s_and_b64 s[12:13], s[20:21], exec -; GFX9-NEXT: s_cselect_b32 s7, s7, s15 -; GFX9-NEXT: s_and_b64 s[12:13], s[16:17], exec -; GFX9-NEXT: v_cmp_o_f64_e64 s[12:13], s[4:5], v[0:1] -; GFX9-NEXT: s_cselect_b32 s6, s6, s14 -; GFX9-NEXT: s_and_b64 s[16:17], s[18:19], exec -; GFX9-NEXT: s_cselect_b32 s6, s10, s6 -; GFX9-NEXT: s_and_b64 s[10:11], s[20:21], exec -; GFX9-NEXT: s_cselect_b32 s6, s6, s14 -; GFX9-NEXT: s_and_b64 s[10:11], vcc, exec -; GFX9-NEXT: s_cselect_b32 s14, s5, s9 -; GFX9-NEXT: s_and_b64 s[10:11], s[12:13], exec -; GFX9-NEXT: s_cselect_b32 s11, s14, 0x7ff80000 -; GFX9-NEXT: s_and_b64 s[14:15], vcc, exec -; GFX9-NEXT: s_cselect_b32 s10, s4, s8 -; GFX9-NEXT: v_cmp_class_f64_e64 s[14:15], s[4:5], 32 -; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX9-NEXT: v_cmp_class_f64_e64 s[12:13], s[8:9], 32 -; GFX9-NEXT: s_cselect_b32 s10, s10, 0 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[16:17], s[10:11], 0 -; GFX9-NEXT: s_and_b64 s[18:19], s[14:15], exec -; GFX9-NEXT: s_cselect_b32 s5, s5, s11 -; GFX9-NEXT: s_and_b64 s[18:19], s[12:13], exec -; GFX9-NEXT: s_cselect_b32 s5, s9, s5 -; GFX9-NEXT: s_and_b64 s[18:19], s[16:17], exec -; GFX9-NEXT: s_cselect_b32 s5, s5, s11 -; GFX9-NEXT: s_and_b64 s[14:15], s[14:15], exec -; GFX9-NEXT: s_cselect_b32 s4, s4, s10 -; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX9-NEXT: s_cselect_b32 s4, s8, s4 -; GFX9-NEXT: s_and_b64 s[8:9], s[16:17], exec -; GFX9-NEXT: s_cselect_b32 s4, s4, s10 +; GFX9-NEXT: v_mov_b32_e32 v5, s9 +; GFX9-NEXT: v_min_f64 v[2:3], s[6:7], v[0:1] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, s[6:7], v[0:1] +; GFX9-NEXT: v_min_f64 v[0:1], s[4:5], v[4:5] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], s[4:5], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v6, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[4:5] ; GFX9-NEXT: ;;#ASMSTART -; GFX9-NEXT: ; use s[4:7] +; GFX9-NEXT: ; use v[0:3] ; GFX9-NEXT: ;;#ASMEND ; GFX9-NEXT: s_setpc_b64 s[30:31] ; @@ -1356,179 +908,52 @@ define void @s_minimum_v2f64(<2 x double> inreg %src0, <2 x double> inreg %src1) ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX940-NEXT: v_mov_b64_e32 v[0:1], s[6:7] -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, s[2:3], v[0:1] -; GFX940-NEXT: s_and_b64 s[8:9], vcc, exec -; GFX940-NEXT: v_cmp_o_f64_e64 s[8:9], s[2:3], v[0:1] -; GFX940-NEXT: s_cselect_b32 s12, s3, s7 -; GFX940-NEXT: s_and_b64 s[10:11], s[8:9], exec -; GFX940-NEXT: s_cselect_b32 s11, s12, 0x7ff80000 -; GFX940-NEXT: s_and_b64 s[12:13], vcc, exec -; GFX940-NEXT: s_cselect_b32 s10, s2, s6 -; GFX940-NEXT: s_and_b64 s[8:9], s[8:9], exec -; GFX940-NEXT: v_cmp_class_f64_e64 s[12:13], s[2:3], 32 -; GFX940-NEXT: s_cselect_b32 s10, s10, 0 -; GFX940-NEXT: s_and_b64 s[14:15], s[12:13], exec -; GFX940-NEXT: v_cmp_class_f64_e64 s[14:15], s[6:7], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[8:9], s[10:11], 0 -; GFX940-NEXT: s_cselect_b32 s3, s3, s11 -; GFX940-NEXT: s_and_b64 s[16:17], s[14:15], exec -; GFX940-NEXT: s_cselect_b32 s3, s7, s3 -; GFX940-NEXT: s_and_b64 s[16:17], s[8:9], exec -; GFX940-NEXT: s_cselect_b32 s7, s3, s11 -; GFX940-NEXT: s_and_b64 s[12:13], s[12:13], exec -; GFX940-NEXT: s_cselect_b32 s11, s2, s10 -; GFX940-NEXT: s_and_b64 s[2:3], s[14:15], exec +; GFX940-NEXT: v_min_f64 v[2:3], s[2:3], v[0:1] +; GFX940-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, s[2:3], v[0:1] ; GFX940-NEXT: v_mov_b64_e32 v[0:1], s[4:5] -; GFX940-NEXT: s_cselect_b32 s6, s6, s11 -; GFX940-NEXT: s_and_b64 s[2:3], s[8:9], exec -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1] -; GFX940-NEXT: s_cselect_b32 s6, s6, s10 -; GFX940-NEXT: s_and_b64 s[2:3], vcc, exec -; GFX940-NEXT: v_cmp_o_f64_e64 s[2:3], s[0:1], v[0:1] -; GFX940-NEXT: s_cselect_b32 s10, s1, s5 -; GFX940-NEXT: s_and_b64 s[8:9], s[2:3], exec -; GFX940-NEXT: s_cselect_b32 s9, s10, 0x7ff80000 -; GFX940-NEXT: s_and_b64 s[10:11], vcc, exec -; GFX940-NEXT: s_cselect_b32 s8, s0, s4 -; GFX940-NEXT: s_and_b64 s[2:3], s[2:3], exec -; GFX940-NEXT: v_cmp_class_f64_e64 s[10:11], s[0:1], 32 -; GFX940-NEXT: s_cselect_b32 s8, s8, 0 -; GFX940-NEXT: s_and_b64 s[12:13], s[10:11], exec -; GFX940-NEXT: v_cmp_class_f64_e64 s[12:13], s[4:5], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], s[8:9], 0 -; GFX940-NEXT: s_cselect_b32 s1, s1, s9 -; GFX940-NEXT: s_and_b64 s[14:15], s[12:13], exec -; GFX940-NEXT: s_cselect_b32 s1, s5, s1 -; GFX940-NEXT: s_and_b64 s[14:15], s[2:3], exec -; GFX940-NEXT: s_cselect_b32 s5, s1, s9 -; GFX940-NEXT: s_and_b64 s[10:11], s[10:11], exec -; GFX940-NEXT: s_cselect_b32 s9, s0, s8 -; GFX940-NEXT: s_and_b64 s[0:1], s[12:13], exec -; GFX940-NEXT: s_cselect_b32 s4, s4, s9 -; GFX940-NEXT: s_and_b64 s[0:1], s[2:3], exec -; GFX940-NEXT: s_cselect_b32 s4, s4, s8 +; GFX940-NEXT: v_min_f64 v[4:5], s[0:1], v[0:1] +; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc +; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, s[0:1], v[0:1] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc +; GFX940-NEXT: v_cndmask_b32_e64 v0, v4, 0, vcc ; GFX940-NEXT: ;;#ASMSTART -; GFX940-NEXT: ; use s[4:7] +; GFX940-NEXT: ; use v[0:3] ; GFX940-NEXT: ;;#ASMEND ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: s_minimum_v2f64: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e64 s12, s[6:7], s[10:11] -; GFX10-NEXT: v_cmp_o_f64_e64 s14, s[6:7], s[10:11] -; GFX10-NEXT: v_cmp_class_f64_e64 s15, s[6:7], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s16, s[10:11], 32 -; GFX10-NEXT: v_cmp_o_f64_e64 s18, s[4:5], s[8:9] -; GFX10-NEXT: v_cmp_class_f64_e64 s19, s[4:5], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s20, s[8:9], 32 -; GFX10-NEXT: s_and_b32 s13, s12, exec_lo -; GFX10-NEXT: s_cselect_b32 s13, s7, s11 -; GFX10-NEXT: s_and_b32 s17, s14, exec_lo -; GFX10-NEXT: s_cselect_b32 s13, s13, 0x7ff80000 -; GFX10-NEXT: s_and_b32 s12, s12, exec_lo -; GFX10-NEXT: s_cselect_b32 s12, s6, s10 -; GFX10-NEXT: s_and_b32 s14, s14, exec_lo -; GFX10-NEXT: s_cselect_b32 s12, s12, 0 -; GFX10-NEXT: v_cmp_lt_f64_e64 s17, s[4:5], s[8:9] -; GFX10-NEXT: v_cmp_eq_f64_e64 s14, s[12:13], 0 -; GFX10-NEXT: s_and_b32 s21, s15, exec_lo -; GFX10-NEXT: s_cselect_b32 s7, s7, s13 -; GFX10-NEXT: s_and_b32 s21, s16, exec_lo -; GFX10-NEXT: s_cselect_b32 s7, s11, s7 -; GFX10-NEXT: s_and_b32 s11, s14, exec_lo -; GFX10-NEXT: s_cselect_b32 s7, s7, s13 -; GFX10-NEXT: s_and_b32 s11, s15, exec_lo -; GFX10-NEXT: s_cselect_b32 s6, s6, s12 -; GFX10-NEXT: s_and_b32 s11, s16, exec_lo -; GFX10-NEXT: s_cselect_b32 s6, s10, s6 -; GFX10-NEXT: s_and_b32 s10, s14, exec_lo -; GFX10-NEXT: s_cselect_b32 s6, s6, s12 -; GFX10-NEXT: s_and_b32 s10, s17, exec_lo -; GFX10-NEXT: s_cselect_b32 s10, s5, s9 -; GFX10-NEXT: s_and_b32 s11, s18, exec_lo -; GFX10-NEXT: s_cselect_b32 s11, s10, 0x7ff80000 -; GFX10-NEXT: s_and_b32 s10, s17, exec_lo -; GFX10-NEXT: s_cselect_b32 s10, s4, s8 -; GFX10-NEXT: s_and_b32 s12, s18, exec_lo -; GFX10-NEXT: s_cselect_b32 s10, s10, 0 -; GFX10-NEXT: s_and_b32 s13, s19, exec_lo -; GFX10-NEXT: v_cmp_eq_f64_e64 s12, s[10:11], 0 -; GFX10-NEXT: s_cselect_b32 s5, s5, s11 -; GFX10-NEXT: s_and_b32 s13, s20, exec_lo -; GFX10-NEXT: s_cselect_b32 s5, s9, s5 -; GFX10-NEXT: s_and_b32 s9, s12, exec_lo -; GFX10-NEXT: s_cselect_b32 s5, s5, s11 -; GFX10-NEXT: s_and_b32 s9, s19, exec_lo -; GFX10-NEXT: s_cselect_b32 s4, s4, s10 -; GFX10-NEXT: s_and_b32 s9, s20, exec_lo -; GFX10-NEXT: s_cselect_b32 s4, s8, s4 -; GFX10-NEXT: s_and_b32 s8, s12, exec_lo -; GFX10-NEXT: s_cselect_b32 s4, s4, s10 +; GFX10-NEXT: v_min_f64 v[0:1], s[6:7], s[10:11] +; GFX10-NEXT: v_cmp_u_f64_e64 s6, s[6:7], s[10:11] +; GFX10-NEXT: v_min_f64 v[4:5], s[4:5], s[8:9] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, s[4:5], s[8:9] +; GFX10-NEXT: v_cndmask_b32_e64 v3, v1, 0x7ff80000, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v2, v0, 0, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v1, v5, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v0, v4, 0, s4 ; GFX10-NEXT: ;;#ASMSTART -; GFX10-NEXT: ; use s[4:7] +; GFX10-NEXT: ; use v[0:3] ; GFX10-NEXT: ;;#ASMEND ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: s_minimum_v2f64: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e64 s8, s[2:3], s[6:7] -; GFX11-NEXT: v_cmp_o_f64_e64 s10, s[2:3], s[6:7] -; GFX11-NEXT: v_cmp_class_f64_e64 s11, s[2:3], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s12, s[6:7], 32 -; GFX11-NEXT: v_cmp_o_f64_e64 s14, s[0:1], s[4:5] -; GFX11-NEXT: v_cmp_class_f64_e64 s15, s[0:1], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s16, s[4:5], 32 -; GFX11-NEXT: s_and_b32 s9, s8, exec_lo -; GFX11-NEXT: s_cselect_b32 s9, s3, s7 -; GFX11-NEXT: s_and_b32 s13, s10, exec_lo -; GFX11-NEXT: s_cselect_b32 s9, s9, 0x7ff80000 -; GFX11-NEXT: s_and_b32 s8, s8, exec_lo -; GFX11-NEXT: s_cselect_b32 s8, s2, s6 -; GFX11-NEXT: s_and_b32 s10, s10, exec_lo -; GFX11-NEXT: s_cselect_b32 s8, s8, 0 -; GFX11-NEXT: v_cmp_lt_f64_e64 s13, s[0:1], s[4:5] -; GFX11-NEXT: v_cmp_eq_f64_e64 s10, s[8:9], 0 -; GFX11-NEXT: s_and_b32 s17, s11, exec_lo -; GFX11-NEXT: s_cselect_b32 s3, s3, s9 -; GFX11-NEXT: s_and_b32 s17, s12, exec_lo -; GFX11-NEXT: s_cselect_b32 s3, s7, s3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: s_and_b32 s7, s10, exec_lo -; GFX11-NEXT: s_cselect_b32 s3, s3, s9 -; GFX11-NEXT: s_and_b32 s7, s11, exec_lo -; GFX11-NEXT: s_cselect_b32 s2, s2, s8 -; GFX11-NEXT: s_and_b32 s7, s12, exec_lo -; GFX11-NEXT: s_cselect_b32 s2, s6, s2 -; GFX11-NEXT: s_and_b32 s6, s10, exec_lo -; GFX11-NEXT: s_cselect_b32 s2, s2, s8 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: s_and_b32 s6, s13, exec_lo -; GFX11-NEXT: s_cselect_b32 s6, s1, s5 -; GFX11-NEXT: s_and_b32 s7, s14, exec_lo -; GFX11-NEXT: s_cselect_b32 s7, s6, 0x7ff80000 -; GFX11-NEXT: s_and_b32 s6, s13, exec_lo -; GFX11-NEXT: s_cselect_b32 s6, s0, s4 -; GFX11-NEXT: s_and_b32 s8, s14, exec_lo -; GFX11-NEXT: s_cselect_b32 s6, s6, 0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-NEXT: s_and_b32 s9, s15, exec_lo -; GFX11-NEXT: v_cmp_eq_f64_e64 s8, s[6:7], 0 -; GFX11-NEXT: s_cselect_b32 s1, s1, s7 -; GFX11-NEXT: s_and_b32 s9, s16, exec_lo -; GFX11-NEXT: s_cselect_b32 s1, s5, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: s_and_b32 s5, s8, exec_lo -; GFX11-NEXT: s_cselect_b32 s1, s1, s7 -; GFX11-NEXT: s_and_b32 s5, s15, exec_lo -; GFX11-NEXT: s_cselect_b32 s0, s0, s6 -; GFX11-NEXT: s_and_b32 s5, s16, exec_lo -; GFX11-NEXT: s_cselect_b32 s0, s4, s0 -; GFX11-NEXT: s_and_b32 s4, s8, exec_lo -; GFX11-NEXT: s_cselect_b32 s0, s0, s6 +; GFX11-NEXT: v_min_f64 v[0:1], s[2:3], s[6:7] +; GFX11-NEXT: v_cmp_u_f64_e64 s2, s[2:3], s[6:7] +; GFX11-NEXT: v_min_f64 v[4:5], s[0:1], s[4:5] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, s[0:1], s[4:5] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_cndmask_b32_e64 v3, v1, 0x7ff80000, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v2, v0, 0, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v1, v5, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v0, v4, 0, s0 ; GFX11-NEXT: ;;#ASMSTART -; GFX11-NEXT: ; use s[0:3] +; GFX11-NEXT: ; use v[0:3] ; GFX11-NEXT: ;;#ASMEND ; GFX11-NEXT: s_setpc_b64 s[30:31] ; @@ -1554,306 +979,110 @@ define <3 x double> @v_minimum_v3f64(<3 x double> %src0, <3 x double> %src1) { ; GFX7-LABEL: v_minimum_v3f64: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7] -; GFX7-NEXT: v_mov_b32_e32 v14, 0x7ff80000 -; GFX7-NEXT: v_cmp_lt_f64_e64 s[8:9], v[2:3], v[8:9] -; GFX7-NEXT: v_cmp_o_f64_e64 s[10:11], v[2:3], v[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v12, v7, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v12, 0, v12, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 32 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[12:13] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v9, v3, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[2:3], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v7, v14, v6, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v8, v2, s[8:9] -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11] -; GFX7-NEXT: v_cmp_class_f64_e64 s[8:9], v[8:9], 32 -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[4:5], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v6, 0, v6, s[10:11] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v12, v11, v5, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v8, v10, v4, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v12, 0, v8, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[10:11], 32 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[12:13] +; GFX7-NEXT: v_min_f64 v[12:13], v[0:1], v[6:7] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX7-NEXT: v_min_f64 v[6:7], v[2:3], v[8:9] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9] +; GFX7-NEXT: v_min_f64 v[8:9], v[4:5], v[10:11] +; GFX7-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11] +; GFX7-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v13, v5, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v5, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v4, v8, 0, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v3f64: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7] -; GFX8-NEXT: v_mov_b32_e32 v14, 0x7ff80000 -; GFX8-NEXT: v_cmp_lt_f64_e64 s[8:9], v[2:3], v[8:9] -; GFX8-NEXT: v_cmp_o_f64_e64 s[10:11], v[2:3], v[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v12, v7, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v12, 0, v12, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 32 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[12:13] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v9, v3, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[2:3], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v7, v14, v6, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v8, v2, s[8:9] -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11] -; GFX8-NEXT: v_cmp_class_f64_e64 s[8:9], v[8:9], 32 -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[4:5], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, v6, s[10:11] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v12, v11, v5, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v8, v10, v4, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v12, 0, v8, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[10:11], 32 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[12:13] +; GFX8-NEXT: v_min_f64 v[12:13], v[0:1], v[6:7] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX8-NEXT: v_min_f64 v[6:7], v[2:3], v[8:9] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9] +; GFX8-NEXT: v_min_f64 v[8:9], v[4:5], v[10:11] +; GFX8-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11] +; GFX8-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v13, v5, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v5, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, 0, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v3f64: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7] -; GFX9-NEXT: v_mov_b32_e32 v14, 0x7ff80000 -; GFX9-NEXT: v_cmp_lt_f64_e64 s[8:9], v[2:3], v[8:9] -; GFX9-NEXT: v_cmp_o_f64_e64 s[10:11], v[2:3], v[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v12, v7, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, v12, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 32 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[12:13] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v9, v3, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[2:3], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v7, v14, v6, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v8, v2, s[8:9] -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11] -; GFX9-NEXT: v_cmp_class_f64_e64 s[8:9], v[8:9], 32 -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[4:5], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, v6, s[10:11] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v12, v11, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v8, v10, v4, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, v8, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[10:11], 32 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[12:13] +; GFX9-NEXT: v_min_f64 v[12:13], v[0:1], v[6:7] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX9-NEXT: v_min_f64 v[6:7], v[2:3], v[8:9] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9] +; GFX9-NEXT: v_min_f64 v[8:9], v[4:5], v[10:11] +; GFX9-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11] +; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v13, v5, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v5, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, 0, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v3f64: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX940-NEXT: v_mov_b32_e32 v14, 0x7ff80000 -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[6:7] -; GFX940-NEXT: v_cndmask_b32_e32 v12, v7, v1, vcc -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v13, v14, v12, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v12, 0, v12, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[6:7], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[12:13] -; GFX940-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v3, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[8:9] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v14, v6, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v8, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v6, 0, v6, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[8:9], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[6:7] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v11, v5, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[4:5], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[2:3] +; GFX940-NEXT: v_min_f64 v[12:13], v[0:1], v[6:7] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX940-NEXT: v_min_f64 v[6:7], v[2:3], v[8:9] ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v7, v14, v6, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v10, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v6, 0, v6, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[10:11], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[6:7] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v4, v6, v4, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[2:3] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX940-NEXT: v_mov_b32_e32 v12, 0x7ff80000 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v13, v12, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[8:9] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v2, v6, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v12, vcc +; GFX940-NEXT: v_min_f64 v[6:7], v[4:5], v[10:11] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[4:5], v[10:11] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v4, v6, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v5, v7, v12, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v3f64: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[2:3], v[8:9] -; GFX10-NEXT: v_cmp_lt_f64_e64 s5, v[4:5], v[10:11] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[0:1], v[6:7] -; GFX10-NEXT: v_cmp_o_f64_e64 s7, v[2:3], v[8:9] -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[4:5], v[10:11] -; GFX10-NEXT: v_cndmask_b32_e32 v12, v7, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v14, v9, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v16, v11, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e32 v17, v6, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v18, v8, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v19, v10, v4, s5 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[2:3], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[4:5], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v13, 0x7ff80000, v12, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v15, 0x7ff80000, v14, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, v17, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v17, 0x7ff80000, v16, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v14, 0, v18, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v16, 0, v19, s8 -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[8:9], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s7, v[6:7], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s8, v[10:11], 32 -; GFX10-NEXT: v_cmp_eq_f64_e64 s9, 0, v[12:13] -; GFX10-NEXT: v_cmp_eq_f64_e64 s10, 0, v[14:15] -; GFX10-NEXT: v_cmp_eq_f64_e64 s11, 0, v[16:17] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v14, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v16, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v15, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v17, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v8, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v6, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v10, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v7, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v9, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v11, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v12, v0, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v14, v2, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v16, v4, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v13, v1, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v15, v3, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v17, v5, s11 +; GFX10-NEXT: v_min_f64 v[12:13], v[0:1], v[6:7] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7] +; GFX10-NEXT: v_min_f64 v[6:7], v[2:3], v[8:9] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[8:9] +; GFX10-NEXT: v_min_f64 v[8:9], v[4:5], v[10:11] +; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[4:5], v[10:11] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v4, v8, 0, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v9, 0x7ff80000, s5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v3f64: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX11-NEXT: v_cmp_lt_f64_e64 s0, v[2:3], v[8:9] -; GFX11-NEXT: v_cmp_lt_f64_e64 s1, v[4:5], v[10:11] -; GFX11-NEXT: v_cmp_o_f64_e64 s2, v[0:1], v[6:7] -; GFX11-NEXT: v_cmp_o_f64_e64 s3, v[2:3], v[8:9] -; GFX11-NEXT: v_cmp_o_f64_e64 s4, v[4:5], v[10:11] -; GFX11-NEXT: v_dual_cndmask_b32 v12, v7, v1 :: v_dual_cndmask_b32 v17, v6, v0 -; GFX11-NEXT: v_cndmask_b32_e64 v14, v9, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v16, v11, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v18, v8, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v19, v10, v4, s1 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[4:5], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v13, 0x7ff80000, v12, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v15, 0x7ff80000, v14, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v12, 0, v17, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v17, 0x7ff80000, v16, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v14, 0, v18, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v16, 0, v19, s4 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[8:9], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s3, v[6:7], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s4, v[10:11], 32 -; GFX11-NEXT: v_cmp_eq_f64_e64 s5, 0, v[12:13] -; GFX11-NEXT: v_cmp_eq_f64_e64 s6, 0, v[14:15] -; GFX11-NEXT: v_cmp_eq_f64_e64 s7, 0, v[16:17] -; GFX11-NEXT: v_dual_cndmask_b32 v0, v12, v0 :: v_dual_cndmask_b32 v1, v13, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v14, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v16, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v15, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v17, v5, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v8, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v6, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v10, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v7, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v9, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v11, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v12, v0, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v14, v2, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v16, v4, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v13, v1, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v15, v3, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v17, v5, s7 +; GFX11-NEXT: v_min_f64 v[12:13], v[0:1], v[6:7] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7] +; GFX11-NEXT: v_min_f64 v[6:7], v[2:3], v[8:9] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[8:9] +; GFX11-NEXT: v_min_f64 v[8:9], v[4:5], v[10:11] +; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[10:11] +; GFX11-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v6, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v4, v8, 0, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v5, v9, 0x7ff80000, s1 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v3f64: @@ -1875,247 +1104,49 @@ define <3 x double> @v_minimum_v3f64__nnan(<3 x double> %src0, <3 x double> %src ; GFX7-LABEL: v_minimum_v3f64__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 32 -; GFX7-NEXT: v_cmp_lt_f64_e64 s[8:9], v[2:3], v[8:9] -; GFX7-NEXT: v_cmp_class_f64_e64 s[10:11], v[10:11], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v13, v7, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[12:13] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[2:3], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v7, v9, v3, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v8, v2, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v13, v11, v5, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v12, v10, v4, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[8:9], 32 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[6:7] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[12:13] +; GFX7-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7] +; GFX7-NEXT: v_min_f64 v[2:3], v[2:3], v[8:9] +; GFX7-NEXT: v_min_f64 v[4:5], v[4:5], v[10:11] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v3f64__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 32 -; GFX8-NEXT: v_cmp_lt_f64_e64 s[8:9], v[2:3], v[8:9] -; GFX8-NEXT: v_cmp_class_f64_e64 s[10:11], v[10:11], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v13, v7, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[12:13] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[2:3], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v7, v9, v3, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v8, v2, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v13, v11, v5, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v12, v10, v4, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[8:9], 32 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[6:7] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[12:13] +; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7] +; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[8:9] +; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[10:11] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v3f64__nnan: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[6:7], 32 -; GFX9-NEXT: v_cmp_lt_f64_e64 s[8:9], v[2:3], v[8:9] -; GFX9-NEXT: v_cmp_class_f64_e64 s[10:11], v[10:11], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v13, v7, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[12:13] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[2:3], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v7, v9, v3, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v8, v2, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v13, v11, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v12, v10, v4, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[8:9], 32 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[6:7] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[12:13] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7] +; GFX9-NEXT: v_min_f64 v[2:3], v[2:3], v[8:9] +; GFX9-NEXT: v_min_f64 v[4:5], v[4:5], v[10:11] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v3f64__nnan: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[6:7], 32 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v13, v7, v1, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[12:13] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v7, v9, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v6, v8, v2, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[8:9], 32 -; GFX940-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[0:1] -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[6:7] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[0:1] -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e32 v7, v11, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v6, v10, v4, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[10:11], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[6:7] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v4, v6, v4, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[2:3] +; GFX940-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7] +; GFX940-NEXT: v_min_f64 v[2:3], v[2:3], v[8:9] +; GFX940-NEXT: v_min_f64 v[4:5], v[4:5], v[10:11] ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v3f64__nnan: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[2:3], v[8:9] -; GFX10-NEXT: v_cmp_lt_f64_e64 s5, v[4:5], v[10:11] -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[8:9], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s7, v[6:7], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s8, v[10:11], 32 -; GFX10-NEXT: v_cndmask_b32_e32 v13, v7, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v15, v9, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v17, v11, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v14, v8, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v16, v10, v4, s5 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[2:3], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[4:5], 32 -; GFX10-NEXT: v_cmp_eq_f64_e64 s9, 0, v[12:13] -; GFX10-NEXT: v_cmp_eq_f64_e64 s10, 0, v[14:15] -; GFX10-NEXT: v_cmp_eq_f64_e64 s11, 0, v[16:17] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v14, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v16, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v15, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v17, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v6, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v8, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v10, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v7, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v9, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v11, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v12, v0, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v14, v2, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v16, v4, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v13, v1, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v15, v3, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v17, v5, s11 +; GFX10-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7] +; GFX10-NEXT: v_min_f64 v[2:3], v[2:3], v[8:9] +; GFX10-NEXT: v_min_f64 v[4:5], v[4:5], v[10:11] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v3f64__nnan: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX11-NEXT: v_cmp_lt_f64_e64 s0, v[2:3], v[8:9] -; GFX11-NEXT: v_cmp_lt_f64_e64 s1, v[4:5], v[10:11] -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[8:9], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s3, v[6:7], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s4, v[10:11], 32 -; GFX11-NEXT: v_dual_cndmask_b32 v13, v7, v1 :: v_dual_cndmask_b32 v12, v6, v0 -; GFX11-NEXT: v_cndmask_b32_e64 v15, v9, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v17, v11, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v14, v8, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v16, v10, v4, s1 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[4:5], 32 -; GFX11-NEXT: v_cmp_eq_f64_e64 s5, 0, v[12:13] -; GFX11-NEXT: v_cmp_eq_f64_e64 s6, 0, v[14:15] -; GFX11-NEXT: v_cmp_eq_f64_e64 s7, 0, v[16:17] -; GFX11-NEXT: v_dual_cndmask_b32 v0, v12, v0 :: v_dual_cndmask_b32 v1, v13, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v14, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v16, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v15, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v17, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v6, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v8, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v10, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v7, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v9, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v11, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v12, v0, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v14, v2, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v16, v4, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v13, v1, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v15, v3, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v17, v5, s7 +; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7] +; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[8:9] +; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[10:11] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v3f64__nnan: @@ -2137,144 +1168,110 @@ define <3 x double> @v_minimum_v3f64__nsz(<3 x double> %src0, <3 x double> %src1 ; GFX7-LABEL: v_minimum_v3f64__nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[8:9], v[4:5], v[10:11] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[2:3], v[8:9] -; GFX7-NEXT: v_cmp_o_f64_e64 s[10:11], v[4:5], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v10, v4, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v11, v5, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v0, 0, v12, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v4, 0, v7, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v8, v2, vcc -; GFX7-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX7-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v2, 0, v6, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[10:11] +; GFX7-NEXT: v_min_f64 v[12:13], v[0:1], v[6:7] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX7-NEXT: v_min_f64 v[6:7], v[2:3], v[8:9] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9] +; GFX7-NEXT: v_min_f64 v[8:9], v[4:5], v[10:11] +; GFX7-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11] +; GFX7-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v13, v5, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v5, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v4, v8, 0, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v3f64__nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[8:9], v[4:5], v[10:11] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[2:3], v[8:9] -; GFX8-NEXT: v_cmp_o_f64_e64 s[10:11], v[4:5], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v10, v4, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v11, v5, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, v12, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v4, 0, v7, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v8, v2, vcc -; GFX8-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, v6, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[10:11] +; GFX8-NEXT: v_min_f64 v[12:13], v[0:1], v[6:7] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX8-NEXT: v_min_f64 v[6:7], v[2:3], v[8:9] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9] +; GFX8-NEXT: v_min_f64 v[8:9], v[4:5], v[10:11] +; GFX8-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11] +; GFX8-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v13, v5, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v5, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, 0, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v3f64__nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[6:7] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[8:9], v[4:5], v[10:11] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[2:3], v[8:9] -; GFX9-NEXT: v_cmp_o_f64_e64 s[10:11], v[4:5], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v10, v4, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v11, v5, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, v12, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, v7, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v8, v2, vcc -; GFX9-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, v6, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[10:11] +; GFX9-NEXT: v_min_f64 v[12:13], v[0:1], v[6:7] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX9-NEXT: v_min_f64 v[6:7], v[2:3], v[8:9] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[8:9] +; GFX9-NEXT: v_min_f64 v[8:9], v[4:5], v[10:11] +; GFX9-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[10:11] +; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v13, v5, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v5, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, 0, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v3f64__nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[6:7] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX940-NEXT: v_mov_b32_e32 v6, 0x7ff80000 -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9] -; GFX940-NEXT: v_cndmask_b32_e64 v0, 0, v12, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v6, v1, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v7, v8, v2, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v2, 0, v7, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v3, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v7, v10, v4, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[4:5], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e32 v5, v11, v5, vcc +; GFX940-NEXT: v_min_f64 v[12:13], v[0:1], v[6:7] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[6:7] +; GFX940-NEXT: v_min_f64 v[6:7], v[2:3], v[8:9] ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v4, 0, v7, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v6, v5, s[0:1] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc +; GFX940-NEXT: v_mov_b32_e32 v12, 0x7ff80000 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v13, v12, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[8:9] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v2, v6, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v7, v12, vcc +; GFX940-NEXT: v_min_f64 v[6:7], v[4:5], v[10:11] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[4:5], v[10:11] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v4, v6, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v5, v7, v12, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v3f64__nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[2:3], v[8:9] -; GFX10-NEXT: v_cmp_lt_f64_e64 s5, v[4:5], v[10:11] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[0:1], v[6:7] -; GFX10-NEXT: v_cmp_o_f64_e64 s7, v[2:3], v[8:9] -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[4:5], v[10:11] -; GFX10-NEXT: v_cndmask_b32_e32 v12, v6, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v6, v8, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v8, v10, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v9, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v11, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, v12, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, v6, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, v8, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v1, 0x7ff80000, v1, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v3, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v5, s8 +; GFX10-NEXT: v_min_f64 v[12:13], v[0:1], v[6:7] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7] +; GFX10-NEXT: v_min_f64 v[6:7], v[2:3], v[8:9] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[8:9] +; GFX10-NEXT: v_min_f64 v[8:9], v[4:5], v[10:11] +; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[4:5], v[10:11] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v4, v8, 0, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v9, 0x7ff80000, s5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v3f64__nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX11-NEXT: v_cmp_lt_f64_e64 s0, v[2:3], v[8:9] -; GFX11-NEXT: v_cmp_lt_f64_e64 s1, v[4:5], v[10:11] -; GFX11-NEXT: v_cmp_o_f64_e64 s2, v[0:1], v[6:7] -; GFX11-NEXT: v_cmp_o_f64_e64 s3, v[2:3], v[8:9] -; GFX11-NEXT: v_cmp_o_f64_e64 s4, v[4:5], v[10:11] -; GFX11-NEXT: v_dual_cndmask_b32 v12, v6, v0 :: v_dual_cndmask_b32 v1, v7, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v8, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v8, v10, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v9, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v11, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, v12, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, v6, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v4, 0, v8, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v1, 0x7ff80000, v1, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v3, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v5, s4 +; GFX11-NEXT: v_min_f64 v[12:13], v[0:1], v[6:7] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[6:7] +; GFX11-NEXT: v_min_f64 v[6:7], v[2:3], v[8:9] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[8:9] +; GFX11-NEXT: v_min_f64 v[8:9], v[4:5], v[10:11] +; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[10:11] +; GFX11-NEXT: v_cndmask_b32_e64 v0, v12, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v13, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v6, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v4, v8, 0, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v5, v9, 0x7ff80000, s1 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v3f64__nsz: @@ -2296,88 +1293,49 @@ define <3 x double> @v_minimum_v3f64__nnan_nsz(<3 x double> %src0, <3 x double> ; GFX7-LABEL: v_minimum_v3f64__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], v[2:3], v[8:9] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[6:7], v[4:5], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v10, v4, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v11, v5, s[6:7] +; GFX7-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7] +; GFX7-NEXT: v_min_f64 v[2:3], v[2:3], v[8:9] +; GFX7-NEXT: v_min_f64 v[4:5], v[4:5], v[10:11] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v3f64__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[4:5], v[2:3], v[8:9] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[6:7], v[4:5], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v10, v4, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v11, v5, s[6:7] +; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7] +; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[8:9] +; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[10:11] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v3f64__nnan_nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[4:5], v[2:3], v[8:9] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[6:7], v[4:5], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v10, v4, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v11, v5, s[6:7] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7] +; GFX9-NEXT: v_min_f64 v[2:3], v[2:3], v[8:9] +; GFX9-NEXT: v_min_f64 v[4:5], v[4:5], v[10:11] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v3f64__nnan_nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[6:7] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[8:9] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[10:11] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v10, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v11, v5, vcc +; GFX940-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7] +; GFX940-NEXT: v_min_f64 v[2:3], v[2:3], v[8:9] +; GFX940-NEXT: v_min_f64 v[4:5], v[4:5], v[10:11] ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v3f64__nnan_nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[2:3], v[8:9] -; GFX10-NEXT: v_cmp_lt_f64_e64 s5, v[4:5], v[10:11] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v8, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v10, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v9, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v11, v5, s5 +; GFX10-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7] +; GFX10-NEXT: v_min_f64 v[2:3], v[2:3], v[8:9] +; GFX10-NEXT: v_min_f64 v[4:5], v[4:5], v[10:11] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v3f64__nnan_nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[6:7] -; GFX11-NEXT: v_cmp_lt_f64_e64 s0, v[2:3], v[8:9] -; GFX11-NEXT: v_cmp_lt_f64_e64 s1, v[4:5], v[10:11] -; GFX11-NEXT: v_dual_cndmask_b32 v0, v6, v0 :: v_dual_cndmask_b32 v1, v7, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v8, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v10, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v9, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v11, v5, s1 +; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7] +; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[8:9] +; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[10:11] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v3f64__nnan_nsz: @@ -2399,404 +1357,135 @@ define <4 x double> @v_minimum_v4f64(<4 x double> %src0, <4 x double> %src1) { ; GFX7-LABEL: v_minimum_v4f64: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9] -; GFX7-NEXT: v_mov_b32_e32 v18, 0x7ff80000 -; GFX7-NEXT: v_cmp_lt_f64_e64 s[6:7], v[2:3], v[10:11] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[2:3], v[10:11] -; GFX7-NEXT: v_cmp_o_f64_e64 s[12:13], v[4:5], v[12:13] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v9, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v17, v18, v16, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[8:9], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v19, v11, v3, s[6:7] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[16:17] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[10:11], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v8, v10, v2, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v18, v19, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[8:9] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[8:9], v[4:5], v[12:13] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX7-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[4:5] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[6:7], v[14:15] -; GFX7-NEXT: v_cndmask_b32_e64 v10, v13, v5, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v18, v10, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e32 v10, v15, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v8, v12, v4, s[8:9] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[4:5], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v11, v18, v10, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v10, v14, v6, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[12:13] -; GFX7-NEXT: v_cmp_class_f64_e64 s[8:9], v[12:13], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v10, 0, v10, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[14:15], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[10:11] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[8:9] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v10, v6, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v11, v7, s[12:13] +; GFX7-NEXT: v_min_f64 v[16:17], v[0:1], v[8:9] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX7-NEXT: v_min_f64 v[8:9], v[2:3], v[10:11] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11] +; GFX7-NEXT: v_min_f64 v[10:11], v[4:5], v[12:13] +; GFX7-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13] +; GFX7-NEXT: v_min_f64 v[12:13], v[6:7], v[14:15] +; GFX7-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15] +; GFX7-NEXT: v_mov_b32_e32 v7, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v17, v7, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v8, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v9, v7, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v4, v10, 0, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v11, v7, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v6, v12, 0, s[8:9] +; GFX7-NEXT: v_cndmask_b32_e64 v7, v13, v7, s[8:9] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v4f64: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9] -; GFX8-NEXT: v_mov_b32_e32 v18, 0x7ff80000 -; GFX8-NEXT: v_cmp_lt_f64_e64 s[6:7], v[2:3], v[10:11] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[2:3], v[10:11] -; GFX8-NEXT: v_cmp_o_f64_e64 s[12:13], v[4:5], v[12:13] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v9, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v17, v18, v16, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[8:9], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v19, v11, v3, s[6:7] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[16:17] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[10:11], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v8, v10, v2, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v18, v19, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[8:9] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[8:9], v[4:5], v[12:13] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX8-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[4:5] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[6:7], v[14:15] -; GFX8-NEXT: v_cndmask_b32_e64 v10, v13, v5, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v18, v10, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e32 v10, v15, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v8, v12, v4, s[8:9] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[4:5], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v11, v18, v10, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v10, v14, v6, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[12:13] -; GFX8-NEXT: v_cmp_class_f64_e64 s[8:9], v[12:13], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v10, 0, v10, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[14:15], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[10:11] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[8:9] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v10, v6, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v11, v7, s[12:13] +; GFX8-NEXT: v_min_f64 v[16:17], v[0:1], v[8:9] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX8-NEXT: v_min_f64 v[8:9], v[2:3], v[10:11] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11] +; GFX8-NEXT: v_min_f64 v[10:11], v[4:5], v[12:13] +; GFX8-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13] +; GFX8-NEXT: v_min_f64 v[12:13], v[6:7], v[14:15] +; GFX8-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15] +; GFX8-NEXT: v_mov_b32_e32 v7, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v7, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v8, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v9, v7, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v4, v10, 0, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v11, v7, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v6, v12, 0, s[8:9] +; GFX8-NEXT: v_cndmask_b32_e64 v7, v13, v7, s[8:9] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v4f64: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9] -; GFX9-NEXT: v_mov_b32_e32 v18, 0x7ff80000 -; GFX9-NEXT: v_cmp_lt_f64_e64 s[6:7], v[2:3], v[10:11] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[2:3], v[10:11] -; GFX9-NEXT: v_cmp_o_f64_e64 s[12:13], v[4:5], v[12:13] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v9, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v17, v18, v16, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[8:9], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v19, v11, v3, s[6:7] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[16:17] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[10:11], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v8, v10, v2, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v18, v19, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[8:9] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[8:9], v[4:5], v[12:13] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX9-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[4:5] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[6:7], v[14:15] -; GFX9-NEXT: v_cndmask_b32_e64 v10, v13, v5, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v18, v10, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e32 v10, v15, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v8, v12, v4, s[8:9] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[4:5], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v11, v18, v10, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v10, v14, v6, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[12:13] -; GFX9-NEXT: v_cmp_class_f64_e64 s[8:9], v[12:13], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, v10, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[14:15], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[10:11] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[8:9] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v10, v6, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v11, v7, s[12:13] +; GFX9-NEXT: v_min_f64 v[16:17], v[0:1], v[8:9] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX9-NEXT: v_min_f64 v[8:9], v[2:3], v[10:11] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11] +; GFX9-NEXT: v_min_f64 v[10:11], v[4:5], v[12:13] +; GFX9-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13] +; GFX9-NEXT: v_min_f64 v[12:13], v[6:7], v[14:15] +; GFX9-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15] +; GFX9-NEXT: v_mov_b32_e32 v7, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v7, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v8, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v9, v7, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v4, v10, 0, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v11, v7, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v6, v12, 0, s[8:9] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v13, v7, s[8:9] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v4f64: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX940-NEXT: v_mov_b32_e32 v18, 0x7ff80000 -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v9, v1, vcc -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v18, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[8:9], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v11, v3, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v9, v18, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[10:11], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[12:13] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v13, v5, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[4:5], v[12:13] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[2:3] +; GFX940-NEXT: v_min_f64 v[16:17], v[0:1], v[8:9] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX940-NEXT: v_min_f64 v[8:9], v[2:3], v[10:11] ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v9, v18, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v12, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[12:13], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v5, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15] -; GFX940-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v15, v7, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[6:7], v[14:15] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v9, v18, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v14, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[14:15], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v6, v8, v6, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[2:3] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX940-NEXT: v_mov_b32_e32 v16, 0x7ff80000 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v17, v16, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[10:11] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v2, v8, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v16, vcc +; GFX940-NEXT: v_min_f64 v[8:9], v[4:5], v[12:13] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[4:5], v[12:13] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v4, v8, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v16, vcc +; GFX940-NEXT: v_min_f64 v[8:9], v[6:7], v[14:15] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[6:7], v[14:15] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v6, v8, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v7, v9, v16, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v4f64: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[2:3], v[10:11] -; GFX10-NEXT: v_cmp_o_f64_e64 s5, v[0:1], v[8:9] -; GFX10-NEXT: v_cmp_lt_f64_e64 s6, v[4:5], v[12:13] -; GFX10-NEXT: v_cmp_o_f64_e64 s7, v[2:3], v[10:11] -; GFX10-NEXT: v_cmp_lt_f64_e64 s8, v[6:7], v[14:15] -; GFX10-NEXT: v_cmp_o_f64_e64 s9, v[4:5], v[12:13] -; GFX10-NEXT: v_cmp_o_f64_e64 s10, v[6:7], v[14:15] -; GFX10-NEXT: v_cmp_class_f64_e64 s11, v[14:15], 32 -; GFX10-NEXT: v_cndmask_b32_e32 v16, v9, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v18, v11, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v19, v8, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v21, v10, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v20, v13, v5, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v22, v15, v7, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v17, 0x7ff80000, v16, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v16, 0, v19, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v19, 0x7ff80000, v18, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v23, v12, v4, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v18, 0, v21, s7 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v24, v14, v6, s8 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[2:3], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s7, v[4:5], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s8, v[6:7], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[8:9], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v21, 0x7ff80000, v20, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v20, 0, v23, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v23, 0x7ff80000, v22, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v22, 0, v24, s10 -; GFX10-NEXT: v_cmp_class_f64_e64 s9, v[10:11], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[12:13], 32 -; GFX10-NEXT: v_cmp_eq_f64_e64 s6, 0, v[16:17] -; GFX10-NEXT: v_cmp_eq_f64_e64 s12, 0, v[18:19] -; GFX10-NEXT: v_cmp_eq_f64_e64 s13, 0, v[20:21] -; GFX10-NEXT: v_cmp_eq_f64_e64 s14, 0, v[22:23] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v18, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v20, v4, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v22, v6, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v19, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v21, v5, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v23, v7, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v8, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v14, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v10, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v12, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v9, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v11, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v13, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v15, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v16, v0, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v17, v1, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v18, v2, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v19, v3, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v20, v4, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v21, v5, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v22, v6, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v23, v7, s14 +; GFX10-NEXT: v_min_f64 v[16:17], v[0:1], v[8:9] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9] +; GFX10-NEXT: v_min_f64 v[8:9], v[2:3], v[10:11] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[10:11] +; GFX10-NEXT: v_min_f64 v[10:11], v[4:5], v[12:13] +; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[4:5], v[12:13] +; GFX10-NEXT: v_min_f64 v[12:13], v[6:7], v[14:15] +; GFX10-NEXT: v_cmp_u_f64_e64 s6, v[6:7], v[14:15] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v8, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v9, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v4, v10, 0, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v11, 0x7ff80000, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v6, v12, 0, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v7, v13, 0x7ff80000, s6 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v4f64: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX11-NEXT: v_cmp_o_f64_e64 s0, v[0:1], v[8:9] -; GFX11-NEXT: v_cmp_lt_f64_e64 s1, v[2:3], v[10:11] -; GFX11-NEXT: v_cmp_lt_f64_e64 s2, v[4:5], v[12:13] -; GFX11-NEXT: v_cmp_lt_f64_e64 s3, v[6:7], v[14:15] -; GFX11-NEXT: v_cmp_o_f64_e64 s4, v[2:3], v[10:11] -; GFX11-NEXT: v_cmp_o_f64_e64 s5, v[4:5], v[12:13] -; GFX11-NEXT: v_cmp_o_f64_e64 s6, v[6:7], v[14:15] -; GFX11-NEXT: v_cndmask_b32_e32 v16, v9, v1, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v18, v11, v3, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v20, v13, v5, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v22, v15, v7, s3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v17, 0x7ff80000, v16, s0 -; GFX11-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc_lo -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v19, 0x7ff80000, v18, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v21, 0x7ff80000, v20, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v18, v10, v2, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v20, v12, v4, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v24, v14, v6, s3 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[4:5], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s3, v[6:7], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v23, 0x7ff80000, v22, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v18, 0, v18, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v20, 0, v20, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v22, 0, v24, s6 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[12:13], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s4, v[14:15], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s5, v[8:9], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s6, v[10:11], 32 -; GFX11-NEXT: v_cmp_eq_f64_e64 s8, 0, v[18:19] -; GFX11-NEXT: v_cmp_eq_f64_e64 s9, 0, v[20:21] -; GFX11-NEXT: v_cmp_eq_f64_e64 s10, 0, v[22:23] -; GFX11-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v16, 0, v16, s0 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_cmp_eq_f64_e64 s7, 0, v[16:17] -; GFX11-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v4, v20, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v22, v6, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v21, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v23, v7, s3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v12, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v8, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v14, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v9, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v13, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v7, v15, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v20, v4, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v22, v6, s10 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v5, v21, v5, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v23, v7, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v18, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v19, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v16, v0, s7 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v10, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v11, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v17, v1, s7 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v18, v2, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v19, v3, s8 +; GFX11-NEXT: v_min_f64 v[16:17], v[0:1], v[8:9] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9] +; GFX11-NEXT: v_min_f64 v[8:9], v[2:3], v[10:11] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[10:11] +; GFX11-NEXT: v_min_f64 v[10:11], v[4:5], v[12:13] +; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[12:13] +; GFX11-NEXT: v_min_f64 v[12:13], v[6:7], v[14:15] +; GFX11-NEXT: v_cmp_u_f64_e64 s2, v[6:7], v[14:15] +; GFX11-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v8, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v9, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v4, v10, 0, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v5, v11, 0x7ff80000, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v6, v12, 0, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v7, v13, 0x7ff80000, s2 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v4f64: @@ -2819,320 +1508,55 @@ define <4 x double> @v_minimum_v4f64__nnan(<4 x double> %src0, <4 x double> %src ; GFX7-LABEL: v_minimum_v4f64__nnan: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], 32 -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[8:9], 32 -; GFX7-NEXT: v_cmp_class_f64_e64 s[10:11], v[10:11], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[4:5] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], v[4:5], v[12:13] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[16:17] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[12:13], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v19, v11, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v18, v10, v2, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[18:19] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v13, v5, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v8, v12, v4, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[4:5], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[8:9] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[10:11] -; GFX7-NEXT: v_cmp_class_f64_e64 s[10:11], v[14:15], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v11, v15, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v10, v14, v6, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v10, v6, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v11, v7, s[12:13] +; GFX7-NEXT: v_min_f64 v[0:1], v[0:1], v[8:9] +; GFX7-NEXT: v_min_f64 v[2:3], v[2:3], v[10:11] +; GFX7-NEXT: v_min_f64 v[4:5], v[4:5], v[12:13] +; GFX7-NEXT: v_min_f64 v[6:7], v[6:7], v[14:15] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v4f64__nnan: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], 32 -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[8:9], 32 -; GFX8-NEXT: v_cmp_class_f64_e64 s[10:11], v[10:11], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[4:5] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[4:5], v[4:5], v[12:13] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[16:17] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[12:13], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v19, v11, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v18, v10, v2, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[18:19] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v13, v5, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v8, v12, v4, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[4:5], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[8:9] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[10:11] -; GFX8-NEXT: v_cmp_class_f64_e64 s[10:11], v[14:15], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v11, v15, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v10, v14, v6, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v10, v6, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v11, v7, s[12:13] +; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[8:9] +; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[10:11] +; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[12:13] +; GFX8-NEXT: v_min_f64 v[6:7], v[6:7], v[14:15] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v4f64__nnan: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[0:1], 32 -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[8:9], 32 -; GFX9-NEXT: v_cmp_class_f64_e64 s[10:11], v[10:11], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[4:5] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[4:5], v[4:5], v[12:13] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[16:17] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[12:13], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v19, v11, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v18, v10, v2, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[18:19] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v13, v5, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v8, v12, v4, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[4:5], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[8:9] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[10:11] -; GFX9-NEXT: v_cmp_class_f64_e64 s[10:11], v[14:15], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v11, v15, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v10, v14, v6, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v7, v11, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v10, v6, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v11, v7, s[12:13] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[8:9] +; GFX9-NEXT: v_min_f64 v[2:3], v[2:3], v[10:11] +; GFX9-NEXT: v_min_f64 v[4:5], v[4:5], v[12:13] +; GFX9-NEXT: v_min_f64 v[6:7], v[6:7], v[14:15] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v4f64__nnan: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[8:9], 32 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v11, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[10:11], 32 -; GFX940-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[0:1] -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[0:1] -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[12:13] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v13, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v8, v12, v4, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[12:13], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, v4, v12, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[0:1] -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15] -; GFX940-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v15, v7, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v8, v14, v6, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[14:15], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[8:9] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v6, v6, v14, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v7, v15, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v6, v8, v6, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[2:3] +; GFX940-NEXT: v_min_f64 v[0:1], v[0:1], v[8:9] +; GFX940-NEXT: v_min_f64 v[2:3], v[2:3], v[10:11] +; GFX940-NEXT: v_min_f64 v[4:5], v[4:5], v[12:13] +; GFX940-NEXT: v_min_f64 v[6:7], v[6:7], v[14:15] ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v4f64__nnan: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[2:3], v[10:11] -; GFX10-NEXT: v_cmp_lt_f64_e64 s5, v[4:5], v[12:13] -; GFX10-NEXT: v_cmp_lt_f64_e64 s6, v[6:7], v[14:15] -; GFX10-NEXT: v_cmp_class_f64_e64 s7, v[10:11], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s8, v[8:9], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[12:13], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s11, v[14:15], 32 -; GFX10-NEXT: v_cndmask_b32_e32 v17, v9, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v19, v11, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v21, v13, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v23, v15, v7, s6 -; GFX10-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v18, v10, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v20, v12, v4, s5 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[4:5], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v22, v14, v6, s6 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[6:7], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[0:1], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[2:3], 32 -; GFX10-NEXT: v_cmp_eq_f64_e64 s9, 0, v[16:17] -; GFX10-NEXT: v_cmp_eq_f64_e64 s12, 0, v[18:19] -; GFX10-NEXT: v_cmp_eq_f64_e64 s13, 0, v[20:21] -; GFX10-NEXT: v_cmp_eq_f64_e64 s14, 0, v[22:23] -; GFX10-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v6, v22, v6, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v16, v0, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v18, v2, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v17, v1, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v19, v3, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v23, v7, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v8, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v10, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v12, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v14, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v9, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v11, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v13, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v15, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v16, v0, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v18, v2, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v17, v1, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v19, v3, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v20, v4, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v21, v5, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v22, v6, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v23, v7, s14 +; GFX10-NEXT: v_min_f64 v[0:1], v[0:1], v[8:9] +; GFX10-NEXT: v_min_f64 v[2:3], v[2:3], v[10:11] +; GFX10-NEXT: v_min_f64 v[4:5], v[4:5], v[12:13] +; GFX10-NEXT: v_min_f64 v[6:7], v[6:7], v[14:15] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v4f64__nnan: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX11-NEXT: v_cmp_lt_f64_e64 s0, v[2:3], v[10:11] -; GFX11-NEXT: v_cmp_lt_f64_e64 s1, v[4:5], v[12:13] -; GFX11-NEXT: v_cmp_lt_f64_e64 s2, v[6:7], v[14:15] -; GFX11-NEXT: v_cmp_class_f64_e64 s3, v[6:7], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s4, v[14:15], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s5, v[8:9], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s6, v[10:11], 32 -; GFX11-NEXT: v_dual_cndmask_b32 v17, v9, v1 :: v_dual_cndmask_b32 v16, v8, v0 -; GFX11-NEXT: v_cndmask_b32_e64 v19, v11, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v21, v13, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v18, v10, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v20, v12, v4, s1 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[4:5], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v23, v15, v7, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v22, v14, v6, s2 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[12:13], 32 -; GFX11-NEXT: v_cmp_eq_f64_e64 s7, 0, v[16:17] -; GFX11-NEXT: v_cmp_eq_f64_e64 s8, 0, v[18:19] -; GFX11-NEXT: v_cmp_eq_f64_e64 s9, 0, v[20:21] -; GFX11-NEXT: v_cmp_eq_f64_e64 s10, 0, v[22:23] -; GFX11-NEXT: v_cndmask_b32_e64 v6, v22, v6, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v23, v7, s3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v14, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v7, v15, s4 -; GFX11-NEXT: v_dual_cndmask_b32 v0, v16, v0 :: v_dual_cndmask_b32 v1, v17, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v18, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v20, v4, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v19, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v21, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v8, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v10, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v12, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v9, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v11, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v13, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v16, v0, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v18, v2, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v20, v4, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v22, v6, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v17, v1, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v19, v3, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v21, v5, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v23, v7, s10 +; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[8:9] +; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[10:11] +; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[12:13] +; GFX11-NEXT: v_min_f64 v[6:7], v[6:7], v[14:15] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v4f64__nnan: @@ -3155,180 +1579,135 @@ define <4 x double> @v_minimum_v4f64__nsz(<4 x double> %src0, <4 x double> %src1 ; GFX7-LABEL: v_minimum_v4f64__nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[6:7], v[4:5], v[12:13] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[10:11], v[6:7], v[14:15] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[4:5], v[12:13] -; GFX7-NEXT: v_cmp_o_f64_e64 s[12:13], v[6:7], v[14:15] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX7-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v0, 0, v16, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[4:5] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v15, v7, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v8, v7, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e32 v9, v10, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v10, v14, v6, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, 0, v9, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v12, v4, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v4, 0, v9, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v6, 0, v10, s[12:13] +; GFX7-NEXT: v_min_f64 v[16:17], v[0:1], v[8:9] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX7-NEXT: v_min_f64 v[8:9], v[2:3], v[10:11] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11] +; GFX7-NEXT: v_min_f64 v[10:11], v[4:5], v[12:13] +; GFX7-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13] +; GFX7-NEXT: v_min_f64 v[12:13], v[6:7], v[14:15] +; GFX7-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15] +; GFX7-NEXT: v_mov_b32_e32 v7, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v1, v17, v7, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v2, v8, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v3, v9, v7, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v4, v10, 0, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v11, v7, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v6, v12, 0, s[8:9] +; GFX7-NEXT: v_cndmask_b32_e64 v7, v13, v7, s[8:9] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v4f64__nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[6:7], v[4:5], v[12:13] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[10:11], v[6:7], v[14:15] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[4:5], v[12:13] -; GFX8-NEXT: v_cmp_o_f64_e64 s[12:13], v[6:7], v[14:15] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX8-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, v16, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[4:5] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v15, v7, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v8, v7, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e32 v9, v10, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v10, v14, v6, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, v9, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v12, v4, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v4, 0, v9, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, v10, s[12:13] +; GFX8-NEXT: v_min_f64 v[16:17], v[0:1], v[8:9] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX8-NEXT: v_min_f64 v[8:9], v[2:3], v[10:11] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11] +; GFX8-NEXT: v_min_f64 v[10:11], v[4:5], v[12:13] +; GFX8-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13] +; GFX8-NEXT: v_min_f64 v[12:13], v[6:7], v[14:15] +; GFX8-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15] +; GFX8-NEXT: v_mov_b32_e32 v7, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v7, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v2, v8, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v3, v9, v7, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v4, v10, 0, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v11, v7, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v6, v12, 0, s[8:9] +; GFX8-NEXT: v_cndmask_b32_e64 v7, v13, v7, s[8:9] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v4f64__nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[8:9] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[6:7], v[4:5], v[12:13] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[10:11], v[6:7], v[14:15] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[4:5], v[12:13] -; GFX9-NEXT: v_cmp_o_f64_e64 s[12:13], v[6:7], v[14:15] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX9-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, v16, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[4:5] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v15, v7, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v8, v7, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v10, v14, v6, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, v9, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v12, v4, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, v9, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, v10, s[12:13] +; GFX9-NEXT: v_min_f64 v[16:17], v[0:1], v[8:9] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX9-NEXT: v_min_f64 v[8:9], v[2:3], v[10:11] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[10:11] +; GFX9-NEXT: v_min_f64 v[10:11], v[4:5], v[12:13] +; GFX9-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[12:13] +; GFX9-NEXT: v_min_f64 v[12:13], v[6:7], v[14:15] +; GFX9-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[14:15] +; GFX9-NEXT: v_mov_b32_e32 v7, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v7, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v8, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v9, v7, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v4, v10, 0, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v11, v7, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v6, v12, 0, s[8:9] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v13, v7, s[8:9] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v4f64__nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[8:9] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX940-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e64 v0, 0, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v10, v2, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[10:11] -; GFX940-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[12:13] -; GFX940-NEXT: v_cndmask_b32_e64 v2, 0, v9, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v12, v4, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[4:5], v[12:13] -; GFX940-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15] -; GFX940-NEXT: v_cndmask_b32_e64 v4, 0, v9, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v14, v6, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[6:7], v[14:15] -; GFX940-NEXT: v_cndmask_b32_e32 v7, v15, v7, vcc +; GFX940-NEXT: v_min_f64 v[16:17], v[0:1], v[8:9] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[8:9] +; GFX940-NEXT: v_min_f64 v[8:9], v[2:3], v[10:11] ; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v6, 0, v9, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v8, v7, s[0:1] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc +; GFX940-NEXT: v_mov_b32_e32 v16, 0x7ff80000 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v17, v16, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[10:11] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v2, v8, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v9, v16, vcc +; GFX940-NEXT: v_min_f64 v[8:9], v[4:5], v[12:13] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[4:5], v[12:13] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v4, v8, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v16, vcc +; GFX940-NEXT: v_min_f64 v[8:9], v[6:7], v[14:15] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[6:7], v[14:15] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v6, v8, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v7, v9, v16, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v4f64__nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[2:3], v[10:11] -; GFX10-NEXT: v_cmp_lt_f64_e64 s5, v[4:5], v[12:13] -; GFX10-NEXT: v_cmp_lt_f64_e64 s7, v[6:7], v[14:15] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[0:1], v[8:9] -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[2:3], v[10:11] -; GFX10-NEXT: v_cmp_o_f64_e64 s9, v[4:5], v[12:13] -; GFX10-NEXT: v_cmp_o_f64_e64 s10, v[6:7], v[14:15] -; GFX10-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v8, v10, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v10, v12, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v12, v14, v6, s7 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v13, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v15, v7, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, v16, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v1, 0x7ff80000, v1, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, v8, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v3, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, v10, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v5, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, v12, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0x7ff80000, v7, s10 +; GFX10-NEXT: v_min_f64 v[16:17], v[0:1], v[8:9] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9] +; GFX10-NEXT: v_min_f64 v[8:9], v[2:3], v[10:11] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[10:11] +; GFX10-NEXT: v_min_f64 v[10:11], v[4:5], v[12:13] +; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[4:5], v[12:13] +; GFX10-NEXT: v_min_f64 v[12:13], v[6:7], v[14:15] +; GFX10-NEXT: v_cmp_u_f64_e64 s6, v[6:7], v[14:15] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v8, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v9, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v4, v10, 0, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v11, 0x7ff80000, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v6, v12, 0, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v7, v13, 0x7ff80000, s6 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v4f64__nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX11-NEXT: v_cmp_lt_f64_e64 s1, v[2:3], v[10:11] -; GFX11-NEXT: v_cmp_lt_f64_e64 s2, v[4:5], v[12:13] -; GFX11-NEXT: v_cmp_lt_f64_e64 s3, v[6:7], v[14:15] -; GFX11-NEXT: v_cmp_o_f64_e64 s0, v[0:1], v[8:9] -; GFX11-NEXT: v_cmp_o_f64_e64 s4, v[2:3], v[10:11] -; GFX11-NEXT: v_cmp_o_f64_e64 s5, v[4:5], v[12:13] -; GFX11-NEXT: v_cmp_o_f64_e64 s6, v[6:7], v[14:15] -; GFX11-NEXT: v_dual_cndmask_b32 v16, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v8, v10, v2, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v12, v4, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v12, v14, v6, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, v3, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v13, v5, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v15, v7, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, v16, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, v8, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v4, 0, v10, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v6, 0, v12, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v1, 0x7ff80000, v1, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v3, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v5, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v7, 0x7ff80000, v7, s6 +; GFX11-NEXT: v_min_f64 v[16:17], v[0:1], v[8:9] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[8:9] +; GFX11-NEXT: v_min_f64 v[8:9], v[2:3], v[10:11] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[10:11] +; GFX11-NEXT: v_min_f64 v[10:11], v[4:5], v[12:13] +; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[12:13] +; GFX11-NEXT: v_min_f64 v[12:13], v[6:7], v[14:15] +; GFX11-NEXT: v_cmp_u_f64_e64 s2, v[6:7], v[14:15] +; GFX11-NEXT: v_cndmask_b32_e64 v0, v16, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v17, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v8, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v9, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v4, v10, 0, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v5, v11, 0x7ff80000, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v6, v12, 0, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v7, v13, 0x7ff80000, s2 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v4f64__nsz: @@ -3351,108 +1730,55 @@ define <4 x double> @v_minimum_v4f64__nnan_nsz(<4 x double> %src0, <4 x double> ; GFX7-LABEL: v_minimum_v4f64__nnan_nsz: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], v[2:3], v[10:11] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[6:7], v[4:5], v[12:13] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[8:9], v[6:7], v[14:15] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v6, v14, v6, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v15, v7, s[8:9] +; GFX7-NEXT: v_min_f64 v[0:1], v[0:1], v[8:9] +; GFX7-NEXT: v_min_f64 v[2:3], v[2:3], v[10:11] +; GFX7-NEXT: v_min_f64 v[4:5], v[4:5], v[12:13] +; GFX7-NEXT: v_min_f64 v[6:7], v[6:7], v[14:15] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v4f64__nnan_nsz: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[4:5], v[2:3], v[10:11] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[6:7], v[4:5], v[12:13] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[8:9], v[6:7], v[14:15] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v6, v14, v6, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v15, v7, s[8:9] +; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[8:9] +; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[10:11] +; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[12:13] +; GFX8-NEXT: v_min_f64 v[6:7], v[6:7], v[14:15] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v4f64__nnan_nsz: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[4:5], v[2:3], v[10:11] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[6:7], v[4:5], v[12:13] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[8:9], v[6:7], v[14:15] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v14, v6, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v15, v7, s[8:9] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[8:9] +; GFX9-NEXT: v_min_f64 v[2:3], v[2:3], v[10:11] +; GFX9-NEXT: v_min_f64 v[4:5], v[4:5], v[12:13] +; GFX9-NEXT: v_min_f64 v[6:7], v[6:7], v[14:15] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v4f64__nnan_nsz: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[8:9] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[10:11] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[12:13] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[14:15] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v14, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v7, v15, v7, vcc +; GFX940-NEXT: v_min_f64 v[0:1], v[0:1], v[8:9] +; GFX940-NEXT: v_min_f64 v[2:3], v[2:3], v[10:11] +; GFX940-NEXT: v_min_f64 v[4:5], v[4:5], v[12:13] +; GFX940-NEXT: v_min_f64 v[6:7], v[6:7], v[14:15] ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v4f64__nnan_nsz: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[2:3], v[10:11] -; GFX10-NEXT: v_cmp_lt_f64_e64 s5, v[4:5], v[12:13] -; GFX10-NEXT: v_cmp_lt_f64_e64 s6, v[6:7], v[14:15] -; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v12, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v14, v6, s6 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v13, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v15, v7, s6 +; GFX10-NEXT: v_min_f64 v[0:1], v[0:1], v[8:9] +; GFX10-NEXT: v_min_f64 v[2:3], v[2:3], v[10:11] +; GFX10-NEXT: v_min_f64 v[4:5], v[4:5], v[12:13] +; GFX10-NEXT: v_min_f64 v[6:7], v[6:7], v[14:15] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v4f64__nnan_nsz: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[8:9] -; GFX11-NEXT: v_cmp_lt_f64_e64 s0, v[2:3], v[10:11] -; GFX11-NEXT: v_cmp_lt_f64_e64 s1, v[4:5], v[12:13] -; GFX11-NEXT: v_cmp_lt_f64_e64 s2, v[6:7], v[14:15] -; GFX11-NEXT: v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v12, v4, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v6, v14, v6, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v13, v5, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v15, v7, s2 +; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[8:9] +; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[10:11] +; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[12:13] +; GFX11-NEXT: v_min_f64 v[6:7], v[6:7], v[14:15] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v4f64__nnan_nsz: @@ -3475,782 +1801,244 @@ define <8 x double> @v_minimum_v8f64(<8 x double> %src0, <8 x double> %src1) { ; GFX7-LABEL: v_minimum_v8f64: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[16:17] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[16:17] -; GFX7-NEXT: v_mov_b32_e32 v32, 0x7ff80000 -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[4:5], v[20:21] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[8:9], v[24:25] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[10:11], v[12:13], v[28:29] -; GFX7-NEXT: v_cmp_o_f64_e64 s[12:13], v[12:13], v[28:29] -; GFX7-NEXT: v_cndmask_b32_e32 v31, v17, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v34, v32, v31, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v31, v16, v0, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v33, 0, v31, s[4:5] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[18:19] ; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v34, v1, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[33:34] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v34, v1, vcc -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[18:19] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[4:5] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], v[4:5], v[20:21] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[18:19], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[16:17] -; GFX7-NEXT: v_cndmask_b32_e64 v18, v21, v5, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v16, v20, v4, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[20:21], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v17, v32, v18, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[6:7] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[6:7], v[22:23] -; GFX7-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v5, v17, v5, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v20, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v21, s[4:5] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[16:17] -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[22:23] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v16, v4, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[4:5] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], v[8:9], v[24:25] -; GFX7-NEXT: v_cndmask_b32_e32 v18, v23, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v18, 0, v16, s[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[22:23], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v16, v25, v9, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v16, v24, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v18, v6, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[18:19] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[8:9], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, v22, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v23, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[8:9] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[24:25], 32 -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[10:11], v[26:27] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v18, v6, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[10:11], v[26:27] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[4:5] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[16:17] -; GFX7-NEXT: v_cndmask_b32_e64 v8, v8, v24, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v9, v25, s[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[26:27], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v18, v27, v11, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v18, 0, v16, s[8:9] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[18:19] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[4:5] +; GFX7-NEXT: v_min_f64 v[32:33], v[2:3], v[18:19] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[18:19] +; GFX7-NEXT: v_min_f64 v[18:19], v[4:5], v[20:21] +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[4:5], v[20:21] +; GFX7-NEXT: v_min_f64 v[2:3], v[0:1], v[16:17] +; GFX7-NEXT: v_cmp_u_f64_e64 s[8:9], v[0:1], v[16:17] +; GFX7-NEXT: v_mov_b32_e32 v34, 0x7ff80000 +; GFX7-NEXT: v_min_f64 v[20:21], v[6:7], v[22:23] +; GFX7-NEXT: v_cmp_u_f64_e64 s[6:7], v[6:7], v[22:23] +; GFX7-NEXT: v_min_f64 v[16:17], v[8:9], v[24:25] +; GFX7-NEXT: v_cmp_u_f64_e64 s[10:11], v[8:9], v[24:25] +; GFX7-NEXT: v_min_f64 v[22:23], v[10:11], v[26:27] +; GFX7-NEXT: v_cmp_u_f64_e64 s[12:13], v[10:11], v[26:27] +; GFX7-NEXT: v_min_f64 v[24:25], v[12:13], v[28:29] +; GFX7-NEXT: v_cmp_u_f64_e64 s[14:15], v[12:13], v[28:29] +; GFX7-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[8:9] +; GFX7-NEXT: v_cndmask_b32_e64 v1, v3, v34, s[8:9] +; GFX7-NEXT: v_cndmask_b32_e64 v2, v32, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v3, v33, v34, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v4, v18, 0, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v19, v34, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v6, v20, 0, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v7, v21, v34, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v8, v16, 0, s[10:11] +; GFX7-NEXT: v_cndmask_b32_e64 v9, v17, v34, s[10:11] +; GFX7-NEXT: v_cndmask_b32_e64 v10, v22, 0, s[12:13] +; GFX7-NEXT: v_cndmask_b32_e64 v11, v23, v34, s[12:13] +; GFX7-NEXT: v_cndmask_b32_e64 v12, v24, 0, s[14:15] +; GFX7-NEXT: v_cndmask_b32_e64 v13, v25, v34, s[14:15] ; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[14:15], v[30:31] -; GFX7-NEXT: v_cndmask_b32_e64 v16, v29, v13, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v16, v28, v12, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e32 v10, v18, v10, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v11, v19, v11, vcc -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[14:15], v[30:31] -; GFX7-NEXT: v_cndmask_b32_e64 v10, v10, v26, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v27, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v10, v18, v10, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v11, v19, v11, s[8:9] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[12:13], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[12:13] -; GFX7-NEXT: v_cmp_class_f64_e64 s[8:9], v[28:29], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v18, v31, v15, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v18, v30, v14, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v18, 0, v18, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[30:31], 32 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[16:17] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[18:19] -; GFX7-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v12, v12, v28, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v14, v18, v14, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v15, v19, v15, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v14, v14, v30, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v13, v13, v29, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v15, v15, v31, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v14, v18, v14, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v15, v19, v15, s[12:13] +; GFX7-NEXT: v_min_f64 v[18:19], v[14:15], v[30:31] +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[14:15], v[30:31] +; GFX7-NEXT: v_cndmask_b32_e64 v14, v18, 0, vcc +; GFX7-NEXT: v_cndmask_b32_e32 v15, v19, v34, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v8f64: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[16:17] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[16:17] -; GFX8-NEXT: v_mov_b32_e32 v32, 0x7ff80000 -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[4:5], v[20:21] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[8:9], v[24:25] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[10:11], v[12:13], v[28:29] -; GFX8-NEXT: v_cmp_o_f64_e64 s[12:13], v[12:13], v[28:29] -; GFX8-NEXT: v_cndmask_b32_e32 v31, v17, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v34, v32, v31, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v31, v16, v0, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v33, 0, v31, s[4:5] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[18:19] ; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v34, v1, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[33:34] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v34, v1, vcc -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[18:19] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[4:5] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[4:5], v[4:5], v[20:21] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[18:19], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[16:17] -; GFX8-NEXT: v_cndmask_b32_e64 v18, v21, v5, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v16, v20, v4, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[20:21], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v17, v32, v18, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[6:7] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[6:7], v[22:23] -; GFX8-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v5, v17, v5, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v20, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v21, s[4:5] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[16:17] -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[22:23] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v16, v4, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[4:5] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[4:5], v[8:9], v[24:25] -; GFX8-NEXT: v_cndmask_b32_e32 v18, v23, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v18, 0, v16, s[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[22:23], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v16, v25, v9, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v16, v24, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v18, v6, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[18:19] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[8:9], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v22, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v23, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[8:9] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[24:25], 32 -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[10:11], v[26:27] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v18, v6, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[10:11], v[26:27] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[4:5] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[16:17] -; GFX8-NEXT: v_cndmask_b32_e64 v8, v8, v24, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v9, v25, s[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[26:27], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v18, v27, v11, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v18, 0, v16, s[8:9] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[18:19] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[4:5] +; GFX8-NEXT: v_min_f64 v[32:33], v[2:3], v[18:19] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[18:19] +; GFX8-NEXT: v_min_f64 v[18:19], v[4:5], v[20:21] +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[4:5], v[20:21] +; GFX8-NEXT: v_min_f64 v[2:3], v[0:1], v[16:17] +; GFX8-NEXT: v_cmp_u_f64_e64 s[8:9], v[0:1], v[16:17] +; GFX8-NEXT: v_mov_b32_e32 v34, 0x7ff80000 +; GFX8-NEXT: v_min_f64 v[20:21], v[6:7], v[22:23] +; GFX8-NEXT: v_cmp_u_f64_e64 s[6:7], v[6:7], v[22:23] +; GFX8-NEXT: v_min_f64 v[16:17], v[8:9], v[24:25] +; GFX8-NEXT: v_cmp_u_f64_e64 s[10:11], v[8:9], v[24:25] +; GFX8-NEXT: v_min_f64 v[22:23], v[10:11], v[26:27] +; GFX8-NEXT: v_cmp_u_f64_e64 s[12:13], v[10:11], v[26:27] +; GFX8-NEXT: v_min_f64 v[24:25], v[12:13], v[28:29] +; GFX8-NEXT: v_cmp_u_f64_e64 s[14:15], v[12:13], v[28:29] +; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[8:9] +; GFX8-NEXT: v_cndmask_b32_e64 v1, v3, v34, s[8:9] +; GFX8-NEXT: v_cndmask_b32_e64 v2, v32, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v3, v33, v34, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v4, v18, 0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v19, v34, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v6, v20, 0, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v7, v21, v34, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v8, v16, 0, s[10:11] +; GFX8-NEXT: v_cndmask_b32_e64 v9, v17, v34, s[10:11] +; GFX8-NEXT: v_cndmask_b32_e64 v10, v22, 0, s[12:13] +; GFX8-NEXT: v_cndmask_b32_e64 v11, v23, v34, s[12:13] +; GFX8-NEXT: v_cndmask_b32_e64 v12, v24, 0, s[14:15] +; GFX8-NEXT: v_cndmask_b32_e64 v13, v25, v34, s[14:15] ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[14:15], v[30:31] -; GFX8-NEXT: v_cndmask_b32_e64 v16, v29, v13, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v16, v28, v12, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e32 v10, v18, v10, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v11, v19, v11, vcc -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[14:15], v[30:31] -; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, v26, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v27, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v10, v18, v10, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v11, v19, v11, s[8:9] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[12:13], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[12:13] -; GFX8-NEXT: v_cmp_class_f64_e64 s[8:9], v[28:29], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v18, v31, v15, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v18, v30, v14, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v18, 0, v18, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[30:31], 32 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[16:17] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[18:19] -; GFX8-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v12, v12, v28, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v14, v18, v14, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v15, v19, v15, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v14, v14, v30, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v13, v13, v29, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v15, v15, v31, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v14, v18, v14, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v15, v19, v15, s[12:13] +; GFX8-NEXT: v_min_f64 v[18:19], v[14:15], v[30:31] +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[14:15], v[30:31] +; GFX8-NEXT: v_cndmask_b32_e64 v14, v18, 0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v15, v19, v34, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v8f64: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[16:17] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[16:17] -; GFX9-NEXT: v_mov_b32_e32 v32, 0x7ff80000 -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[4:5], v[20:21] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[8:9], v[24:25] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[10:11], v[12:13], v[28:29] -; GFX9-NEXT: v_cmp_o_f64_e64 s[12:13], v[12:13], v[28:29] -; GFX9-NEXT: v_cndmask_b32_e32 v31, v17, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v34, v32, v31, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v31, v16, v0, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v33, 0, v31, s[4:5] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[2:3], v[18:19] ; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v34, v1, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v17, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[33:34] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v33, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v34, v1, vcc -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[18:19] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[4:5] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[4:5], v[4:5], v[20:21] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[18:19], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[16:17] -; GFX9-NEXT: v_cndmask_b32_e64 v18, v21, v5, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v16, v20, v4, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[20:21], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v17, v32, v18, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[6:7] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[6:7], v[22:23] -; GFX9-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v5, v17, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v20, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v21, s[4:5] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[16:17] -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[22:23] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v16, v4, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[4:5] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[4:5], v[8:9], v[24:25] -; GFX9-NEXT: v_cndmask_b32_e32 v18, v23, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v18, 0, v16, s[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[22:23], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v16, v25, v9, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v16, v24, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v18, v6, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[18:19] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[8:9], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v22, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v23, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[8:9] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[24:25], 32 -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[10:11], v[26:27] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v18, v6, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[10:11], v[26:27] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[4:5] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[16:17] -; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v24, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v25, s[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[26:27], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v18, v27, v11, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v18, 0, v16, s[8:9] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[8:9], 0, v[18:19] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[4:5] +; GFX9-NEXT: v_min_f64 v[32:33], v[2:3], v[18:19] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[18:19] +; GFX9-NEXT: v_min_f64 v[18:19], v[4:5], v[20:21] +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[4:5], v[20:21] +; GFX9-NEXT: v_min_f64 v[2:3], v[0:1], v[16:17] +; GFX9-NEXT: v_cmp_u_f64_e64 s[8:9], v[0:1], v[16:17] +; GFX9-NEXT: v_mov_b32_e32 v34, 0x7ff80000 +; GFX9-NEXT: v_min_f64 v[20:21], v[6:7], v[22:23] +; GFX9-NEXT: v_cmp_u_f64_e64 s[6:7], v[6:7], v[22:23] +; GFX9-NEXT: v_min_f64 v[16:17], v[8:9], v[24:25] +; GFX9-NEXT: v_cmp_u_f64_e64 s[10:11], v[8:9], v[24:25] +; GFX9-NEXT: v_min_f64 v[22:23], v[10:11], v[26:27] +; GFX9-NEXT: v_cmp_u_f64_e64 s[12:13], v[10:11], v[26:27] +; GFX9-NEXT: v_min_f64 v[24:25], v[12:13], v[28:29] +; GFX9-NEXT: v_cmp_u_f64_e64 s[14:15], v[12:13], v[28:29] +; GFX9-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[8:9] +; GFX9-NEXT: v_cndmask_b32_e64 v1, v3, v34, s[8:9] +; GFX9-NEXT: v_cndmask_b32_e64 v2, v32, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v3, v33, v34, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v4, v18, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v19, v34, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v6, v20, 0, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v21, v34, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v8, v16, 0, s[10:11] +; GFX9-NEXT: v_cndmask_b32_e64 v9, v17, v34, s[10:11] +; GFX9-NEXT: v_cndmask_b32_e64 v10, v22, 0, s[12:13] +; GFX9-NEXT: v_cndmask_b32_e64 v11, v23, v34, s[12:13] +; GFX9-NEXT: v_cndmask_b32_e64 v12, v24, 0, s[14:15] +; GFX9-NEXT: v_cndmask_b32_e64 v13, v25, v34, s[14:15] ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[14:15], v[30:31] -; GFX9-NEXT: v_cndmask_b32_e64 v16, v29, v13, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v16, v28, v12, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e32 v10, v18, v10, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v11, v19, v11, vcc -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[14:15], v[30:31] -; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v26, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v27, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v10, v18, v10, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v11, v19, v11, s[8:9] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[12:13], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[12:13] -; GFX9-NEXT: v_cmp_class_f64_e64 s[8:9], v[28:29], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v18, v31, v15, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v19, v32, v18, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v18, v30, v14, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v18, 0, v18, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[30:31], 32 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[16:17] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[18:19] -; GFX9-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, v28, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v14, v18, v14, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v15, v19, v15, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v14, v14, v30, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v13, v13, v29, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v15, v15, v31, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v14, v18, v14, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v15, v19, v15, s[12:13] +; GFX9-NEXT: v_min_f64 v[18:19], v[14:15], v[30:31] +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[14:15], v[30:31] +; GFX9-NEXT: v_cndmask_b32_e64 v14, v18, 0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v15, v19, v34, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v8f64: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX940-NEXT: scratch_load_dword v31, off, s32 -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[16:17] -; GFX940-NEXT: v_mov_b32_e32 v32, 0x7ff80000 -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v33, v17, v1, vcc -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v35, v32, v33, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v33, v16, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v34, 0, v33, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[16:17], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[34:35] -; GFX940-NEXT: v_cndmask_b32_e32 v0, v34, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v35, v1, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[18:19] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v17, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v19, v3, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[18:19] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v34, v0, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v35, v1, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v18, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[18:19], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v2, v2, v18, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[20:21] -; GFX940-NEXT: v_cndmask_b32_e64 v2, v16, v2, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v3, v19, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v21, v5, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[4:5], v[20:21] -; GFX940-NEXT: v_cndmask_b32_e64 v3, v17, v3, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v20, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[20:21], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, v4, v20, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v5, v17, v5, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[22:23] -; GFX940-NEXT: v_cndmask_b32_e64 v4, v16, v4, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v5, v21, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v23, v7, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[6:7], v[22:23] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v22, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[22:23], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v6, v6, v22, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v7, v17, v7, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[8:9], v[24:25] -; GFX940-NEXT: v_cndmask_b32_e64 v6, v16, v6, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v7, v23, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v25, v9, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[8:9], v[24:25] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v17, v7, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v24, v8, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[8:9], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[24:25], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v8, v8, v24, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v9, v17, v9, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[10:11], v[26:27] -; GFX940-NEXT: v_cndmask_b32_e64 v8, v16, v8, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v9, v9, v25, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v27, v11, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[10:11], v[26:27] -; GFX940-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v26, v10, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[26:27], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v10, v16, v10, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v10, v10, v26, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v11, v17, v11, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[12:13], v[28:29] -; GFX940-NEXT: v_cndmask_b32_e64 v10, v16, v10, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v11, v11, v27, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v29, v13, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[12:13], v[28:29] -; GFX940-NEXT: v_cndmask_b32_e64 v11, v17, v11, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v28, v12, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[12:13], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[28:29], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v12, v16, v12, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v12, v12, v28, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v13, v17, v13, vcc +; GFX940-NEXT: v_mov_b32_e32 v54, 0x7ff80000 +; GFX940-NEXT: v_min_f64 v[32:33], v[0:1], v[16:17] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[16:17] +; GFX940-NEXT: v_min_f64 v[34:35], v[2:3], v[18:19] +; GFX940-NEXT: v_min_f64 v[36:37], v[4:5], v[20:21] +; GFX940-NEXT: v_cndmask_b32_e64 v0, v32, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v1, v33, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[18:19] +; GFX940-NEXT: v_min_f64 v[38:39], v[6:7], v[22:23] +; GFX940-NEXT: v_min_f64 v[48:49], v[8:9], v[24:25] +; GFX940-NEXT: v_cndmask_b32_e64 v2, v34, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v35, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[4:5], v[20:21] +; GFX940-NEXT: v_min_f64 v[50:51], v[10:11], v[26:27] +; GFX940-NEXT: v_min_f64 v[52:53], v[12:13], v[28:29] +; GFX940-NEXT: v_cndmask_b32_e64 v4, v36, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v5, v37, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[6:7], v[22:23] ; GFX940-NEXT: s_waitcnt vmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[14:15], v[30:31] -; GFX940-NEXT: v_cndmask_b32_e64 v12, v16, v12, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v13, v13, v29, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v31, v15, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[14:15], v[30:31] -; GFX940-NEXT: v_cndmask_b32_e64 v13, v17, v13, s[2:3] -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e64 v17, v32, v16, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v30, v14, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, 0, v16, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[30:31], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[16:17] -; GFX940-NEXT: v_cndmask_b32_e32 v14, v16, v14, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v15, v17, v15, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v14, v14, v30, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v15, v15, v31, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v14, v16, v14, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v15, v17, v15, s[2:3] +; GFX940-NEXT: v_min_f64 v[16:17], v[14:15], v[30:31] +; GFX940-NEXT: v_cndmask_b32_e64 v6, v38, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v7, v39, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[8:9], v[24:25] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v8, v48, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v9, v49, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[10:11], v[26:27] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v10, v50, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v11, v51, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[12:13], v[28:29] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v12, v52, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v13, v53, v54, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[14:15], v[30:31] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v14, v16, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v15, v17, v54, vcc ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v8f64: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[16:17] -; GFX10-NEXT: v_cmp_o_f64_e64 s4, v[0:1], v[16:17] -; GFX10-NEXT: v_cmp_lt_f64_e64 s9, v[6:7], v[22:23] -; GFX10-NEXT: v_cmp_lt_f64_e64 s10, v[8:9], v[24:25] -; GFX10-NEXT: v_cmp_lt_f64_e64 s11, v[10:11], v[26:27] -; GFX10-NEXT: v_cmp_lt_f64_e64 s12, v[12:13], v[28:29] -; GFX10-NEXT: v_cmp_o_f64_e64 s13, v[6:7], v[22:23] -; GFX10-NEXT: v_cmp_o_f64_e64 s14, v[8:9], v[24:25] -; GFX10-NEXT: v_cmp_o_f64_e64 s15, v[10:11], v[26:27] -; GFX10-NEXT: v_cmp_o_f64_e64 s16, v[12:13], v[28:29] -; GFX10-NEXT: v_cmp_lt_f64_e64 s5, v[2:3], v[18:19] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[2:3], v[18:19] -; GFX10-NEXT: v_cmp_lt_f64_e64 s7, v[4:5], v[20:21] -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[4:5], v[20:21] -; GFX10-NEXT: v_cmp_class_f64_e64 s17, v[26:27], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s18, v[28:29], 32 -; GFX10-NEXT: v_cndmask_b32_e32 v32, v17, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v38, v23, v7, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v48, v25, v9, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v50, v27, v11, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v33, 0x7ff80000, v32, s4 -; GFX10-NEXT: v_cndmask_b32_e32 v32, v16, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v52, v29, v13, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v39, 0x7ff80000, v38, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v49, 0x7ff80000, v48, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v32, 0, v32, s4 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[2:3], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v51, 0x7ff80000, v50, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v53, 0x7ff80000, v52, s16 -; GFX10-NEXT: v_cndmask_b32_e64 v38, v22, v6, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v48, v24, v8, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v50, v26, v10, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v52, v28, v12, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 s11, v[16:17], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s12, v[18:19], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v34, v19, v3, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v38, 0, v38, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v36, v21, v5, s7 -; GFX10-NEXT: v_cmp_class_f64_e64 s9, v[12:13], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v48, 0, v48, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v35, 0x7ff80000, v34, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v34, v18, v2, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v37, 0x7ff80000, v36, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v36, v20, v4, s7 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[4:5], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v34, 0, v34, s6 -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[6:7], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v36, 0, v36, s8 -; GFX10-NEXT: v_cmp_class_f64_e64 s7, v[8:9], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s8, v[10:11], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v34, v2, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v50, 0, v50, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v52, 0, v52, s16 -; GFX10-NEXT: v_cmp_class_f64_e64 s14, v[20:21], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v16, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v18, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 s15, v[22:23], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s16, v[24:25], 32 -; GFX10-NEXT: v_cmp_eq_f64_e64 s19, 0, v[32:33] -; GFX10-NEXT: v_cmp_eq_f64_e64 s20, 0, v[34:35] -; GFX10-NEXT: v_cmp_eq_f64_e64 s21, 0, v[36:37] -; GFX10-NEXT: v_cmp_eq_f64_e64 s22, 0, v[48:49] -; GFX10-NEXT: v_cmp_eq_f64_e64 s23, 0, v[50:51] -; GFX10-NEXT: v_cmp_eq_f64_e64 s24, 0, v[52:53] -; GFX10-NEXT: v_cndmask_b32_e32 v1, v33, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v12, v52, v12, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v36, v4, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v35, v3, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v38, v6, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v37, v5, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v8, v48, v8, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v10, v50, v10, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v39, v7, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v9, v49, v9, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v51, v11, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v53, v13, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v20, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v26, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v22, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, v24, s16 -; GFX10-NEXT: v_cndmask_b32_e64 v12, v12, v28, s18 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v17, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v19, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v21, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v23, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v25, s16 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v11, v27, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v13, v29, s18 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v32, v0, s19 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v34, v2, s20 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v36, v4, s21 -; GFX10-NEXT: v_cndmask_b32_e64 v8, v48, v8, s22 -; GFX10-NEXT: v_cndmask_b32_e64 v10, v50, v10, s23 -; GFX10-NEXT: v_cndmask_b32_e64 v12, v52, v12, s24 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v33, v1, s19 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v35, v3, s20 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v37, v5, s21 -; GFX10-NEXT: v_cndmask_b32_e64 v9, v49, v9, s22 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v51, v11, s23 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v53, v13, s24 +; GFX10-NEXT: v_min_f64 v[32:33], v[0:1], v[16:17] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[16:17] +; GFX10-NEXT: v_min_f64 v[16:17], v[2:3], v[18:19] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[2:3], v[18:19] +; GFX10-NEXT: v_min_f64 v[18:19], v[4:5], v[20:21] +; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[4:5], v[20:21] +; GFX10-NEXT: v_min_f64 v[20:21], v[6:7], v[22:23] +; GFX10-NEXT: v_cmp_u_f64_e64 s6, v[6:7], v[22:23] +; GFX10-NEXT: v_min_f64 v[22:23], v[8:9], v[24:25] +; GFX10-NEXT: v_cmp_u_f64_e64 s7, v[8:9], v[24:25] +; GFX10-NEXT: v_min_f64 v[24:25], v[10:11], v[26:27] +; GFX10-NEXT: v_cmp_u_f64_e64 s8, v[10:11], v[26:27] +; GFX10-NEXT: v_min_f64 v[26:27], v[12:13], v[28:29] +; GFX10-NEXT: v_cmp_u_f64_e64 s9, v[12:13], v[28:29] +; GFX10-NEXT: v_cndmask_b32_e64 v0, v32, 0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v33, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v16, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v17, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v4, v18, 0, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v19, 0x7ff80000, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v6, v20, 0, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v7, v21, 0x7ff80000, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v8, v22, 0, s7 +; GFX10-NEXT: v_cndmask_b32_e64 v9, v23, 0x7ff80000, s7 +; GFX10-NEXT: v_cndmask_b32_e64 v10, v24, 0, s8 +; GFX10-NEXT: v_cndmask_b32_e64 v11, v25, 0x7ff80000, s8 +; GFX10-NEXT: v_cndmask_b32_e64 v12, v26, 0, s9 +; GFX10-NEXT: v_cndmask_b32_e64 v13, v27, 0x7ff80000, s9 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_cmp_lt_f64_e64 s10, v[14:15], v[30:31] -; GFX10-NEXT: v_cmp_o_f64_e64 s13, v[14:15], v[30:31] -; GFX10-NEXT: v_cmp_class_f64_e64 s25, v[30:31], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v16, v31, v15, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v18, v30, v14, s10 -; GFX10-NEXT: v_cmp_eq_f64_e64 s10, 0, v[38:39] -; GFX10-NEXT: v_cndmask_b32_e64 v55, 0x7ff80000, v16, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v54, 0, v18, s13 -; GFX10-NEXT: v_cmp_class_f64_e64 s13, v[14:15], 32 -; GFX10-NEXT: v_cmp_eq_f64_e32 vcc_lo, 0, v[54:55] -; GFX10-NEXT: v_cndmask_b32_e64 v6, v38, v6, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v39, v7, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v14, v54, v14, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v15, v55, v15, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v14, v14, v30, s25 -; GFX10-NEXT: v_cndmask_b32_e64 v15, v15, v31, s25 -; GFX10-NEXT: v_cndmask_b32_e32 v14, v54, v14, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v15, v55, v15, vcc_lo +; GFX10-NEXT: v_min_f64 v[28:29], v[14:15], v[30:31] +; GFX10-NEXT: v_cmp_u_f64_e64 s10, v[14:15], v[30:31] +; GFX10-NEXT: v_cndmask_b32_e64 v14, v28, 0, s10 +; GFX10-NEXT: v_cndmask_b32_e64 v15, v29, 0x7ff80000, s10 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v8f64: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-NEXT: v_cmp_lt_f64_e64 s4, v[6:7], v[22:23] -; GFX11-NEXT: v_cmp_o_f64_e64 s9, v[6:7], v[22:23] -; GFX11-NEXT: v_cmp_lt_f64_e64 s1, v[2:3], v[18:19] -; GFX11-NEXT: v_cmp_lt_f64_e64 s6, v[10:11], v[26:27] -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[16:17] -; GFX11-NEXT: v_cmp_o_f64_e64 s2, v[2:3], v[18:19] -; GFX11-NEXT: v_cmp_o_f64_e64 s11, v[10:11], v[26:27] -; GFX11-NEXT: v_cmp_o_f64_e64 s0, v[0:1], v[16:17] -; GFX11-NEXT: v_cmp_lt_f64_e64 s3, v[4:5], v[20:21] -; GFX11-NEXT: v_cmp_lt_f64_e64 s5, v[8:9], v[24:25] -; GFX11-NEXT: v_cmp_lt_f64_e64 s7, v[12:13], v[28:29] -; GFX11-NEXT: v_cmp_o_f64_e64 s8, v[4:5], v[20:21] -; GFX11-NEXT: v_cmp_o_f64_e64 s10, v[8:9], v[24:25] -; GFX11-NEXT: v_cmp_o_f64_e64 s12, v[12:13], v[28:29] -; GFX11-NEXT: v_cmp_class_f64_e64 s13, v[18:19], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s15, v[20:21], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v38, v23, v7, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v34, v19, v3, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v50, v27, v11, s6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v39, 0x7ff80000, v38, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v38, v22, v6, s4 -; GFX11-NEXT: v_cmp_class_f64_e64 s4, v[6:7], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v35, 0x7ff80000, v34, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v51, 0x7ff80000, v50, s11 -; GFX11-NEXT: v_cndmask_b32_e64 v34, v18, v2, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v50, v26, v10, s6 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[0:1], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v36, v21, v5, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v48, v25, v9, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v52, v29, v13, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v50, 0, v50, s11 -; GFX11-NEXT: v_cmp_class_f64_e64 s11, v[16:17], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v37, 0x7ff80000, v36, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v49, 0x7ff80000, v48, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v53, 0x7ff80000, v52, s12 -; GFX11-NEXT: v_cndmask_b32_e64 v36, v20, v4, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v48, v24, v8, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v52, v28, v12, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v34, 0, v34, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v38, 0, v38, s9 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[2:3], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s3, v[4:5], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s5, v[8:9], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s7, v[10:11], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s9, v[12:13], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v36, 0, v36, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v48, 0, v48, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v52, 0, v52, s12 -; GFX11-NEXT: v_cmp_class_f64_e64 s6, v[24:25], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s8, v[26:27], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s10, v[28:29], 32 -; GFX11-NEXT: v_cmp_eq_f64_e64 s14, 0, v[34:35] -; GFX11-NEXT: v_cmp_eq_f64_e64 s16, 0, v[36:37] -; GFX11-NEXT: v_cmp_eq_f64_e64 s17, 0, v[38:39] -; GFX11-NEXT: v_cmp_eq_f64_e64 s18, 0, v[48:49] -; GFX11-NEXT: v_cmp_eq_f64_e64 s20, 0, v[50:51] -; GFX11-NEXT: v_cmp_eq_f64_e64 s21, 0, v[52:53] -; GFX11-NEXT: v_cndmask_b32_e64 v7, v39, v7, s4 -; GFX11-NEXT: v_cndmask_b32_e32 v32, v17, v1, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v6, v38, v6, s4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v33, 0x7ff80000, v32, s0 -; GFX11-NEXT: v_cndmask_b32_e32 v32, v16, v0, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v1, v33, v1, s1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v32, 0, v32, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v17, s11 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v0, v32, v0, s1 -; GFX11-NEXT: v_cmp_eq_f64_e64 s12, 0, v[32:33] -; GFX11-NEXT: v_cndmask_b32_e64 v2, v34, v2, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v36, v4, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v8, v48, v8, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v16, s11 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v50, v10, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v12, v52, v12, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v35, v3, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v37, v5, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v9, v49, v9, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v11, v51, v11, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v13, v53, v13, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v18, s13 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v20, s15 -; GFX11-NEXT: v_cndmask_b32_e64 v8, v8, v24, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v26, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v12, v12, v28, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v19, s13 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v21, s15 -; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v25, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v11, v11, v27, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v13, v13, v29, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v34, v2, s14 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v36, v4, s16 -; GFX11-NEXT: v_cndmask_b32_e64 v8, v48, v8, s18 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v50, v10, s20 -; GFX11-NEXT: v_cndmask_b32_e64 v12, v52, v12, s21 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v35, v3, s14 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v37, v5, s16 -; GFX11-NEXT: v_cndmask_b32_e64 v9, v49, v9, s18 -; GFX11-NEXT: v_cndmask_b32_e64 v11, v51, v11, s20 -; GFX11-NEXT: v_cndmask_b32_e64 v13, v53, v13, s21 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v32, v0, s12 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v33, v1, s12 +; GFX11-NEXT: v_min_f64 v[32:33], v[0:1], v[16:17] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[16:17] +; GFX11-NEXT: v_min_f64 v[16:17], v[2:3], v[18:19] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[18:19] +; GFX11-NEXT: v_min_f64 v[18:19], v[4:5], v[20:21] +; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[20:21] +; GFX11-NEXT: v_min_f64 v[20:21], v[6:7], v[22:23] +; GFX11-NEXT: v_cmp_u_f64_e64 s2, v[6:7], v[22:23] +; GFX11-NEXT: v_min_f64 v[22:23], v[8:9], v[24:25] +; GFX11-NEXT: v_cmp_u_f64_e64 s3, v[8:9], v[24:25] +; GFX11-NEXT: v_min_f64 v[24:25], v[10:11], v[26:27] +; GFX11-NEXT: v_cmp_u_f64_e64 s4, v[10:11], v[26:27] +; GFX11-NEXT: v_min_f64 v[26:27], v[12:13], v[28:29] +; GFX11-NEXT: v_cmp_u_f64_e64 s5, v[12:13], v[28:29] +; GFX11-NEXT: v_cndmask_b32_e64 v0, v32, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v33, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v16, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v17, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v4, v18, 0, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v5, v19, 0x7ff80000, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v6, v20, 0, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v7, v21, 0x7ff80000, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v8, v22, 0, s3 +; GFX11-NEXT: v_cndmask_b32_e64 v9, v23, 0x7ff80000, s3 +; GFX11-NEXT: v_cndmask_b32_e64 v10, v24, 0, s4 +; GFX11-NEXT: v_cndmask_b32_e64 v11, v25, 0x7ff80000, s4 +; GFX11-NEXT: v_cndmask_b32_e64 v12, v26, 0, s5 +; GFX11-NEXT: v_cndmask_b32_e64 v13, v27, 0x7ff80000, s5 ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[14:15], v[30:31] -; GFX11-NEXT: v_cmp_o_f64_e64 s0, v[14:15], v[30:31] -; GFX11-NEXT: v_cmp_class_f64_e64 s19, v[30:31], 32 -; GFX11-NEXT: v_cndmask_b32_e32 v54, v31, v15, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e32 v16, v30, v14, vcc_lo -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[22:23], 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v55, 0x7ff80000, v54, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v54, 0, v16, s0 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[14:15], 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cmp_eq_f64_e64 s22, 0, v[54:55] -; GFX11-NEXT: v_dual_cndmask_b32 v7, v7, v23 :: v_dual_cndmask_b32 v6, v6, v22 -; GFX11-NEXT: v_cndmask_b32_e64 v14, v54, v14, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v15, v55, v15, s0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v6, v38, v6, s17 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v39, v7, s17 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v14, v14, v30, s19 -; GFX11-NEXT: v_cndmask_b32_e64 v15, v15, v31, s19 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v14, v54, v14, s22 -; GFX11-NEXT: v_cndmask_b32_e64 v15, v55, v15, s22 +; GFX11-NEXT: v_min_f64 v[28:29], v[14:15], v[30:31] +; GFX11-NEXT: v_cmp_u_f64_e64 s6, v[14:15], v[30:31] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_cndmask_b32_e64 v14, v28, 0, s6 +; GFX11-NEXT: v_cndmask_b32_e64 v15, v29, 0x7ff80000, s6 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v8f64: @@ -4279,1799 +2067,798 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1) ; GFX7-LABEL: v_minimum_v16f64: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX7-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill +; GFX7-NEXT: s_mov_b64 exec, s[4:5] ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 ; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4 -; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:16 -; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12 -; GFX7-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:24 -; GFX7-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:20 -; GFX7-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:32 -; GFX7-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:28 -; GFX7-NEXT: v_mov_b32_e32 v39, 0x7ff80000 -; GFX7-NEXT: s_waitcnt vmcnt(6) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[31:32] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[31:32] -; GFX7-NEXT: s_waitcnt vmcnt(4) -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[2:3], v[33:34] -; GFX7-NEXT: v_cndmask_b32_e32 v48, v32, v1, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v49, v39, v48, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v48, v31, v0, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[31:32], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v0, v48, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v49, v1, vcc -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[33:34] -; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v31, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v32, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v50, v34, v3, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v31, v33, v2, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v32, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v31, 0, v31, s[6:7] -; GFX7-NEXT: s_waitcnt vmcnt(2) -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[4:5], v[35:36] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v31, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v32, v3, vcc -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[35:36] -; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v33, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v34, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[35:36], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v50, v36, v5, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v33, v35, v4, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v34, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v33, 0, v33, s[6:7] +; GFX7-NEXT: v_writelane_b32 v34, s30, 0 +; GFX7-NEXT: v_writelane_b32 v34, s31, 1 +; GFX7-NEXT: v_writelane_b32 v34, s34, 2 +; GFX7-NEXT: v_writelane_b32 v34, s35, 3 ; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[6:7], v[37:38] -; GFX7-NEXT: v_cndmask_b32_e32 v4, v33, v4, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v5, v34, v5, vcc -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[37:38] -; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v35, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v36, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v50, v38, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v35, v37, v6, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v36, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v35, v6, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v7, v36, v7, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, v37, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v38, s[4:5] -; GFX7-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36 -; GFX7-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:40 -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX7-NEXT: v_cndmask_b32_e32 v0, v48, v0, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v1, v49, v1, vcc -; GFX7-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:48 -; GFX7-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:44 -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[31:32] -; GFX7-NEXT: v_cndmask_b32_e32 v2, v31, v2, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v3, v32, v3, vcc +; GFX7-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[31:32] +; GFX7-NEXT: v_min_f64 v[0:1], v[0:1], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:12 +; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[31:32] +; GFX7-NEXT: v_min_f64 v[2:3], v[2:3], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:20 +; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[4:5] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[31:32] +; GFX7-NEXT: v_min_f64 v[4:5], v[4:5], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:28 +; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, 0, s[6:7] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[31:32] +; GFX7-NEXT: v_min_f64 v[6:7], v[6:7], v[31:32] +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:36 +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 +; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, 0, s[8:9] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[10:11], v[8:9], v[31:32] +; GFX7-NEXT: v_min_f64 v[8:9], v[8:9], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:44 +; GFX7-NEXT: v_cndmask_b32_e64 v8, v8, 0, s[10:11] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[12:13], v[10:11], v[31:32] +; GFX7-NEXT: v_min_f64 v[10:11], v[10:11], v[31:32] ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 ; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:52 -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[33:34] -; GFX7-NEXT: v_cndmask_b32_e32 v4, v33, v4, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v5, v34, v5, vcc -; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 -; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[35:36] -; GFX7-NEXT: v_cndmask_b32_e32 v6, v35, v6, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v7, v36, v7, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[8:9], 32 -; GFX7-NEXT: s_waitcnt vmcnt(6) -; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], v[8:9], v[37:38] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[8:9], v[37:38] -; GFX7-NEXT: v_cndmask_b32_e64 v50, v38, v9, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v35, v37, v8, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v36, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v8, v35, v8, vcc -; GFX7-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[35:36] -; GFX7-NEXT: v_cndmask_b32_e32 v9, v36, v9, vcc -; GFX7-NEXT: s_waitcnt vmcnt(4) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[10:11], v[48:49] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[10:11], v[48:49] -; GFX7-NEXT: v_cndmask_b32_e64 v8, v8, v37, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v9, v38, s[4:5] -; GFX7-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:68 -; GFX7-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:72 -; GFX7-NEXT: v_cndmask_b32_e64 v8, v35, v8, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v50, v49, v11, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v35, v48, v10, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 32 -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[48:49], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v51, v39, v50, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v50, 0, v35, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v9, v36, v9, s[6:7] -; GFX7-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:80 -; GFX7-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:76 -; GFX7-NEXT: s_waitcnt vmcnt(6) -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[12:13], v[31:32] -; GFX7-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v10, v10, v48, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v49, s[4:5] -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[50:51] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], v[12:13], v[31:32] -; GFX7-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v48, v32, v13, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[12:13], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v49, v39, v48, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v48, v31, v12, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[31:32], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX7-NEXT: s_waitcnt vmcnt(4) -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[14:15], v[33:34] -; GFX7-NEXT: v_cndmask_b32_e32 v12, v48, v12, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v13, v49, v13, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX7-NEXT: v_cndmask_b32_e64 v12, v12, v31, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v13, v13, v32, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v10, v10, 0, s[12:13] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[14:15], v[12:13], v[31:32] +; GFX7-NEXT: v_min_f64 v[12:13], v[12:13], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:60 +; GFX7-NEXT: v_cndmask_b32_e64 v12, v12, 0, s[14:15] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[16:17], v[14:15], v[31:32] +; GFX7-NEXT: v_min_f64 v[14:15], v[14:15], v[31:32] +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:68 +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 +; GFX7-NEXT: v_cndmask_b32_e64 v14, v14, 0, s[16:17] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[18:19], v[16:17], v[31:32] +; GFX7-NEXT: v_min_f64 v[16:17], v[16:17], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:76 +; GFX7-NEXT: v_cndmask_b32_e64 v16, v16, 0, s[18:19] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[20:21], v[18:19], v[31:32] +; GFX7-NEXT: v_min_f64 v[18:19], v[18:19], v[31:32] ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:84 -; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], v[14:15], v[33:34] -; GFX7-NEXT: v_cndmask_b32_e32 v12, v48, v12, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v13, v49, v13, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v50, v34, v15, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v48, v33, v14, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v49, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v14, v48, v14, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v15, v49, v15, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX7-NEXT: v_cndmask_b32_e64 v14, v14, v33, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v15, v15, v34, s[4:5] -; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:96 -; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:92 -; GFX7-NEXT: v_cndmask_b32_e32 v14, v48, v14, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v15, v49, v15, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 32 -; GFX7-NEXT: s_waitcnt vmcnt(6) -; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], v[16:17], v[37:38] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[16:17], v[37:38] -; GFX7-NEXT: v_cndmask_b32_e64 v50, v38, v17, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v48, v37, v16, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v49, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v16, v48, v16, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v17, v49, v17, vcc -; GFX7-NEXT: s_waitcnt vmcnt(4) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[18:19], v[35:36] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[18:19], v[35:36] -; GFX7-NEXT: v_cndmask_b32_e64 v16, v16, v37, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v17, v17, v38, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v50, v36, v19, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v37, v35, v18, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[18:19], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v51, v39, v50, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v50, 0, v37, s[6:7] -; GFX7-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:100 -; GFX7-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:104 -; GFX7-NEXT: v_cndmask_b32_e32 v16, v48, v16, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v17, v49, v17, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[35:36], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v18, v50, v18, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v19, v51, v19, s[4:5] -; GFX7-NEXT: s_waitcnt vmcnt(4) -; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], v[20:21], v[31:32] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[20:21], v[31:32] -; GFX7-NEXT: v_cndmask_b32_e32 v18, v18, v35, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v19, v19, v36, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[50:51] -; GFX7-NEXT: v_cndmask_b32_e64 v48, v32, v21, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v35, v31, v20, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[20:21], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v36, v39, v48, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v18, v50, v18, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v19, v51, v19, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[31:32], 32 -; GFX7-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:112 -; GFX7-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:108 -; GFX7-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:120 -; GFX7-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:116 -; GFX7-NEXT: v_cndmask_b32_e64 v20, v35, v20, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v21, v36, v21, s[4:5] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[35:36] -; GFX7-NEXT: s_waitcnt vmcnt(6) -; GFX7-NEXT: v_cmp_lt_f64_e64 s[6:7], v[22:23], v[33:34] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[22:23], v[33:34] -; GFX7-NEXT: v_cndmask_b32_e32 v20, v20, v31, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v21, v21, v32, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[22:23], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v20, v35, v20, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v21, v36, v21, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v31, v34, v23, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v36, v39, v31, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v31, v33, v22, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v35, 0, v31, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc -; GFX7-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[35:36] -; GFX7-NEXT: v_cndmask_b32_e64 v22, v22, v33, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v18, v18, 0, s[20:21] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[22:23], v[20:21], v[31:32] +; GFX7-NEXT: v_min_f64 v[20:21], v[20:21], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:92 +; GFX7-NEXT: v_cndmask_b32_e64 v20, v20, 0, s[22:23] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[24:25], v[22:23], v[31:32] +; GFX7-NEXT: v_min_f64 v[22:23], v[22:23], v[31:32] +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:100 +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 +; GFX7-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[24:25] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[26:27], v[24:25], v[31:32] +; GFX7-NEXT: v_min_f64 v[24:25], v[24:25], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:108 +; GFX7-NEXT: v_cndmask_b32_e64 v24, v24, 0, s[26:27] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[28:29], v[26:27], v[31:32] +; GFX7-NEXT: v_min_f64 v[26:27], v[26:27], v[31:32] +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 +; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:116 +; GFX7-NEXT: v_cndmask_b32_e64 v26, v26, 0, s[28:29] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32] +; GFX7-NEXT: v_min_f64 v[28:29], v[28:29], v[31:32] ; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:128 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 -; GFX7-NEXT: v_cndmask_b32_e64 v23, v23, v34, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[24:25], 32 -; GFX7-NEXT: s_waitcnt vmcnt(7) -; GFX7-NEXT: v_cmp_lt_f64_e64 s[4:5], v[24:25], v[37:38] -; GFX7-NEXT: v_cmp_o_f64_e64 s[6:7], v[24:25], v[37:38] -; GFX7-NEXT: v_cndmask_b32_e64 v34, v38, v25, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v35, v39, v34, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v34, v37, v24, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e32 v24, v34, v24, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v25, v35, v25, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[37:38], 32 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[34:35] -; GFX7-NEXT: v_cndmask_b32_e32 v24, v24, v37, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v25, v25, v38, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[26:27], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v24, v34, v24, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v25, v35, v25, s[4:5] -; GFX7-NEXT: s_waitcnt vmcnt(5) -; GFX7-NEXT: v_cmp_lt_f64_e64 s[6:7], v[26:27], v[48:49] -; GFX7-NEXT: v_cmp_o_f64_e64 s[8:9], v[26:27], v[48:49] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[48:49], 32 -; GFX7-NEXT: s_waitcnt vmcnt(3) -; GFX7-NEXT: v_cmp_o_f64_e64 s[10:11], v[28:29], v[50:51] -; GFX7-NEXT: v_cndmask_b32_e64 v36, v49, v27, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v34, v48, v26, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v35, v39, v36, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[8:9] -; GFX7-NEXT: v_cmp_lt_f64_e64 s[8:9], v[28:29], v[50:51] -; GFX7-NEXT: v_cndmask_b32_e32 v26, v34, v26, vcc -; GFX7-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[34:35] -; GFX7-NEXT: v_cndmask_b32_e32 v27, v35, v27, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v26, v26, v48, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v27, v27, v49, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v36, v51, v29, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v27, v35, v27, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v28, v28, 0, s[30:31] +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33] +; GFX7-NEXT: v_min_f64 v[30:31], v[30:31], v[32:33] +; GFX7-NEXT: v_mov_b32_e32 v32, 0x7ff80000 +; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v32, vcc +; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v32, s[4:5] +; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v32, s[6:7] +; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v32, s[8:9] +; GFX7-NEXT: v_cndmask_b32_e64 v9, v9, v32, s[10:11] +; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v32, s[12:13] +; GFX7-NEXT: v_cndmask_b32_e64 v13, v13, v32, s[14:15] +; GFX7-NEXT: v_cndmask_b32_e64 v15, v15, v32, s[16:17] +; GFX7-NEXT: v_cndmask_b32_e64 v17, v17, v32, s[18:19] +; GFX7-NEXT: v_cndmask_b32_e64 v19, v19, v32, s[20:21] +; GFX7-NEXT: v_cndmask_b32_e64 v21, v21, v32, s[22:23] +; GFX7-NEXT: v_cndmask_b32_e64 v23, v23, v32, s[24:25] +; GFX7-NEXT: v_cndmask_b32_e64 v25, v25, v32, s[26:27] +; GFX7-NEXT: v_cndmask_b32_e64 v27, v27, v32, s[28:29] +; GFX7-NEXT: v_cndmask_b32_e64 v29, v29, v32, s[30:31] +; GFX7-NEXT: v_cndmask_b32_e64 v31, v31, v32, s[34:35] +; GFX7-NEXT: v_cndmask_b32_e64 v30, v30, 0, s[34:35] +; GFX7-NEXT: v_readlane_b32 s35, v34, 3 +; GFX7-NEXT: v_readlane_b32 s34, v34, 2 +; GFX7-NEXT: v_readlane_b32 s31, v34, 1 +; GFX7-NEXT: v_readlane_b32 s30, v34, 0 +; GFX7-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload +; GFX7-NEXT: s_mov_b64 exec, s[4:5] ; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_cmp_lt_f64_e32 vcc, v[30:31], v[32:33] -; GFX7-NEXT: v_cmp_o_f64_e64 s[4:5], v[30:31], v[32:33] -; GFX7-NEXT: v_cndmask_b32_e64 v35, v39, v36, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v26, v34, v26, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v34, v50, v28, s[8:9] -; GFX7-NEXT: v_cmp_class_f64_e64 s[6:7], v[28:29], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[10:11] -; GFX7-NEXT: v_cmp_class_f64_e64 s[8:9], v[50:51], 32 -; GFX7-NEXT: v_cndmask_b32_e32 v36, v33, v31, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v37, v39, v36, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e32 v36, v32, v30, vcc -; GFX7-NEXT: v_cmp_class_f64_e64 vcc, v[30:31], 32 -; GFX7-NEXT: v_cndmask_b32_e64 v36, 0, v36, s[4:5] -; GFX7-NEXT: v_cmp_class_f64_e64 s[4:5], v[32:33], 32 -; GFX7-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[34:35] -; GFX7-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[36:37] -; GFX7-NEXT: v_cndmask_b32_e64 v28, v34, v28, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v29, v35, v29, s[6:7] -; GFX7-NEXT: v_cndmask_b32_e64 v28, v28, v50, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e32 v30, v36, v30, vcc -; GFX7-NEXT: v_cndmask_b32_e32 v31, v37, v31, vcc -; GFX7-NEXT: v_cndmask_b32_e64 v30, v30, v32, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v29, v29, v51, s[8:9] -; GFX7-NEXT: v_cndmask_b32_e64 v31, v31, v33, s[4:5] -; GFX7-NEXT: v_cndmask_b32_e64 v28, v34, v28, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v29, v35, v29, s[10:11] -; GFX7-NEXT: v_cndmask_b32_e64 v30, v36, v30, s[12:13] -; GFX7-NEXT: v_cndmask_b32_e64 v31, v37, v31, s[12:13] ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_minimum_v16f64: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX8-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill +; GFX8-NEXT: s_mov_b64 exec, s[4:5] ; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 ; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4 -; GFX8-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:16 -; GFX8-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12 -; GFX8-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:24 -; GFX8-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:20 -; GFX8-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:32 -; GFX8-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:28 -; GFX8-NEXT: v_mov_b32_e32 v39, 0x7ff80000 -; GFX8-NEXT: s_waitcnt vmcnt(6) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[31:32] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[31:32] -; GFX8-NEXT: s_waitcnt vmcnt(4) -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[2:3], v[33:34] -; GFX8-NEXT: v_cndmask_b32_e32 v48, v32, v1, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v49, v39, v48, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v48, v31, v0, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[31:32], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v48, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v49, v1, vcc -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[33:34] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v31, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v32, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v50, v34, v3, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v31, v33, v2, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v32, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v31, 0, v31, s[6:7] -; GFX8-NEXT: s_waitcnt vmcnt(2) -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[4:5], v[35:36] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v31, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v32, v3, vcc -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[35:36] -; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v33, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v34, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[35:36], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v50, v36, v5, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v33, v35, v4, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v34, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v33, 0, v33, s[6:7] +; GFX8-NEXT: v_writelane_b32 v34, s30, 0 +; GFX8-NEXT: v_writelane_b32 v34, s31, 1 +; GFX8-NEXT: v_writelane_b32 v34, s34, 2 +; GFX8-NEXT: v_writelane_b32 v34, s35, 3 +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[31:32] +; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:12 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[31:32] +; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:20 +; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[4:5] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[31:32] +; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:28 +; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, 0, s[6:7] ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[6:7], v[37:38] -; GFX8-NEXT: v_cndmask_b32_e32 v4, v33, v4, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v5, v34, v5, vcc -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[37:38] -; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v35, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v36, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v50, v38, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v35, v37, v6, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v36, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v35, v6, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v7, v36, v7, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v37, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v38, s[4:5] -; GFX8-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36 -; GFX8-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:40 -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX8-NEXT: v_cndmask_b32_e32 v0, v48, v0, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v1, v49, v1, vcc -; GFX8-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:48 -; GFX8-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:44 -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[31:32] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v31, v2, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v3, v32, v3, vcc +; GFX8-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[31:32] +; GFX8-NEXT: v_min_f64 v[6:7], v[6:7], v[31:32] +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:36 +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 +; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, 0, s[8:9] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[10:11], v[8:9], v[31:32] +; GFX8-NEXT: v_min_f64 v[8:9], v[8:9], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:44 +; GFX8-NEXT: v_cndmask_b32_e64 v8, v8, 0, s[10:11] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[12:13], v[10:11], v[31:32] +; GFX8-NEXT: v_min_f64 v[10:11], v[10:11], v[31:32] ; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 ; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:52 -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[33:34] -; GFX8-NEXT: v_cndmask_b32_e32 v4, v33, v4, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v5, v34, v5, vcc -; GFX8-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 -; GFX8-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[35:36] -; GFX8-NEXT: v_cndmask_b32_e32 v6, v35, v6, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v7, v36, v7, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[8:9], 32 -; GFX8-NEXT: s_waitcnt vmcnt(6) -; GFX8-NEXT: v_cmp_lt_f64_e64 s[4:5], v[8:9], v[37:38] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[8:9], v[37:38] -; GFX8-NEXT: v_cndmask_b32_e64 v50, v38, v9, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v35, v37, v8, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v36, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v35, v8, vcc -; GFX8-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[35:36] -; GFX8-NEXT: v_cndmask_b32_e32 v9, v36, v9, vcc -; GFX8-NEXT: s_waitcnt vmcnt(4) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[10:11], v[48:49] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[10:11], v[48:49] -; GFX8-NEXT: v_cndmask_b32_e64 v8, v8, v37, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v9, v38, s[4:5] -; GFX8-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:68 -; GFX8-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:72 -; GFX8-NEXT: v_cndmask_b32_e64 v8, v35, v8, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v50, v49, v11, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v35, v48, v10, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 32 -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[48:49], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v51, v39, v50, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v50, 0, v35, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v9, v36, v9, s[6:7] -; GFX8-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:80 -; GFX8-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:76 -; GFX8-NEXT: s_waitcnt vmcnt(6) -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[12:13], v[31:32] -; GFX8-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, v48, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v49, s[4:5] -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[50:51] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[4:5], v[12:13], v[31:32] -; GFX8-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v48, v32, v13, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[12:13], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v49, v39, v48, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v48, v31, v12, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[31:32], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX8-NEXT: s_waitcnt vmcnt(4) -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[14:15], v[33:34] -; GFX8-NEXT: v_cndmask_b32_e32 v12, v48, v12, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v13, v49, v13, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX8-NEXT: v_cndmask_b32_e64 v12, v12, v31, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v13, v13, v32, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, 0, s[12:13] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[14:15], v[12:13], v[31:32] +; GFX8-NEXT: v_min_f64 v[12:13], v[12:13], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:60 +; GFX8-NEXT: v_cndmask_b32_e64 v12, v12, 0, s[14:15] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[16:17], v[14:15], v[31:32] +; GFX8-NEXT: v_min_f64 v[14:15], v[14:15], v[31:32] +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:68 +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 +; GFX8-NEXT: v_cndmask_b32_e64 v14, v14, 0, s[16:17] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[18:19], v[16:17], v[31:32] +; GFX8-NEXT: v_min_f64 v[16:17], v[16:17], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:76 +; GFX8-NEXT: v_cndmask_b32_e64 v16, v16, 0, s[18:19] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[20:21], v[18:19], v[31:32] +; GFX8-NEXT: v_min_f64 v[18:19], v[18:19], v[31:32] ; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:84 -; GFX8-NEXT: v_cmp_lt_f64_e64 s[4:5], v[14:15], v[33:34] -; GFX8-NEXT: v_cndmask_b32_e32 v12, v48, v12, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v13, v49, v13, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v50, v34, v15, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v48, v33, v14, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v49, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v14, v48, v14, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v15, v49, v15, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX8-NEXT: v_cndmask_b32_e64 v14, v14, v33, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v15, v15, v34, s[4:5] -; GFX8-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:96 -; GFX8-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:92 -; GFX8-NEXT: v_cndmask_b32_e32 v14, v48, v14, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v15, v49, v15, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 32 -; GFX8-NEXT: s_waitcnt vmcnt(6) -; GFX8-NEXT: v_cmp_lt_f64_e64 s[4:5], v[16:17], v[37:38] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[16:17], v[37:38] -; GFX8-NEXT: v_cndmask_b32_e64 v50, v38, v17, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v48, v37, v16, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v49, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v16, v48, v16, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v17, v49, v17, vcc -; GFX8-NEXT: s_waitcnt vmcnt(4) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[18:19], v[35:36] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[18:19], v[35:36] -; GFX8-NEXT: v_cndmask_b32_e64 v16, v16, v37, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v17, v17, v38, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v50, v36, v19, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v37, v35, v18, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[18:19], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v51, v39, v50, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v50, 0, v37, s[6:7] -; GFX8-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:100 -; GFX8-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:104 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v48, v16, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v17, v49, v17, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[35:36], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v18, v50, v18, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v19, v51, v19, s[4:5] -; GFX8-NEXT: s_waitcnt vmcnt(4) -; GFX8-NEXT: v_cmp_lt_f64_e64 s[4:5], v[20:21], v[31:32] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[20:21], v[31:32] -; GFX8-NEXT: v_cndmask_b32_e32 v18, v18, v35, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v19, v19, v36, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[50:51] -; GFX8-NEXT: v_cndmask_b32_e64 v48, v32, v21, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v35, v31, v20, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[20:21], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v36, v39, v48, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v18, v50, v18, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v19, v51, v19, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[31:32], 32 -; GFX8-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:112 -; GFX8-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:108 -; GFX8-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:120 -; GFX8-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:116 -; GFX8-NEXT: v_cndmask_b32_e64 v20, v35, v20, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v21, v36, v21, s[4:5] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[35:36] -; GFX8-NEXT: s_waitcnt vmcnt(6) -; GFX8-NEXT: v_cmp_lt_f64_e64 s[6:7], v[22:23], v[33:34] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[22:23], v[33:34] -; GFX8-NEXT: v_cndmask_b32_e32 v20, v20, v31, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v21, v21, v32, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[22:23], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v20, v35, v20, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v21, v36, v21, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v31, v34, v23, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v36, v39, v31, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v31, v33, v22, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v35, 0, v31, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc -; GFX8-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[35:36] -; GFX8-NEXT: v_cndmask_b32_e64 v22, v22, v33, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v18, v18, 0, s[20:21] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[22:23], v[20:21], v[31:32] +; GFX8-NEXT: v_min_f64 v[20:21], v[20:21], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:92 +; GFX8-NEXT: v_cndmask_b32_e64 v20, v20, 0, s[22:23] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[24:25], v[22:23], v[31:32] +; GFX8-NEXT: v_min_f64 v[22:23], v[22:23], v[31:32] +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:100 +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 +; GFX8-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[24:25] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[26:27], v[24:25], v[31:32] +; GFX8-NEXT: v_min_f64 v[24:25], v[24:25], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:108 +; GFX8-NEXT: v_cndmask_b32_e64 v24, v24, 0, s[26:27] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[28:29], v[26:27], v[31:32] +; GFX8-NEXT: v_min_f64 v[26:27], v[26:27], v[31:32] +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:116 +; GFX8-NEXT: v_cndmask_b32_e64 v26, v26, 0, s[28:29] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32] +; GFX8-NEXT: v_min_f64 v[28:29], v[28:29], v[31:32] ; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; GFX8-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:128 ; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 -; GFX8-NEXT: v_cndmask_b32_e64 v23, v23, v34, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[24:25], 32 -; GFX8-NEXT: s_waitcnt vmcnt(7) -; GFX8-NEXT: v_cmp_lt_f64_e64 s[4:5], v[24:25], v[37:38] -; GFX8-NEXT: v_cmp_o_f64_e64 s[6:7], v[24:25], v[37:38] -; GFX8-NEXT: v_cndmask_b32_e64 v34, v38, v25, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v35, v39, v34, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v34, v37, v24, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e32 v24, v34, v24, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v25, v35, v25, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[37:38], 32 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[34:35] -; GFX8-NEXT: v_cndmask_b32_e32 v24, v24, v37, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v25, v25, v38, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[26:27], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v24, v34, v24, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v25, v35, v25, s[4:5] -; GFX8-NEXT: s_waitcnt vmcnt(5) -; GFX8-NEXT: v_cmp_lt_f64_e64 s[6:7], v[26:27], v[48:49] -; GFX8-NEXT: v_cmp_o_f64_e64 s[8:9], v[26:27], v[48:49] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[48:49], 32 -; GFX8-NEXT: s_waitcnt vmcnt(3) -; GFX8-NEXT: v_cmp_o_f64_e64 s[10:11], v[28:29], v[50:51] -; GFX8-NEXT: v_cndmask_b32_e64 v36, v49, v27, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v34, v48, v26, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v35, v39, v36, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[8:9] -; GFX8-NEXT: v_cmp_lt_f64_e64 s[8:9], v[28:29], v[50:51] -; GFX8-NEXT: v_cndmask_b32_e32 v26, v34, v26, vcc -; GFX8-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[34:35] -; GFX8-NEXT: v_cndmask_b32_e32 v27, v35, v27, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v26, v26, v48, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v27, v27, v49, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v36, v51, v29, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v27, v35, v27, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v28, v28, 0, s[30:31] +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33] +; GFX8-NEXT: v_min_f64 v[30:31], v[30:31], v[32:33] +; GFX8-NEXT: v_mov_b32_e32 v32, 0x7ff80000 +; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v32, vcc +; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v32, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v32, s[6:7] +; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v32, s[8:9] +; GFX8-NEXT: v_cndmask_b32_e64 v9, v9, v32, s[10:11] +; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v32, s[12:13] +; GFX8-NEXT: v_cndmask_b32_e64 v13, v13, v32, s[14:15] +; GFX8-NEXT: v_cndmask_b32_e64 v15, v15, v32, s[16:17] +; GFX8-NEXT: v_cndmask_b32_e64 v17, v17, v32, s[18:19] +; GFX8-NEXT: v_cndmask_b32_e64 v19, v19, v32, s[20:21] +; GFX8-NEXT: v_cndmask_b32_e64 v21, v21, v32, s[22:23] +; GFX8-NEXT: v_cndmask_b32_e64 v23, v23, v32, s[24:25] +; GFX8-NEXT: v_cndmask_b32_e64 v25, v25, v32, s[26:27] +; GFX8-NEXT: v_cndmask_b32_e64 v27, v27, v32, s[28:29] +; GFX8-NEXT: v_cndmask_b32_e64 v29, v29, v32, s[30:31] +; GFX8-NEXT: v_cndmask_b32_e64 v31, v31, v32, s[34:35] +; GFX8-NEXT: v_cndmask_b32_e64 v30, v30, 0, s[34:35] +; GFX8-NEXT: v_readlane_b32 s35, v34, 3 +; GFX8-NEXT: v_readlane_b32 s34, v34, 2 +; GFX8-NEXT: v_readlane_b32 s31, v34, 1 +; GFX8-NEXT: v_readlane_b32 s30, v34, 0 +; GFX8-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX8-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload +; GFX8-NEXT: s_mov_b64 exec, s[4:5] ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_cmp_lt_f64_e32 vcc, v[30:31], v[32:33] -; GFX8-NEXT: v_cmp_o_f64_e64 s[4:5], v[30:31], v[32:33] -; GFX8-NEXT: v_cndmask_b32_e64 v35, v39, v36, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v26, v34, v26, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v34, v50, v28, s[8:9] -; GFX8-NEXT: v_cmp_class_f64_e64 s[6:7], v[28:29], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[10:11] -; GFX8-NEXT: v_cmp_class_f64_e64 s[8:9], v[50:51], 32 -; GFX8-NEXT: v_cndmask_b32_e32 v36, v33, v31, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v37, v39, v36, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v36, v32, v30, vcc -; GFX8-NEXT: v_cmp_class_f64_e64 vcc, v[30:31], 32 -; GFX8-NEXT: v_cndmask_b32_e64 v36, 0, v36, s[4:5] -; GFX8-NEXT: v_cmp_class_f64_e64 s[4:5], v[32:33], 32 -; GFX8-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[34:35] -; GFX8-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[36:37] -; GFX8-NEXT: v_cndmask_b32_e64 v28, v34, v28, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v29, v35, v29, s[6:7] -; GFX8-NEXT: v_cndmask_b32_e64 v28, v28, v50, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e32 v30, v36, v30, vcc -; GFX8-NEXT: v_cndmask_b32_e32 v31, v37, v31, vcc -; GFX8-NEXT: v_cndmask_b32_e64 v30, v30, v32, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v29, v29, v51, s[8:9] -; GFX8-NEXT: v_cndmask_b32_e64 v31, v31, v33, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e64 v28, v34, v28, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v29, v35, v29, s[10:11] -; GFX8-NEXT: v_cndmask_b32_e64 v30, v36, v30, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v31, v37, v31, s[12:13] ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minimum_v16f64: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill +; GFX9-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 ; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4 -; GFX9-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:16 -; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12 -; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:24 -; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:20 -; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:32 -; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:28 -; GFX9-NEXT: v_mov_b32_e32 v39, 0x7ff80000 -; GFX9-NEXT: s_waitcnt vmcnt(6) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[31:32] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[31:32] -; GFX9-NEXT: s_waitcnt vmcnt(4) -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[2:3], v[33:34] -; GFX9-NEXT: v_cndmask_b32_e32 v48, v32, v1, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v49, v39, v48, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v48, v31, v0, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[31:32], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v48, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v49, v1, vcc -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[33:34] -; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v31, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v32, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v50, v34, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v31, v33, v2, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v32, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v31, 0, v31, s[6:7] -; GFX9-NEXT: s_waitcnt vmcnt(2) -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[4:5], v[35:36] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v31, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v32, v3, vcc -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[35:36] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v33, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v34, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[35:36], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v50, v36, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v33, v35, v4, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v34, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v33, 0, v33, s[6:7] +; GFX9-NEXT: v_writelane_b32 v34, s30, 0 +; GFX9-NEXT: v_writelane_b32 v34, s31, 1 +; GFX9-NEXT: v_writelane_b32 v34, s34, 2 +; GFX9-NEXT: v_writelane_b32 v34, s35, 3 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[31:32] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:12 +; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[6:7], v[37:38] -; GFX9-NEXT: v_cndmask_b32_e32 v4, v33, v4, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v5, v34, v5, vcc -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[37:38] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v35, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v36, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v50, v38, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v35, v37, v6, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v36, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v35, v6, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v7, v36, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v37, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v38, s[4:5] -; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36 -; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:40 -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX9-NEXT: v_cndmask_b32_e32 v0, v48, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v49, v1, vcc -; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:48 -; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:44 -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[31:32] -; GFX9-NEXT: v_cndmask_b32_e32 v2, v31, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v32, v3, vcc +; GFX9-NEXT: v_cmp_u_f64_e64 s[4:5], v[2:3], v[31:32] +; GFX9-NEXT: v_min_f64 v[2:3], v[2:3], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:20 +; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[4:5] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[6:7], v[4:5], v[31:32] +; GFX9-NEXT: v_min_f64 v[4:5], v[4:5], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:28 +; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, 0, s[6:7] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[8:9], v[6:7], v[31:32] +; GFX9-NEXT: v_min_f64 v[6:7], v[6:7], v[31:32] +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:36 +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 +; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, 0, s[8:9] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[10:11], v[8:9], v[31:32] +; GFX9-NEXT: v_min_f64 v[8:9], v[8:9], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:44 +; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, 0, s[10:11] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[12:13], v[10:11], v[31:32] +; GFX9-NEXT: v_min_f64 v[10:11], v[10:11], v[31:32] ; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 ; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:52 -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[33:34] -; GFX9-NEXT: v_cndmask_b32_e32 v4, v33, v4, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v5, v34, v5, vcc -; GFX9-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 -; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[35:36] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v35, v6, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v7, v36, v7, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[8:9], 32 -; GFX9-NEXT: s_waitcnt vmcnt(6) -; GFX9-NEXT: v_cmp_lt_f64_e64 s[4:5], v[8:9], v[37:38] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[8:9], v[37:38] -; GFX9-NEXT: v_cndmask_b32_e64 v50, v38, v9, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v35, v37, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v36, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v35, v8, vcc -; GFX9-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[35:36] -; GFX9-NEXT: v_cndmask_b32_e32 v9, v36, v9, vcc -; GFX9-NEXT: s_waitcnt vmcnt(4) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[10:11], v[48:49] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[10:11], v[48:49] -; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v37, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v38, s[4:5] -; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:68 -; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:72 -; GFX9-NEXT: v_cndmask_b32_e64 v8, v35, v8, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v50, v49, v11, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v35, v48, v10, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 32 -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[48:49], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v51, v39, v50, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v50, 0, v35, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v36, v9, s[6:7] -; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:80 -; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:76 -; GFX9-NEXT: s_waitcnt vmcnt(6) -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[12:13], v[31:32] -; GFX9-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v48, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v49, s[4:5] -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[50:51] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[4:5], v[12:13], v[31:32] -; GFX9-NEXT: v_cndmask_b32_e32 v10, v50, v10, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v48, v32, v13, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v11, v51, v11, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[12:13], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v49, v39, v48, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v48, v31, v12, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[31:32], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX9-NEXT: s_waitcnt vmcnt(4) -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[14:15], v[33:34] -; GFX9-NEXT: v_cndmask_b32_e32 v12, v48, v12, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v13, v49, v13, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, v31, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v13, v13, v32, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, 0, s[12:13] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[14:15], v[12:13], v[31:32] +; GFX9-NEXT: v_min_f64 v[12:13], v[12:13], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:60 +; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, 0, s[14:15] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[16:17], v[14:15], v[31:32] +; GFX9-NEXT: v_min_f64 v[14:15], v[14:15], v[31:32] +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:68 +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 +; GFX9-NEXT: v_cndmask_b32_e64 v14, v14, 0, s[16:17] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[18:19], v[16:17], v[31:32] +; GFX9-NEXT: v_min_f64 v[16:17], v[16:17], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:76 +; GFX9-NEXT: v_cndmask_b32_e64 v16, v16, 0, s[18:19] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[20:21], v[18:19], v[31:32] +; GFX9-NEXT: v_min_f64 v[18:19], v[18:19], v[31:32] ; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:84 -; GFX9-NEXT: v_cmp_lt_f64_e64 s[4:5], v[14:15], v[33:34] -; GFX9-NEXT: v_cndmask_b32_e32 v12, v48, v12, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v13, v49, v13, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v50, v34, v15, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v48, v33, v14, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v49, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v14, v48, v14, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v15, v49, v15, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX9-NEXT: v_cndmask_b32_e64 v14, v14, v33, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v15, v15, v34, s[4:5] -; GFX9-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:96 -; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:92 -; GFX9-NEXT: v_cndmask_b32_e32 v14, v48, v14, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v15, v49, v15, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 32 -; GFX9-NEXT: s_waitcnt vmcnt(6) -; GFX9-NEXT: v_cmp_lt_f64_e64 s[4:5], v[16:17], v[37:38] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[16:17], v[37:38] -; GFX9-NEXT: v_cndmask_b32_e64 v50, v38, v17, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v48, v37, v16, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v49, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v16, v48, v16, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[37:38], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v17, v49, v17, vcc -; GFX9-NEXT: s_waitcnt vmcnt(4) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[18:19], v[35:36] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[18:19], v[35:36] -; GFX9-NEXT: v_cndmask_b32_e64 v16, v16, v37, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v17, v17, v38, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v50, v36, v19, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v37, v35, v18, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[48:49] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[18:19], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v51, v39, v50, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v50, 0, v37, s[6:7] -; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:100 -; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:104 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v48, v16, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v17, v49, v17, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[35:36], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v18, v50, v18, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v19, v51, v19, s[4:5] -; GFX9-NEXT: s_waitcnt vmcnt(4) -; GFX9-NEXT: v_cmp_lt_f64_e64 s[4:5], v[20:21], v[31:32] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[20:21], v[31:32] -; GFX9-NEXT: v_cndmask_b32_e32 v18, v18, v35, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v19, v19, v36, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[50:51] -; GFX9-NEXT: v_cndmask_b32_e64 v48, v32, v21, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v35, v31, v20, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[20:21], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v36, v39, v48, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v35, 0, v35, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v18, v50, v18, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v19, v51, v19, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[31:32], 32 -; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:112 -; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:108 -; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:120 -; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:116 -; GFX9-NEXT: v_cndmask_b32_e64 v20, v35, v20, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v21, v36, v21, s[4:5] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[35:36] -; GFX9-NEXT: s_waitcnt vmcnt(6) -; GFX9-NEXT: v_cmp_lt_f64_e64 s[6:7], v[22:23], v[33:34] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[22:23], v[33:34] -; GFX9-NEXT: v_cndmask_b32_e32 v20, v20, v31, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v21, v21, v32, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[22:23], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v20, v35, v20, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v21, v36, v21, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[33:34], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v31, v34, v23, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v36, v39, v31, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v31, v33, v22, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v35, 0, v31, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc -; GFX9-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[35:36] -; GFX9-NEXT: v_cndmask_b32_e64 v22, v22, v33, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v18, v18, 0, s[20:21] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[22:23], v[20:21], v[31:32] +; GFX9-NEXT: v_min_f64 v[20:21], v[20:21], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:92 +; GFX9-NEXT: v_cndmask_b32_e64 v20, v20, 0, s[22:23] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[24:25], v[22:23], v[31:32] +; GFX9-NEXT: v_min_f64 v[22:23], v[22:23], v[31:32] +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:100 +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 +; GFX9-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[24:25] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[26:27], v[24:25], v[31:32] +; GFX9-NEXT: v_min_f64 v[24:25], v[24:25], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:108 +; GFX9-NEXT: v_cndmask_b32_e64 v24, v24, 0, s[26:27] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[28:29], v[26:27], v[31:32] +; GFX9-NEXT: v_min_f64 v[26:27], v[26:27], v[31:32] +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 +; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:116 +; GFX9-NEXT: v_cndmask_b32_e64 v26, v26, 0, s[28:29] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32] +; GFX9-NEXT: v_min_f64 v[28:29], v[28:29], v[31:32] ; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:128 ; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 -; GFX9-NEXT: v_cndmask_b32_e64 v23, v23, v34, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[24:25], 32 -; GFX9-NEXT: s_waitcnt vmcnt(7) -; GFX9-NEXT: v_cmp_lt_f64_e64 s[4:5], v[24:25], v[37:38] -; GFX9-NEXT: v_cmp_o_f64_e64 s[6:7], v[24:25], v[37:38] -; GFX9-NEXT: v_cndmask_b32_e64 v34, v38, v25, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v35, v39, v34, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v34, v37, v24, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v24, v34, v24, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v25, v35, v25, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[37:38], 32 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[4:5], 0, v[34:35] -; GFX9-NEXT: v_cndmask_b32_e32 v24, v24, v37, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v25, v25, v38, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[26:27], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v24, v34, v24, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v25, v35, v25, s[4:5] -; GFX9-NEXT: s_waitcnt vmcnt(5) -; GFX9-NEXT: v_cmp_lt_f64_e64 s[6:7], v[26:27], v[48:49] -; GFX9-NEXT: v_cmp_o_f64_e64 s[8:9], v[26:27], v[48:49] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[48:49], 32 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: v_cmp_o_f64_e64 s[10:11], v[28:29], v[50:51] -; GFX9-NEXT: v_cndmask_b32_e64 v36, v49, v27, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v34, v48, v26, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v35, v39, v36, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[8:9] -; GFX9-NEXT: v_cmp_lt_f64_e64 s[8:9], v[28:29], v[50:51] -; GFX9-NEXT: v_cndmask_b32_e32 v26, v34, v26, vcc -; GFX9-NEXT: v_cmp_eq_f64_e64 s[6:7], 0, v[34:35] -; GFX9-NEXT: v_cndmask_b32_e32 v27, v35, v27, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v26, v26, v48, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v27, v27, v49, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v36, v51, v29, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v27, v35, v27, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v28, v28, 0, s[30:31] +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33] +; GFX9-NEXT: v_min_f64 v[30:31], v[30:31], v[32:33] +; GFX9-NEXT: v_mov_b32_e32 v32, 0x7ff80000 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v32, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v32, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v32, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v32, s[8:9] +; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v32, s[10:11] +; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v32, s[12:13] +; GFX9-NEXT: v_cndmask_b32_e64 v13, v13, v32, s[14:15] +; GFX9-NEXT: v_cndmask_b32_e64 v15, v15, v32, s[16:17] +; GFX9-NEXT: v_cndmask_b32_e64 v17, v17, v32, s[18:19] +; GFX9-NEXT: v_cndmask_b32_e64 v19, v19, v32, s[20:21] +; GFX9-NEXT: v_cndmask_b32_e64 v21, v21, v32, s[22:23] +; GFX9-NEXT: v_cndmask_b32_e64 v23, v23, v32, s[24:25] +; GFX9-NEXT: v_cndmask_b32_e64 v25, v25, v32, s[26:27] +; GFX9-NEXT: v_cndmask_b32_e64 v27, v27, v32, s[28:29] +; GFX9-NEXT: v_cndmask_b32_e64 v29, v29, v32, s[30:31] +; GFX9-NEXT: v_cndmask_b32_e64 v31, v31, v32, s[34:35] +; GFX9-NEXT: v_cndmask_b32_e64 v30, v30, 0, s[34:35] +; GFX9-NEXT: v_readlane_b32 s35, v34, 3 +; GFX9-NEXT: v_readlane_b32 s34, v34, 2 +; GFX9-NEXT: v_readlane_b32 s31, v34, 1 +; GFX9-NEXT: v_readlane_b32 s30, v34, 0 +; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX9-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload +; GFX9-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_cmp_lt_f64_e32 vcc, v[30:31], v[32:33] -; GFX9-NEXT: v_cmp_o_f64_e64 s[4:5], v[30:31], v[32:33] -; GFX9-NEXT: v_cndmask_b32_e64 v35, v39, v36, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v26, v34, v26, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v34, v50, v28, s[8:9] -; GFX9-NEXT: v_cmp_class_f64_e64 s[6:7], v[28:29], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[10:11] -; GFX9-NEXT: v_cmp_class_f64_e64 s[8:9], v[50:51], 32 -; GFX9-NEXT: v_cndmask_b32_e32 v36, v33, v31, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v37, v39, v36, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v36, v32, v30, vcc -; GFX9-NEXT: v_cmp_class_f64_e64 vcc, v[30:31], 32 -; GFX9-NEXT: v_cndmask_b32_e64 v36, 0, v36, s[4:5] -; GFX9-NEXT: v_cmp_class_f64_e64 s[4:5], v[32:33], 32 -; GFX9-NEXT: v_cmp_eq_f64_e64 s[10:11], 0, v[34:35] -; GFX9-NEXT: v_cmp_eq_f64_e64 s[12:13], 0, v[36:37] -; GFX9-NEXT: v_cndmask_b32_e64 v28, v34, v28, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v29, v35, v29, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v28, v28, v50, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e32 v30, v36, v30, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v31, v37, v31, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v30, v30, v32, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v29, v29, v51, s[8:9] -; GFX9-NEXT: v_cndmask_b32_e64 v31, v31, v33, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v28, v34, v28, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v29, v35, v29, s[10:11] -; GFX9-NEXT: v_cndmask_b32_e64 v30, v36, v30, s[12:13] -; GFX9-NEXT: v_cndmask_b32_e64 v31, v37, v31, s[12:13] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX940-LABEL: v_minimum_v16f64: ; GFX940: ; %bb.0: ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX940-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a1, v41 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a2, v42 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a3, v43 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a4, v44 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a5, v45 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a6, v46 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a7, v47 ; Reload Reuse -; GFX940-NEXT: scratch_load_dword v41, off, s32 offset:8 -; GFX940-NEXT: scratch_load_dword v40, off, s32 offset:4 -; GFX940-NEXT: scratch_load_dword v51, off, s32 offset:16 -; GFX940-NEXT: scratch_load_dword v50, off, s32 offset:12 -; GFX940-NEXT: scratch_load_dword v45, off, s32 offset:24 -; GFX940-NEXT: scratch_load_dword v44, off, s32 offset:20 -; GFX940-NEXT: scratch_load_dword v47, off, s32 offset:32 -; GFX940-NEXT: scratch_load_dword v46, off, s32 offset:28 +; GFX940-NEXT: v_accvgpr_write_b32 a1, v40 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a2, v41 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a3, v42 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a4, v43 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a5, v44 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a6, v45 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a7, v46 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a8, v47 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a9, v56 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a10, v57 ; Reload Reuse +; GFX940-NEXT: scratch_load_dword v37, off, s32 offset:16 +; GFX940-NEXT: scratch_load_dword v36, off, s32 offset:12 +; GFX940-NEXT: scratch_load_dword v39, off, s32 offset:24 +; GFX940-NEXT: scratch_load_dword v38, off, s32 offset:20 +; GFX940-NEXT: scratch_load_dword v49, off, s32 offset:32 +; GFX940-NEXT: scratch_load_dword v48, off, s32 offset:28 +; GFX940-NEXT: scratch_load_dword v57, off, s32 offset:8 +; GFX940-NEXT: scratch_load_dword v56, off, s32 offset:4 +; GFX940-NEXT: scratch_load_dword v47, off, s32 offset:40 +; GFX940-NEXT: scratch_load_dword v46, off, s32 offset:36 +; GFX940-NEXT: scratch_load_dword v45, off, s32 offset:48 +; GFX940-NEXT: scratch_load_dword v44, off, s32 offset:44 +; GFX940-NEXT: scratch_load_dword v43, off, s32 offset:56 +; GFX940-NEXT: scratch_load_dword v42, off, s32 offset:52 +; GFX940-NEXT: scratch_load_dword v41, off, s32 offset:64 +; GFX940-NEXT: scratch_load_dword v40, off, s32 offset:60 +; GFX940-NEXT: scratch_load_dword v55, off, s32 offset:72 +; GFX940-NEXT: scratch_load_dword v54, off, s32 offset:68 +; GFX940-NEXT: scratch_load_dword v53, off, s32 offset:80 +; GFX940-NEXT: scratch_load_dword v52, off, s32 offset:76 +; GFX940-NEXT: scratch_load_dword v51, off, s32 offset:88 +; GFX940-NEXT: scratch_load_dword v50, off, s32 offset:84 +; GFX940-NEXT: scratch_load_dword v35, off, s32 offset:96 +; GFX940-NEXT: scratch_load_dword v34, off, s32 offset:92 ; GFX940-NEXT: scratch_load_dword v31, off, s32 -; GFX940-NEXT: scratch_load_dword v33, off, s32 offset:128 -; GFX940-NEXT: scratch_load_dword v32, off, s32 offset:124 -; GFX940-NEXT: scratch_load_dword v35, off, s32 offset:120 -; GFX940-NEXT: scratch_load_dword v34, off, s32 offset:116 -; GFX940-NEXT: scratch_load_dword v43, off, s32 offset:40 -; GFX940-NEXT: scratch_load_dword v42, off, s32 offset:36 +; GFX940-NEXT: scratch_load_dword v33, off, s32 offset:104 +; GFX940-NEXT: scratch_load_dword v32, off, s32 offset:100 +; GFX940-NEXT: v_accvgpr_write_b32 a11, v58 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a12, v59 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a13, v60 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a14, v61 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a15, v62 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_write_b32 a16, v63 ; Reload Reuse +; GFX940-NEXT: s_waitcnt vmcnt(25) +; GFX940-NEXT: v_min_f64 v[58:59], v[2:3], v[36:37] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[36:37] ; GFX940-NEXT: scratch_load_dword v37, off, s32 offset:112 ; GFX940-NEXT: scratch_load_dword v36, off, s32 offset:108 -; GFX940-NEXT: scratch_load_dword v39, off, s32 offset:104 -; GFX940-NEXT: scratch_load_dword v38, off, s32 offset:100 -; GFX940-NEXT: scratch_load_dword v49, off, s32 offset:96 -; GFX940-NEXT: scratch_load_dword v48, off, s32 offset:92 -; GFX940-NEXT: scratch_load_dword v53, off, s32 offset:56 -; GFX940-NEXT: scratch_load_dword v52, off, s32 offset:52 -; GFX940-NEXT: scratch_load_dword v55, off, s32 offset:48 -; GFX940-NEXT: scratch_load_dword v54, off, s32 offset:44 -; GFX940-NEXT: v_accvgpr_write_b32 a8, v56 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a9, v57 ; Reload Reuse -; GFX940-NEXT: v_mov_b32_e32 v56, 0x7ff80000 -; GFX940-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a10, v58 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a12, v60 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_write_b32 a13, v61 ; Reload Reuse +; GFX940-NEXT: s_waitcnt vmcnt(25) +; GFX940-NEXT: v_min_f64 v[60:61], v[4:5], v[38:39] +; GFX940-NEXT: v_cmp_u_f64_e64 s[0:1], v[4:5], v[38:39] +; GFX940-NEXT: scratch_load_dword v39, off, s32 offset:120 +; GFX940-NEXT: scratch_load_dword v38, off, s32 offset:116 +; GFX940-NEXT: s_waitcnt vmcnt(25) +; GFX940-NEXT: v_min_f64 v[62:63], v[6:7], v[48:49] +; GFX940-NEXT: v_cmp_u_f64_e64 s[2:3], v[6:7], v[48:49] +; GFX940-NEXT: scratch_load_dword v49, off, s32 offset:128 +; GFX940-NEXT: scratch_load_dword v48, off, s32 offset:124 +; GFX940-NEXT: s_waitcnt vmcnt(25) +; GFX940-NEXT: v_min_f64 v[2:3], v[0:1], v[56:57] +; GFX940-NEXT: v_cmp_u_f64_e64 s[4:5], v[0:1], v[56:57] +; GFX940-NEXT: v_mov_b32_e32 v0, 0x7ff80000 ; GFX940-NEXT: s_waitcnt vmcnt(23) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[0:1], v[40:41] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v57, v41, v1, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[0:1], v[40:41] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e64 v59, v56, v57, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v57, v40, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v58, 0, v57, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[40:41], 32 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v58, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v59, v1, vcc +; GFX940-NEXT: v_min_f64 v[56:57], v[8:9], v[46:47] +; GFX940-NEXT: v_cndmask_b32_e64 v1, v2, 0, s[4:5] +; GFX940-NEXT: v_accvgpr_write_b32 a0, v1 +; GFX940-NEXT: v_cndmask_b32_e64 v1, v3, v0, s[4:5] +; GFX940-NEXT: v_cndmask_b32_e64 v2, v58, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v3, v59, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[8:9], v[46:47] ; GFX940-NEXT: s_waitcnt vmcnt(21) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[2:3], v[50:51] -; GFX940-NEXT: v_cndmask_b32_e64 v0, v0, v40, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v1, v1, v41, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v57, v51, v3, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[2:3], v[50:51] -; GFX940-NEXT: v_cndmask_b32_e32 v40, v50, v2, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[2:3], 32 -; GFX940-NEXT: v_cndmask_b32_e64 v61, v56, v57, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v60, 0, v40, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v2, v60, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v61, v3, vcc -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[50:51], 32 -; GFX940-NEXT: scratch_load_dword v41, off, s32 offset:64 -; GFX940-NEXT: scratch_load_dword v40, off, s32 offset:60 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v50, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v51, vcc -; GFX940-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[58:59] -; GFX940-NEXT: s_waitcnt vmcnt(21) -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[4:5], v[44:45] -; GFX940-NEXT: scratch_load_dword v51, off, s32 offset:88 -; GFX940-NEXT: v_cndmask_b32_e32 v0, v58, v0, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v1, v59, v1, vcc -; GFX940-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[60:61] +; GFX940-NEXT: v_min_f64 v[46:47], v[10:11], v[44:45] +; GFX940-NEXT: v_cndmask_b32_e64 v4, v60, 0, s[0:1] +; GFX940-NEXT: v_cndmask_b32_e64 v8, v56, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v9, v57, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[10:11], v[44:45] +; GFX940-NEXT: s_waitcnt vmcnt(19) +; GFX940-NEXT: v_min_f64 v[44:45], v[12:13], v[42:43] +; GFX940-NEXT: v_cndmask_b32_e64 v5, v61, v0, s[0:1] +; GFX940-NEXT: v_cndmask_b32_e64 v10, v46, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v11, v47, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[12:13], v[42:43] +; GFX940-NEXT: s_waitcnt vmcnt(17) +; GFX940-NEXT: v_min_f64 v[42:43], v[14:15], v[40:41] +; GFX940-NEXT: v_cndmask_b32_e64 v6, v62, 0, s[2:3] +; GFX940-NEXT: v_cndmask_b32_e64 v12, v44, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v13, v45, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[14:15], v[40:41] +; GFX940-NEXT: s_waitcnt vmcnt(15) +; GFX940-NEXT: v_min_f64 v[40:41], v[16:17], v[54:55] +; GFX940-NEXT: v_cndmask_b32_e64 v7, v63, v0, s[2:3] +; GFX940-NEXT: v_cndmask_b32_e64 v14, v42, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v15, v43, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[16:17], v[54:55] +; GFX940-NEXT: s_waitcnt vmcnt(13) +; GFX940-NEXT: v_min_f64 v[54:55], v[18:19], v[52:53] +; GFX940-NEXT: v_accvgpr_read_b32 v63, a16 ; Reload Reuse +; GFX940-NEXT: v_cndmask_b32_e64 v16, v40, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v17, v41, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[18:19], v[52:53] +; GFX940-NEXT: s_waitcnt vmcnt(11) +; GFX940-NEXT: v_min_f64 v[52:53], v[20:21], v[50:51] +; GFX940-NEXT: v_accvgpr_read_b32 v62, a15 ; Reload Reuse +; GFX940-NEXT: v_cndmask_b32_e64 v18, v54, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v19, v55, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[20:21], v[50:51] +; GFX940-NEXT: s_waitcnt vmcnt(9) +; GFX940-NEXT: v_min_f64 v[50:51], v[22:23], v[34:35] +; GFX940-NEXT: v_accvgpr_read_b32 v61, a14 ; Reload Reuse +; GFX940-NEXT: v_cndmask_b32_e64 v20, v52, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v21, v53, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[22:23], v[34:35] +; GFX940-NEXT: s_waitcnt vmcnt(6) +; GFX940-NEXT: v_min_f64 v[34:35], v[24:25], v[32:33] +; GFX940-NEXT: v_accvgpr_read_b32 v60, a13 ; Reload Reuse +; GFX940-NEXT: v_cndmask_b32_e64 v22, v50, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v23, v51, v0, vcc +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[24:25], v[32:33] +; GFX940-NEXT: v_accvgpr_read_b32 v59, a12 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v58, a11 ; Reload Reuse +; GFX940-NEXT: v_cndmask_b32_e64 v24, v34, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v25, v35, v0, vcc +; GFX940-NEXT: v_accvgpr_read_b32 v57, a10 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v56, a9 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v47, a8 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v46, a7 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v45, a6 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v44, a5 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v43, a4 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v42, a3 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v41, a2 ; Reload Reuse +; GFX940-NEXT: v_accvgpr_read_b32 v40, a1 ; Reload Reuse +; GFX940-NEXT: s_waitcnt vmcnt(4) +; GFX940-NEXT: v_min_f64 v[32:33], v[26:27], v[36:37] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[26:27], v[36:37] ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v2, v60, v2, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v3, v61, v3, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[4:5], v[44:45] -; GFX940-NEXT: v_accvgpr_read_b32 v61, a13 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v60, a12 ; Reload Reuse -; GFX940-NEXT: v_cndmask_b32_e32 v50, v45, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v59, v56, v50, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v50, v44, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v58, 0, v50, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[4:5], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[44:45], 32 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v4, v58, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v59, v5, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v4, v4, v44, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v5, v5, v45, s[0:1] -; GFX940-NEXT: scratch_load_dword v45, off, s32 offset:72 -; GFX940-NEXT: scratch_load_dword v44, off, s32 offset:68 -; GFX940-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[58:59] -; GFX940-NEXT: s_waitcnt vmcnt(22) -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[6:7], v[46:47] -; GFX940-NEXT: v_cndmask_b32_e32 v4, v58, v4, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v5, v59, v5, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[6:7], v[46:47] +; GFX940-NEXT: v_cndmask_b32_e64 v26, v32, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v27, v33, v0, vcc +; GFX940-NEXT: s_waitcnt vmcnt(2) +; GFX940-NEXT: v_min_f64 v[32:33], v[28:29], v[38:39] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[28:29], v[38:39] ; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v50, v47, v7, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v59, v56, v50, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v50, v46, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v58, 0, v50, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[6:7], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[46:47], 32 -; GFX940-NEXT: s_nop 0 -; GFX940-NEXT: v_cndmask_b32_e32 v6, v58, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v7, v59, v7, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v6, v6, v46, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v7, v7, v47, s[0:1] -; GFX940-NEXT: scratch_load_dword v47, off, s32 offset:80 -; GFX940-NEXT: scratch_load_dword v46, off, s32 offset:76 -; GFX940-NEXT: scratch_load_dword v50, off, s32 offset:84 -; GFX940-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[58:59] -; GFX940-NEXT: s_waitcnt vmcnt(18) -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[8:9], v[42:43] -; GFX940-NEXT: v_cndmask_b32_e32 v6, v58, v6, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v7, v59, v7, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[8:9], v[42:43] -; GFX940-NEXT: s_nop 1 -; GFX940-NEXT: v_cndmask_b32_e32 v57, v43, v9, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v59, v56, v57, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v57, v42, v8, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v58, 0, v57, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[8:9], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[42:43], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[58:59] -; GFX940-NEXT: v_cndmask_b32_e32 v8, v58, v8, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v9, v59, v9, vcc -; GFX940-NEXT: s_waitcnt vmcnt(8) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[10:11], v[54:55] -; GFX940-NEXT: v_cndmask_b32_e64 v8, v8, v42, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v9, v9, v43, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v42, v55, v11, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[10:11], v[54:55] -; GFX940-NEXT: v_cndmask_b32_e64 v8, v58, v8, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v9, v59, v9, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v43, v56, v42, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v42, v54, v10, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v42, 0, v42, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[10:11], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[54:55], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[42:43] -; GFX940-NEXT: v_cndmask_b32_e32 v10, v42, v10, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v11, v43, v11, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[12:13], v[52:53] -; GFX940-NEXT: v_cndmask_b32_e64 v10, v10, v54, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v11, v11, v55, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v54, v53, v13, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[12:13], v[52:53] -; GFX940-NEXT: v_cndmask_b32_e64 v10, v42, v10, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v11, v43, v11, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v55, v56, v54, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v54, v52, v12, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v54, 0, v54, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[12:13], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[52:53], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[54:55] -; GFX940-NEXT: v_cndmask_b32_e32 v12, v54, v12, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v13, v55, v13, vcc -; GFX940-NEXT: s_waitcnt vmcnt(6) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[14:15], v[40:41] -; GFX940-NEXT: v_cndmask_b32_e64 v12, v12, v52, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v13, v13, v53, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v41, v15, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[14:15], v[40:41] -; GFX940-NEXT: v_cndmask_b32_e64 v12, v54, v12, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v13, v55, v13, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v53, v56, v52, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v40, v14, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v52, 0, v52, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[14:15], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[40:41], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[52:53] -; GFX940-NEXT: v_cndmask_b32_e32 v14, v52, v14, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v14, v14, v40, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v15, v53, v15, vcc -; GFX940-NEXT: s_waitcnt vmcnt(3) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[16:17], v[44:45] -; GFX940-NEXT: v_cndmask_b32_e64 v14, v52, v14, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v15, v15, v41, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v45, v17, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[16:17], v[44:45] -; GFX940-NEXT: v_cndmask_b32_e64 v15, v53, v15, s[2:3] -; GFX940-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse -; GFX940-NEXT: v_cndmask_b32_e64 v53, v56, v52, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v44, v16, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v52, 0, v52, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[16:17], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[44:45], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[52:53] -; GFX940-NEXT: v_cndmask_b32_e32 v16, v52, v16, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v16, v16, v44, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v17, v53, v17, vcc -; GFX940-NEXT: s_waitcnt vmcnt(1) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[18:19], v[46:47] -; GFX940-NEXT: v_cndmask_b32_e64 v16, v52, v16, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v17, v17, v45, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v47, v19, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[18:19], v[46:47] -; GFX940-NEXT: v_cndmask_b32_e64 v17, v53, v17, s[2:3] -; GFX940-NEXT: v_accvgpr_read_b32 v58, a10 ; Reload Reuse -; GFX940-NEXT: v_cndmask_b32_e64 v53, v56, v52, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v46, v18, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v52, 0, v52, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[18:19], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[46:47], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[52:53] -; GFX940-NEXT: v_cndmask_b32_e32 v18, v52, v18, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v18, v18, v46, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v19, v53, v19, vcc +; GFX940-NEXT: v_cndmask_b32_e64 v28, v32, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v29, v33, v0, vcc ; GFX940-NEXT: s_waitcnt vmcnt(0) -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[20:21], v[50:51] -; GFX940-NEXT: v_cndmask_b32_e64 v18, v52, v18, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v19, v19, v47, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v51, v21, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[20:21], v[50:51] -; GFX940-NEXT: v_cndmask_b32_e64 v19, v53, v19, s[2:3] -; GFX940-NEXT: v_accvgpr_read_b32 v57, a9 ; Reload Reuse -; GFX940-NEXT: v_cndmask_b32_e64 v53, v56, v52, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v52, v50, v20, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v52, 0, v52, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[20:21], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[50:51], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[52:53] -; GFX940-NEXT: v_cndmask_b32_e32 v20, v52, v20, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v21, v53, v21, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[22:23], v[48:49] -; GFX940-NEXT: v_cndmask_b32_e64 v20, v20, v50, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v21, v21, v51, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v50, v49, v23, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[22:23], v[48:49] -; GFX940-NEXT: v_cndmask_b32_e64 v20, v52, v20, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v21, v53, v21, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v51, v56, v50, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v50, v48, v22, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v50, 0, v50, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[22:23], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[48:49], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[50:51] -; GFX940-NEXT: v_cndmask_b32_e32 v22, v50, v22, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v23, v51, v23, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[24:25], v[38:39] -; GFX940-NEXT: v_cndmask_b32_e64 v22, v22, v48, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v23, v23, v49, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v48, v39, v25, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[24:25], v[38:39] -; GFX940-NEXT: v_cndmask_b32_e64 v22, v50, v22, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v23, v51, v23, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v49, v56, v48, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v48, v38, v24, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v48, 0, v48, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[24:25], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[38:39], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[48:49] -; GFX940-NEXT: v_cndmask_b32_e32 v24, v48, v24, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v25, v49, v25, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[26:27], v[36:37] -; GFX940-NEXT: v_cndmask_b32_e64 v24, v24, v38, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v25, v25, v39, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v38, v37, v27, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[26:27], v[36:37] -; GFX940-NEXT: v_cndmask_b32_e64 v24, v48, v24, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v25, v49, v25, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v39, v56, v38, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v38, v36, v26, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v38, 0, v38, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[26:27], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[36:37], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[38:39] -; GFX940-NEXT: v_cndmask_b32_e32 v26, v38, v26, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v27, v39, v27, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[28:29], v[34:35] -; GFX940-NEXT: v_cndmask_b32_e64 v26, v26, v36, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v27, v27, v37, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v36, v35, v29, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[28:29], v[34:35] -; GFX940-NEXT: v_cndmask_b32_e64 v26, v38, v26, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v27, v39, v27, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v37, v56, v36, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v36, v34, v28, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v36, 0, v36, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[28:29], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[34:35], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[36:37] -; GFX940-NEXT: v_cndmask_b32_e32 v28, v36, v28, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v29, v37, v29, vcc -; GFX940-NEXT: v_cmp_lt_f64_e32 vcc, v[30:31], v[32:33] -; GFX940-NEXT: v_cndmask_b32_e64 v28, v28, v34, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v29, v29, v35, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v34, v33, v31, vcc -; GFX940-NEXT: v_cmp_o_f64_e64 s[0:1], v[30:31], v[32:33] -; GFX940-NEXT: v_cndmask_b32_e64 v28, v36, v28, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v29, v37, v29, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v35, v56, v34, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e32 v34, v32, v30, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v34, 0, v34, s[0:1] -; GFX940-NEXT: v_cmp_class_f64_e64 vcc, v[30:31], 32 -; GFX940-NEXT: v_cmp_class_f64_e64 s[0:1], v[32:33], 32 -; GFX940-NEXT: v_cmp_eq_f64_e64 s[2:3], 0, v[34:35] -; GFX940-NEXT: v_cndmask_b32_e32 v30, v34, v30, vcc -; GFX940-NEXT: v_cndmask_b32_e32 v31, v35, v31, vcc -; GFX940-NEXT: v_cndmask_b32_e64 v30, v30, v32, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v31, v31, v33, s[0:1] -; GFX940-NEXT: v_cndmask_b32_e64 v30, v34, v30, s[2:3] -; GFX940-NEXT: v_cndmask_b32_e64 v31, v35, v31, s[2:3] -; GFX940-NEXT: v_accvgpr_read_b32 v56, a8 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v47, a7 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v46, a6 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v45, a5 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v44, a4 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v43, a3 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse -; GFX940-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse +; GFX940-NEXT: v_min_f64 v[32:33], v[30:31], v[48:49] +; GFX940-NEXT: v_cmp_u_f64_e32 vcc, v[30:31], v[48:49] +; GFX940-NEXT: s_nop 1 +; GFX940-NEXT: v_cndmask_b32_e64 v30, v32, 0, vcc +; GFX940-NEXT: v_cndmask_b32_e32 v31, v33, v0, vcc +; GFX940-NEXT: v_accvgpr_read_b32 v0, a0 ; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minimum_v16f64: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: s_clause 0x20 +; GFX10-NEXT: s_clause 0x19 +; GFX10-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:12 +; GFX10-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:24 +; GFX10-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:20 +; GFX10-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:32 +; GFX10-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:28 +; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36 +; GFX10-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:68 +; GFX10-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:64 +; GFX10-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:60 +; GFX10-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:56 +; GFX10-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:52 +; GFX10-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:48 +; GFX10-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:44 +; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:40 ; GFX10-NEXT: buffer_load_dword v65, off, s[0:3], s32 offset:8 ; GFX10-NEXT: buffer_load_dword v64, off, s[0:3], s32 offset:4 -; GFX10-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:16 -; GFX10-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:12 -; GFX10-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:24 -; GFX10-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:20 -; GFX10-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:32 -; GFX10-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:28 -; GFX10-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:36 -; GFX10-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:40 -; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:56 -; GFX10-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:52 -; GFX10-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:48 -; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:44 -; GFX10-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:64 -; GFX10-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:60 -; GFX10-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 -; GFX10-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:72 -; GFX10-NEXT: buffer_load_dword v83, off, s[0:3], s32 offset:80 -; GFX10-NEXT: buffer_load_dword v82, off, s[0:3], s32 offset:76 -; GFX10-NEXT: buffer_load_dword v67, off, s[0:3], s32 offset:88 -; GFX10-NEXT: buffer_load_dword v66, off, s[0:3], s32 offset:84 +; GFX10-NEXT: buffer_load_dword v66, off, s[0:3], s32 offset:100 ; GFX10-NEXT: buffer_load_dword v69, off, s[0:3], s32 offset:96 ; GFX10-NEXT: buffer_load_dword v68, off, s[0:3], s32 offset:92 -; GFX10-NEXT: buffer_load_dword v70, off, s[0:3], s32 offset:100 -; GFX10-NEXT: buffer_load_dword v71, off, s[0:3], s32 offset:104 -; GFX10-NEXT: buffer_load_dword v81, off, s[0:3], s32 offset:112 -; GFX10-NEXT: buffer_load_dword v80, off, s[0:3], s32 offset:108 -; GFX10-NEXT: buffer_load_dword v85, off, s[0:3], s32 offset:120 -; GFX10-NEXT: buffer_load_dword v84, off, s[0:3], s32 offset:116 +; GFX10-NEXT: buffer_load_dword v71, off, s[0:3], s32 offset:88 +; GFX10-NEXT: buffer_load_dword v70, off, s[0:3], s32 offset:84 +; GFX10-NEXT: buffer_load_dword v81, off, s[0:3], s32 offset:80 +; GFX10-NEXT: buffer_load_dword v80, off, s[0:3], s32 offset:76 +; GFX10-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:72 +; GFX10-NEXT: buffer_load_dword v67, off, s[0:3], s32 offset:104 +; GFX10-NEXT: s_waitcnt vmcnt(24) +; GFX10-NEXT: v_min_f64 v[82:83], v[2:3], v[31:32] +; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[2:3], v[31:32] +; GFX10-NEXT: s_waitcnt vmcnt(22) +; GFX10-NEXT: v_min_f64 v[84:85], v[4:5], v[33:34] +; GFX10-NEXT: v_cmp_u_f64_e64 s4, v[4:5], v[33:34] +; GFX10-NEXT: s_clause 0x3 +; GFX10-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 +; GFX10-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 +; GFX10-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:112 +; GFX10-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:108 +; GFX10-NEXT: s_waitcnt vmcnt(24) +; GFX10-NEXT: v_min_f64 v[32:33], v[6:7], v[35:36] +; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[6:7], v[35:36] +; GFX10-NEXT: s_clause 0x2 ; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX10-NEXT: buffer_load_dword v87, off, s[0:3], s32 offset:128 -; GFX10-NEXT: buffer_load_dword v86, off, s[0:3], s32 offset:124 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[0:1], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s12, v[2:3], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s17, v[10:11], 32 -; GFX10-NEXT: s_waitcnt vmcnt(31) -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[0:1], v[64:65] -; GFX10-NEXT: v_cmp_o_f64_e64 s4, v[0:1], v[64:65] -; GFX10-NEXT: s_waitcnt vmcnt(29) -; GFX10-NEXT: v_cmp_lt_f64_e64 s5, v[2:3], v[54:55] -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[2:3], v[54:55] -; GFX10-NEXT: s_waitcnt vmcnt(27) -; GFX10-NEXT: v_cmp_lt_f64_e64 s7, v[4:5], v[52:53] -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[4:5], v[52:53] -; GFX10-NEXT: s_waitcnt vmcnt(25) -; GFX10-NEXT: v_cmp_lt_f64_e64 s9, v[6:7], v[50:51] -; GFX10-NEXT: v_cmp_o_f64_e64 s11, v[6:7], v[50:51] +; GFX10-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:128 +; GFX10-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:124 ; GFX10-NEXT: s_waitcnt vmcnt(23) -; GFX10-NEXT: v_cmp_lt_f64_e64 s13, v[8:9], v[48:49] -; GFX10-NEXT: v_cmp_class_f64_e64 s14, v[64:65], 32 +; GFX10-NEXT: v_cmp_u_f64_e64 s10, v[14:15], v[50:51] ; GFX10-NEXT: s_waitcnt vmcnt(21) -; GFX10-NEXT: v_cmp_lt_f64_e64 s15, v[12:13], v[36:37] -; GFX10-NEXT: s_waitcnt vmcnt(17) -; GFX10-NEXT: v_cmp_o_f64_e64 s16, v[14:15], v[34:35] -; GFX10-NEXT: v_cndmask_b32_e32 v96, v64, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v97, v54, v2, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v99, v55, v3, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v100, v52, v4, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v96, 0, v96, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v101, v50, v6, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v98, 0, v97, s6 -; GFX10-NEXT: v_cndmask_b32_e32 v97, v65, v1, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[54:55], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v96, v0, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v99, 0x7ff80000, v99, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v98, v2, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v97, 0x7ff80000, v97, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v100, 0, v100, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v102, 0, v101, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v99, v3, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 s12, v[6:7], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v97, v1, s10 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[4:5], 32 -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[10:11], v[38:39] -; GFX10-NEXT: v_cndmask_b32_e64 v112, v48, v8, s13 -; GFX10-NEXT: v_cmp_o_f64_e64 s5, v[12:13], v[36:37] -; GFX10-NEXT: v_cmp_lt_f64_e64 s6, v[14:15], v[34:35] -; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v64, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v65, s14 -; GFX10-NEXT: v_cmp_class_f64_e64 s14, v[52:53], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v113, v36, v12, s15 +; GFX10-NEXT: v_cmp_u_f64_e64 s9, v[12:13], v[52:53] +; GFX10-NEXT: s_waitcnt vmcnt(19) +; GFX10-NEXT: v_cmp_u_f64_e64 s7, v[10:11], v[54:55] +; GFX10-NEXT: s_waitcnt vmcnt(18) +; GFX10-NEXT: v_min_f64 v[34:35], v[8:9], v[37:38] +; GFX10-NEXT: v_cmp_u_f64_e64 s6, v[8:9], v[37:38] +; GFX10-NEXT: s_waitcnt vmcnt(16) +; GFX10-NEXT: v_min_f64 v[8:9], v[0:1], v[64:65] +; GFX10-NEXT: v_min_f64 v[36:37], v[10:11], v[54:55] +; GFX10-NEXT: v_cmp_u_f64_e64 s8, v[0:1], v[64:65] +; GFX10-NEXT: v_min_f64 v[38:39], v[12:13], v[52:53] +; GFX10-NEXT: v_min_f64 v[52:53], v[14:15], v[50:51] +; GFX10-NEXT: s_waitcnt vmcnt(11) +; GFX10-NEXT: v_min_f64 v[54:55], v[20:21], v[70:71] +; GFX10-NEXT: v_cmp_u_f64_e64 s13, v[20:21], v[70:71] +; GFX10-NEXT: s_waitcnt vmcnt(9) +; GFX10-NEXT: v_cmp_u_f64_e64 s12, v[18:19], v[80:81] +; GFX10-NEXT: s_waitcnt vmcnt(8) +; GFX10-NEXT: v_min_f64 v[50:51], v[16:17], v[48:49] +; GFX10-NEXT: v_cmp_u_f64_e64 s11, v[16:17], v[48:49] +; GFX10-NEXT: v_min_f64 v[48:49], v[18:19], v[80:81] +; GFX10-NEXT: v_min_f64 v[64:65], v[22:23], v[68:69] +; GFX10-NEXT: v_cmp_u_f64_e64 s14, v[22:23], v[68:69] +; GFX10-NEXT: s_waitcnt vmcnt(7) +; GFX10-NEXT: v_min_f64 v[68:69], v[24:25], v[66:67] +; GFX10-NEXT: v_cmp_u_f64_e64 s15, v[24:25], v[66:67] +; GFX10-NEXT: v_cndmask_b32_e64 v10, v36, 0, s7 +; GFX10-NEXT: v_cndmask_b32_e64 v0, v8, 0, s8 +; GFX10-NEXT: v_cndmask_b32_e64 v1, v9, 0x7ff80000, s8 +; GFX10-NEXT: v_cndmask_b32_e64 v8, v34, 0, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v9, v35, 0x7ff80000, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v11, v37, 0x7ff80000, s7 +; GFX10-NEXT: v_cndmask_b32_e64 v12, v38, 0, s9 +; GFX10-NEXT: v_cndmask_b32_e64 v13, v39, 0x7ff80000, s9 +; GFX10-NEXT: v_cndmask_b32_e64 v14, v52, 0, s10 +; GFX10-NEXT: v_cndmask_b32_e64 v15, v53, 0x7ff80000, s10 +; GFX10-NEXT: v_cndmask_b32_e64 v16, v50, 0, s11 +; GFX10-NEXT: v_cndmask_b32_e64 v17, v51, 0x7ff80000, s11 +; GFX10-NEXT: v_cndmask_b32_e64 v18, v48, 0, s12 +; GFX10-NEXT: v_cndmask_b32_e64 v19, v49, 0x7ff80000, s12 +; GFX10-NEXT: v_cndmask_b32_e64 v20, v54, 0, s13 +; GFX10-NEXT: v_cndmask_b32_e64 v21, v55, 0x7ff80000, s13 +; GFX10-NEXT: v_cndmask_b32_e64 v22, v64, 0, s14 +; GFX10-NEXT: v_cndmask_b32_e64 v23, v65, 0x7ff80000, s14 +; GFX10-NEXT: v_cndmask_b32_e64 v24, v68, 0, s15 +; GFX10-NEXT: v_cndmask_b32_e64 v25, v69, 0x7ff80000, s15 +; GFX10-NEXT: s_waitcnt vmcnt(5) +; GFX10-NEXT: v_min_f64 v[70:71], v[28:29], v[2:3] +; GFX10-NEXT: v_cmp_u_f64_e64 s17, v[28:29], v[2:3] +; GFX10-NEXT: s_waitcnt vmcnt(3) +; GFX10-NEXT: v_min_f64 v[66:67], v[26:27], v[4:5] +; GFX10-NEXT: v_cmp_u_f64_e64 s16, v[26:27], v[4:5] +; GFX10-NEXT: v_cndmask_b32_e64 v2, v82, 0, vcc_lo ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_cmp_o_f64_e64 s18, v[30:31], v[86:87] -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v54, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v54, v53, v5, s7 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v55, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[50:51], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v55, v51, v7, s9 -; GFX10-NEXT: v_cmp_o_f64_e64 s9, v[8:9], v[48:49] -; GFX10-NEXT: v_cndmask_b32_e64 v101, 0x7ff80000, v54, s8 -; GFX10-NEXT: v_cmp_lt_f64_e64 s7, v[16:17], v[32:33] -; GFX10-NEXT: v_cndmask_b32_e64 v6, v102, v6, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v103, 0x7ff80000, v55, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v100, v4, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v101, v5, s10 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[8:9], 32 -; GFX10-NEXT: v_cmp_o_f64_e64 s11, v[10:11], v[38:39] -; GFX10-NEXT: v_cndmask_b32_e64 v7, v103, v7, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 s12, v[48:49], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v114, v38, v10, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v115, v34, v14, s6 -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[16:17], v[32:33] -; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v52, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v53, s14 -; GFX10-NEXT: v_cmp_lt_f64_e64 s14, v[18:19], v[82:83] -; GFX10-NEXT: v_cndmask_b32_e64 v52, 0, v115, s16 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v50, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v50, v49, v9, s13 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v51, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[38:39], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v54, 0, v112, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v51, v39, v11, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v55, 0x7ff80000, v50, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v50, 0, v113, s5 -; GFX10-NEXT: v_cmp_o_f64_e64 s4, v[18:19], v[82:83] -; GFX10-NEXT: v_cndmask_b32_e64 v8, v54, v8, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v64, 0, v114, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v9, v55, v9, s10 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[12:13], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v65, 0x7ff80000, v51, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, v48, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v10, v64, v10, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v49, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 s12, v[14:15], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v65, v11, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v48, v37, v13, s15 -; GFX10-NEXT: v_cmp_class_f64_e64 s17, v[34:35], 32 -; GFX10-NEXT: v_cmp_lt_f64_e64 s9, v[20:21], v[66:67] -; GFX10-NEXT: v_cmp_o_f64_e64 s11, v[20:21], v[66:67] -; GFX10-NEXT: v_cndmask_b32_e64 v116, v32, v16, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v51, 0x7ff80000, v48, s5 -; GFX10-NEXT: v_cndmask_b32_e32 v10, v10, v38, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v11, v11, v39, vcc_lo -; GFX10-NEXT: v_cmp_lt_f64_e32 vcc_lo, v[22:23], v[68:69] -; GFX10-NEXT: v_cndmask_b32_e64 v38, v35, v15, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v49, v82, v18, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v48, 0, v116, s8 -; GFX10-NEXT: v_cmp_class_f64_e64 s13, v[36:37], 32 -; GFX10-NEXT: v_cmp_o_f64_e64 s5, v[22:23], v[68:69] -; GFX10-NEXT: v_cndmask_b32_e64 v53, 0x7ff80000, v38, s16 -; GFX10-NEXT: v_cndmask_b32_e64 v12, v50, v12, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v51, v13, s10 -; GFX10-NEXT: v_cmp_class_f64_e64 s10, v[16:17], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v38, 0, v49, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v112, v83, v19, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v14, v52, v14, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v15, v53, v15, s12 -; GFX10-NEXT: v_cmp_class_f64_e64 s12, v[32:33], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s14, v[18:19], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v114, v67, v21, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v14, v14, v34, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v15, v15, v35, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v34, v33, v17, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v39, 0x7ff80000, v112, s4 -; GFX10-NEXT: v_cmp_lt_f64_e64 s4, v[24:25], v[70:71] -; GFX10-NEXT: v_cndmask_b32_e32 v113, v69, v23, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v35, v68, v22, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[20:21], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v49, 0x7ff80000, v34, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v34, 0x7ff80000, v114, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v12, v12, v36, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v13, v37, s13 -; GFX10-NEXT: v_cmp_class_f64_e64 s13, v[82:83], 32 -; GFX10-NEXT: v_cmp_o_f64_e64 s6, v[24:25], v[70:71] -; GFX10-NEXT: v_cndmask_b32_e64 v16, v48, v16, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v17, v49, v17, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v36, 0x7ff80000, v113, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v35, 0, v35, s5 -; GFX10-NEXT: v_cmp_lt_f64_e64 s7, v[26:27], v[80:81] -; GFX10-NEXT: v_cndmask_b32_e64 v16, v16, v32, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v32, v66, v20, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v17, v17, v33, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v18, v38, v18, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v19, v39, v19, s14 -; GFX10-NEXT: v_cmp_o_f64_e64 s15, v[26:27], v[80:81] -; GFX10-NEXT: v_cndmask_b32_e64 v33, 0, v32, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v112, v71, v25, s4 -; GFX10-NEXT: v_cmp_lt_f64_e64 s16, v[28:29], v[84:85] -; GFX10-NEXT: v_cmp_o_f64_e64 s8, v[28:29], v[84:85] -; GFX10-NEXT: v_cndmask_b32_e32 v21, v34, v21, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v20, v33, v20, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[22:23], 32 -; GFX10-NEXT: v_cmp_lt_f64_e64 s17, v[30:31], v[86:87] -; GFX10-NEXT: v_cmp_class_f64_e64 s5, v[70:71], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v18, v18, v82, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v82, v70, v24, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v19, v19, v83, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v83, 0x7ff80000, v112, s6 -; GFX10-NEXT: v_cmp_class_f64_e64 s4, v[68:69], 32 -; GFX10-NEXT: v_cmp_eq_f64_e64 s9, 0, v[96:97] -; GFX10-NEXT: v_cndmask_b32_e64 v82, 0, v82, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v37, v81, v27, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v32, v80, v26, s7 -; GFX10-NEXT: v_cmp_class_f64_e64 s6, v[80:81], 32 -; GFX10-NEXT: v_cmp_class_f64_e64 s7, v[84:85], 32 -; GFX10-NEXT: v_cmp_eq_f64_e64 s10, 0, v[98:99] -; GFX10-NEXT: v_cndmask_b32_e64 v113, 0x7ff80000, v37, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v112, 0, v32, s15 -; GFX10-NEXT: v_cmp_eq_f64_e64 s11, 0, v[100:101] -; GFX10-NEXT: v_cndmask_b32_e64 v115, v85, v29, s16 -; GFX10-NEXT: v_cndmask_b32_e64 v114, v84, v28, s16 -; GFX10-NEXT: v_cmp_eq_f64_e64 s12, 0, v[102:103] -; GFX10-NEXT: v_cmp_eq_f64_e64 s13, 0, v[54:55] -; GFX10-NEXT: v_cndmask_b32_e32 v22, v35, v22, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v23, v36, v23, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[24:25], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v115, 0x7ff80000, v115, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v114, 0, v114, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v116, v87, v31, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v32, v86, v30, s17 -; GFX10-NEXT: v_cmp_class_f64_e64 s8, v[86:87], 32 -; GFX10-NEXT: v_cmp_eq_f64_e64 s14, 0, v[64:65] -; GFX10-NEXT: v_cmp_eq_f64_e64 s15, 0, v[50:51] -; GFX10-NEXT: v_cndmask_b32_e64 v117, 0x7ff80000, v116, s18 -; GFX10-NEXT: v_cndmask_b32_e64 v116, 0, v32, s18 -; GFX10-NEXT: v_cmp_eq_f64_e64 s16, 0, v[52:53] -; GFX10-NEXT: v_cmp_eq_f64_e64 s17, 0, v[48:49] -; GFX10-NEXT: v_cmp_eq_f64_e64 s18, 0, v[38:39] -; GFX10-NEXT: v_cmp_eq_f64_e64 s19, 0, v[33:34] -; GFX10-NEXT: v_cmp_eq_f64_e64 s20, 0, v[35:36] -; GFX10-NEXT: v_cmp_eq_f64_e64 s21, 0, v[82:83] -; GFX10-NEXT: v_cmp_eq_f64_e64 s22, 0, v[112:113] -; GFX10-NEXT: v_cmp_eq_f64_e64 s23, 0, v[114:115] -; GFX10-NEXT: v_cmp_eq_f64_e64 s24, 0, v[116:117] -; GFX10-NEXT: v_cndmask_b32_e64 v22, v22, v68, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v23, v23, v69, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v96, v0, s9 -; GFX10-NEXT: v_cndmask_b32_e32 v24, v82, v24, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v25, v83, v25, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[26:27], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v98, v2, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v100, v4, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v24, v24, v70, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v25, v25, v71, s5 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v102, v6, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v8, v54, v8, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v10, v64, v10, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v12, v50, v12, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v14, v52, v14, s16 -; GFX10-NEXT: v_cndmask_b32_e64 v16, v48, v16, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v18, v38, v18, s18 -; GFX10-NEXT: v_cndmask_b32_e64 v22, v35, v22, s20 -; GFX10-NEXT: v_cndmask_b32_e64 v24, v82, v24, s21 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v97, v1, s9 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v99, v3, s10 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v101, v5, s11 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v103, v7, s12 -; GFX10-NEXT: v_cndmask_b32_e64 v9, v55, v9, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v65, v11, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v51, v13, s15 -; GFX10-NEXT: v_cndmask_b32_e64 v15, v53, v15, s16 -; GFX10-NEXT: v_cndmask_b32_e32 v26, v112, v26, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v27, v113, v27, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[28:29], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v17, v49, v17, s17 -; GFX10-NEXT: v_cndmask_b32_e64 v19, v39, v19, s18 -; GFX10-NEXT: v_cndmask_b32_e64 v26, v26, v80, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v27, v27, v81, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v23, v36, v23, s20 -; GFX10-NEXT: v_cndmask_b32_e64 v25, v83, v25, s21 -; GFX10-NEXT: v_cndmask_b32_e64 v26, v112, v26, s22 -; GFX10-NEXT: v_cndmask_b32_e64 v27, v113, v27, s22 -; GFX10-NEXT: v_cndmask_b32_e32 v28, v114, v28, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v29, v115, v29, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[30:31], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v28, v28, v84, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v29, v29, v85, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v28, v114, v28, s23 -; GFX10-NEXT: v_cndmask_b32_e64 v29, v115, v29, s23 -; GFX10-NEXT: v_cndmask_b32_e32 v30, v116, v30, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v31, v117, v31, vcc_lo -; GFX10-NEXT: v_cmp_class_f64_e64 vcc_lo, v[66:67], 32 -; GFX10-NEXT: v_cndmask_b32_e64 v30, v30, v86, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v31, v31, v87, s8 -; GFX10-NEXT: v_cndmask_b32_e64 v30, v116, v30, s24 -; GFX10-NEXT: v_cndmask_b32_e64 v31, v117, v31, s24 -; GFX10-NEXT: v_cndmask_b32_e32 v20, v20, v66, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v21, v21, v67, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v20, v33, v20, s19 -; GFX10-NEXT: v_cndmask_b32_e64 v21, v34, v21, s19 +; GFX10-NEXT: v_min_f64 v[80:81], v[30:31], v[6:7] +; GFX10-NEXT: v_cmp_u_f64_e64 s18, v[30:31], v[6:7] +; GFX10-NEXT: v_cndmask_b32_e64 v3, v83, 0x7ff80000, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v4, v84, 0, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v85, 0x7ff80000, s4 +; GFX10-NEXT: v_cndmask_b32_e64 v6, v32, 0, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v7, v33, 0x7ff80000, s5 +; GFX10-NEXT: v_cndmask_b32_e64 v28, v70, 0, s17 +; GFX10-NEXT: v_cndmask_b32_e64 v29, v71, 0x7ff80000, s17 +; GFX10-NEXT: v_cndmask_b32_e64 v26, v66, 0, s16 +; GFX10-NEXT: v_cndmask_b32_e64 v27, v67, 0x7ff80000, s16 +; GFX10-NEXT: v_cndmask_b32_e64 v30, v80, 0, s18 +; GFX10-NEXT: v_cndmask_b32_e64 v31, v81, 0x7ff80000, s18 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minimum_v16f64: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_clause 0x1f -; GFX11-NEXT: scratch_load_b32 v87, off, s32 offset:8 -; GFX11-NEXT: scratch_load_b32 v86, off, s32 offset:4 -; GFX11-NEXT: scratch_load_b32 v85, off, s32 offset:16 -; GFX11-NEXT: scratch_load_b32 v84, off, s32 offset:12 -; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:24 -; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:20 -; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:32 -; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:28 -; GFX11-NEXT: scratch_load_b32 v37, off, s32 offset:40 -; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:36 -; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:48 -; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:44 -; GFX11-NEXT: scratch_load_b32 v49, off, s32 offset:56 -; GFX11-NEXT: scratch_load_b32 v48, off, s32 offset:52 -; GFX11-NEXT: scratch_load_b32 v51, off, s32 offset:64 -; GFX11-NEXT: scratch_load_b32 v50, off, s32 offset:60 -; GFX11-NEXT: scratch_load_b32 v53, off, s32 offset:72 -; GFX11-NEXT: scratch_load_b32 v52, off, s32 offset:68 -; GFX11-NEXT: scratch_load_b32 v55, off, s32 offset:80 -; GFX11-NEXT: scratch_load_b32 v54, off, s32 offset:76 -; GFX11-NEXT: scratch_load_b32 v65, off, s32 offset:88 -; GFX11-NEXT: scratch_load_b32 v64, off, s32 offset:84 -; GFX11-NEXT: scratch_load_b32 v67, off, s32 offset:96 -; GFX11-NEXT: scratch_load_b32 v66, off, s32 offset:92 -; GFX11-NEXT: scratch_load_b32 v69, off, s32 offset:104 -; GFX11-NEXT: scratch_load_b32 v68, off, s32 offset:100 -; GFX11-NEXT: scratch_load_b32 v71, off, s32 offset:112 -; GFX11-NEXT: scratch_load_b32 v70, off, s32 offset:108 -; GFX11-NEXT: scratch_load_b32 v81, off, s32 offset:120 -; GFX11-NEXT: scratch_load_b32 v80, off, s32 offset:116 ; GFX11-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-NEXT: scratch_load_b32 v83, off, s32 offset:128 -; GFX11-NEXT: scratch_load_b32 v82, off, s32 offset:124 -; GFX11-NEXT: s_waitcnt vmcnt(31) -; GFX11-NEXT: v_cmp_lt_f64_e64 s9, v[0:1], v[86:87] -; GFX11-NEXT: v_cmp_o_f64_e64 s11, v[0:1], v[86:87] -; GFX11-NEXT: s_waitcnt vmcnt(29) -; GFX11-NEXT: v_cmp_lt_f64_e64 s10, v[2:3], v[84:85] -; GFX11-NEXT: v_cmp_class_f64_e64 s14, v[86:87], 32 -; GFX11-NEXT: s_waitcnt vmcnt(27) -; GFX11-NEXT: v_cmp_lt_f64_e64 s0, v[4:5], v[32:33] -; GFX11-NEXT: v_cmp_o_f64_e32 vcc_lo, v[4:5], v[32:33] -; GFX11-NEXT: s_waitcnt vmcnt(25) -; GFX11-NEXT: v_cmp_lt_f64_e64 s2, v[6:7], v[34:35] -; GFX11-NEXT: v_cmp_o_f64_e64 s12, v[2:3], v[84:85] -; GFX11-NEXT: v_cmp_o_f64_e64 s1, v[6:7], v[34:35] -; GFX11-NEXT: s_waitcnt vmcnt(23) -; GFX11-NEXT: v_cmp_lt_f64_e64 s4, v[8:9], v[36:37] -; GFX11-NEXT: v_cmp_o_f64_e64 s3, v[8:9], v[36:37] -; GFX11-NEXT: v_cmp_class_f64_e64 s16, v[84:85], 32 -; GFX11-NEXT: s_waitcnt vmcnt(21) -; GFX11-NEXT: v_cmp_lt_f64_e64 s6, v[10:11], v[38:39] -; GFX11-NEXT: v_cmp_o_f64_e64 s5, v[10:11], v[38:39] -; GFX11-NEXT: s_waitcnt vmcnt(19) -; GFX11-NEXT: v_cmp_lt_f64_e64 s8, v[12:13], v[48:49] -; GFX11-NEXT: v_cmp_o_f64_e64 s7, v[12:13], v[48:49] -; GFX11-NEXT: s_waitcnt vmcnt(17) -; GFX11-NEXT: v_cmp_lt_f64_e64 s13, v[14:15], v[50:51] -; GFX11-NEXT: s_waitcnt vmcnt(15) -; GFX11-NEXT: v_cmp_o_f64_e64 s15, v[16:17], v[52:53] -; GFX11-NEXT: s_waitcnt vmcnt(13) -; GFX11-NEXT: v_cmp_lt_f64_e64 s17, v[18:19], v[54:55] -; GFX11-NEXT: v_cmp_o_f64_e64 s18, v[18:19], v[54:55] -; GFX11-NEXT: s_waitcnt vmcnt(11) -; GFX11-NEXT: v_cmp_lt_f64_e64 s19, v[20:21], v[64:65] -; GFX11-NEXT: v_cmp_o_f64_e64 s20, v[20:21], v[64:65] -; GFX11-NEXT: s_waitcnt vmcnt(9) -; GFX11-NEXT: v_cmp_lt_f64_e64 s21, v[22:23], v[66:67] -; GFX11-NEXT: v_cmp_o_f64_e64 s22, v[22:23], v[66:67] -; GFX11-NEXT: s_waitcnt vmcnt(7) -; GFX11-NEXT: v_cmp_lt_f64_e64 s23, v[24:25], v[68:69] -; GFX11-NEXT: v_cmp_o_f64_e64 s24, v[24:25], v[68:69] -; GFX11-NEXT: s_waitcnt vmcnt(5) -; GFX11-NEXT: v_cmp_lt_f64_e64 s25, v[26:27], v[70:71] -; GFX11-NEXT: v_cmp_o_f64_e64 s26, v[26:27], v[70:71] -; GFX11-NEXT: s_waitcnt vmcnt(3) -; GFX11-NEXT: v_cmp_lt_f64_e64 s27, v[28:29], v[80:81] -; GFX11-NEXT: v_cmp_o_f64_e64 s28, v[28:29], v[80:81] +; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:8 +; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:4 +; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:16 +; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:12 +; GFX11-NEXT: scratch_load_b32 v37, off, s32 offset:24 +; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:20 +; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:32 +; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:28 +; GFX11-NEXT: scratch_load_b32 v49, off, s32 offset:40 +; GFX11-NEXT: scratch_load_b32 v48, off, s32 offset:36 +; GFX11-NEXT: scratch_load_b32 v51, off, s32 offset:48 +; GFX11-NEXT: scratch_load_b32 v50, off, s32 offset:44 +; GFX11-NEXT: scratch_load_b32 v53, off, s32 offset:56 +; GFX11-NEXT: scratch_load_b32 v52, off, s32 offset:52 +; GFX11-NEXT: scratch_load_b32 v55, off, s32 offset:64 +; GFX11-NEXT: scratch_load_b32 v54, off, s32 offset:60 +; GFX11-NEXT: scratch_load_b32 v65, off, s32 offset:72 +; GFX11-NEXT: scratch_load_b32 v64, off, s32 offset:68 +; GFX11-NEXT: scratch_load_b32 v67, off, s32 offset:80 +; GFX11-NEXT: scratch_load_b32 v66, off, s32 offset:76 +; GFX11-NEXT: scratch_load_b32 v69, off, s32 offset:88 +; GFX11-NEXT: scratch_load_b32 v68, off, s32 offset:84 +; GFX11-NEXT: scratch_load_b32 v71, off, s32 offset:96 +; GFX11-NEXT: scratch_load_b32 v70, off, s32 offset:92 +; GFX11-NEXT: scratch_load_b32 v81, off, s32 offset:104 +; GFX11-NEXT: scratch_load_b32 v80, off, s32 offset:100 +; GFX11-NEXT: scratch_load_b32 v83, off, s32 offset:112 +; GFX11-NEXT: scratch_load_b32 v82, off, s32 offset:108 +; GFX11-NEXT: scratch_load_b32 v85, off, s32 offset:120 +; GFX11-NEXT: scratch_load_b32 v84, off, s32 offset:116 +; GFX11-NEXT: scratch_load_b32 v87, off, s32 offset:128 +; GFX11-NEXT: scratch_load_b32 v86, off, s32 offset:124 +; GFX11-NEXT: s_waitcnt vmcnt(30) +; GFX11-NEXT: v_min_f64 v[96:97], v[0:1], v[32:33] +; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[32:33] +; GFX11-NEXT: s_waitcnt vmcnt(28) +; GFX11-NEXT: v_min_f64 v[32:33], v[2:3], v[34:35] +; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[34:35] +; GFX11-NEXT: s_waitcnt vmcnt(26) +; GFX11-NEXT: v_min_f64 v[34:35], v[4:5], v[36:37] +; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[36:37] +; GFX11-NEXT: s_waitcnt vmcnt(24) +; GFX11-NEXT: v_min_f64 v[36:37], v[6:7], v[38:39] +; GFX11-NEXT: v_cmp_u_f64_e64 s2, v[6:7], v[38:39] +; GFX11-NEXT: s_waitcnt vmcnt(22) +; GFX11-NEXT: v_min_f64 v[38:39], v[8:9], v[48:49] +; GFX11-NEXT: v_cmp_u_f64_e64 s3, v[8:9], v[48:49] +; GFX11-NEXT: s_waitcnt vmcnt(20) +; GFX11-NEXT: v_min_f64 v[48:49], v[10:11], v[50:51] +; GFX11-NEXT: v_cmp_u_f64_e64 s4, v[10:11], v[50:51] +; GFX11-NEXT: s_waitcnt vmcnt(18) +; GFX11-NEXT: v_min_f64 v[50:51], v[12:13], v[52:53] +; GFX11-NEXT: v_cmp_u_f64_e64 s5, v[12:13], v[52:53] +; GFX11-NEXT: s_waitcnt vmcnt(16) +; GFX11-NEXT: v_min_f64 v[52:53], v[14:15], v[54:55] +; GFX11-NEXT: v_cmp_u_f64_e64 s6, v[14:15], v[54:55] +; GFX11-NEXT: s_waitcnt vmcnt(14) +; GFX11-NEXT: v_min_f64 v[54:55], v[16:17], v[64:65] +; GFX11-NEXT: v_cmp_u_f64_e64 s7, v[16:17], v[64:65] +; GFX11-NEXT: s_waitcnt vmcnt(12) +; GFX11-NEXT: v_min_f64 v[64:65], v[18:19], v[66:67] +; GFX11-NEXT: v_cmp_u_f64_e64 s8, v[18:19], v[66:67] +; GFX11-NEXT: s_waitcnt vmcnt(10) +; GFX11-NEXT: v_min_f64 v[66:67], v[20:21], v[68:69] +; GFX11-NEXT: v_cmp_u_f64_e64 s9, v[20:21], v[68:69] +; GFX11-NEXT: s_waitcnt vmcnt(8) +; GFX11-NEXT: v_min_f64 v[68:69], v[22:23], v[70:71] +; GFX11-NEXT: v_cmp_u_f64_e64 s10, v[22:23], v[70:71] +; GFX11-NEXT: s_waitcnt vmcnt(6) +; GFX11-NEXT: v_min_f64 v[70:71], v[24:25], v[80:81] +; GFX11-NEXT: v_cmp_u_f64_e64 s11, v[24:25], v[80:81] +; GFX11-NEXT: s_waitcnt vmcnt(4) +; GFX11-NEXT: v_min_f64 v[80:81], v[26:27], v[82:83] +; GFX11-NEXT: v_cmp_u_f64_e64 s12, v[26:27], v[82:83] +; GFX11-NEXT: s_waitcnt vmcnt(2) +; GFX11-NEXT: v_min_f64 v[82:83], v[28:29], v[84:85] +; GFX11-NEXT: v_cmp_u_f64_e64 s13, v[28:29], v[84:85] ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_cmp_lt_f64_e64 s29, v[30:31], v[82:83] -; GFX11-NEXT: v_cmp_o_f64_e64 vcc_hi, v[30:31], v[82:83] -; GFX11-NEXT: v_cndmask_b32_e64 v96, v87, v1, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v101, v86, v0, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v98, v85, v3, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v103, v84, v2, s10 -; GFX11-NEXT: v_cmp_class_f64_e64 s10, v[0:1], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v97, 0x7ff80000, v96, s11 -; GFX11-NEXT: v_cndmask_b32_e64 v96, 0, v101, s11 -; GFX11-NEXT: v_cndmask_b32_e64 v100, v33, v5, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v102, v35, v7, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v99, 0x7ff80000, v98, s12 -; GFX11-NEXT: v_cndmask_b32_e64 v98, 0, v103, s12 -; GFX11-NEXT: v_cmp_class_f64_e64 s11, v[2:3], 32 -; GFX11-NEXT: v_cndmask_b32_e32 v101, 0x7ff80000, v100, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v103, 0x7ff80000, v102, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v112, v37, v9, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v114, v39, v11, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v116, v49, v13, s8 -; GFX11-NEXT: v_cmp_o_f64_e64 s9, v[14:15], v[50:51] -; GFX11-NEXT: v_cndmask_b32_e64 v118, v51, v15, s13 -; GFX11-NEXT: v_cndmask_b32_e64 v113, 0x7ff80000, v112, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v115, 0x7ff80000, v114, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v117, 0x7ff80000, v116, s7 -; GFX11-NEXT: v_cmp_lt_f64_e64 s12, v[16:17], v[52:53] -; GFX11-NEXT: v_cndmask_b32_e64 v130, v55, v19, s17 -; GFX11-NEXT: v_cndmask_b32_e64 v132, v65, v21, s19 -; GFX11-NEXT: v_cndmask_b32_e64 v134, v67, v23, s21 -; GFX11-NEXT: v_cndmask_b32_e64 v144, v69, v25, s23 -; GFX11-NEXT: v_cndmask_b32_e64 v145, v71, v27, s25 -; GFX11-NEXT: v_cndmask_b32_e64 v131, 0x7ff80000, v130, s18 -; GFX11-NEXT: v_cndmask_b32_e64 v133, 0x7ff80000, v132, s20 -; GFX11-NEXT: v_cndmask_b32_e64 v135, 0x7ff80000, v134, s22 -; GFX11-NEXT: v_cndmask_b32_e64 v146, v81, v29, s27 -; GFX11-NEXT: v_cndmask_b32_e64 v148, v80, v28, s27 -; GFX11-NEXT: v_cndmask_b32_e64 v147, v83, v31, s29 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v147, 0x7ff80000, v147, vcc_hi -; GFX11-NEXT: v_cndmask_b32_e64 v0, v96, v0, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v97, v1, s10 -; GFX11-NEXT: v_cmp_class_f64_e64 s10, v[36:37], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v86, s14 -; GFX11-NEXT: v_cndmask_b32_e64 v86, v32, v4, s0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v87, s14 -; GFX11-NEXT: v_cndmask_b32_e64 v87, v34, v6, s2 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v98, v2, s11 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v99, v3, s11 -; GFX11-NEXT: v_cndmask_b32_e32 v100, 0, v86, vcc_lo -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[4:5], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v102, 0, v87, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v84, s16 -; GFX11-NEXT: v_cndmask_b32_e64 v84, v36, v8, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v86, v38, v10, s6 -; GFX11-NEXT: v_cndmask_b32_e64 v87, v48, v12, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v119, 0x7ff80000, v118, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v128, v53, v17, s12 -; GFX11-NEXT: v_cndmask_b32_e64 v112, 0, v84, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v114, 0, v86, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v116, 0, v87, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v84, v50, v14, s13 -; GFX11-NEXT: v_cndmask_b32_e64 v129, 0x7ff80000, v128, s15 -; GFX11-NEXT: v_cndmask_b32_e64 v86, v52, v16, s12 -; GFX11-NEXT: v_cndmask_b32_e64 v87, v54, v18, s17 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v85, s16 -; GFX11-NEXT: v_cndmask_b32_e64 v118, 0, v84, s9 -; GFX11-NEXT: v_cndmask_b32_e64 v84, v64, v20, s19 -; GFX11-NEXT: v_cndmask_b32_e64 v128, 0, v86, s15 -; GFX11-NEXT: v_cndmask_b32_e64 v130, 0, v87, s18 -; GFX11-NEXT: v_cndmask_b32_e64 v86, v66, v22, s21 -; GFX11-NEXT: v_cndmask_b32_e64 v85, 0x7ff80000, v144, s24 -; GFX11-NEXT: v_cndmask_b32_e64 v132, 0, v84, s20 -; GFX11-NEXT: v_cndmask_b32_e64 v87, v68, v24, s23 -; GFX11-NEXT: v_cndmask_b32_e64 v144, v70, v26, s25 -; GFX11-NEXT: v_cndmask_b32_e64 v134, 0, v86, s22 -; GFX11-NEXT: v_cmp_class_f64_e64 s0, v[68:69], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s1, v[70:71], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v84, 0, v87, s24 -; GFX11-NEXT: v_cndmask_b32_e64 v87, 0x7ff80000, v145, s26 -; GFX11-NEXT: v_cndmask_b32_e64 v86, 0, v144, s26 -; GFX11-NEXT: v_cndmask_b32_e64 v145, 0x7ff80000, v146, s28 -; GFX11-NEXT: v_cndmask_b32_e64 v144, 0, v148, s28 -; GFX11-NEXT: v_cndmask_b32_e64 v146, v82, v30, s29 -; GFX11-NEXT: v_cmp_class_f64_e64 s2, v[80:81], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s3, v[82:83], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s6, v[32:33], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s8, v[34:35], 32 -; GFX11-NEXT: v_dual_cndmask_b32 v5, v101, v5 :: v_dual_cndmask_b32 v4, v100, v4 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[6:7], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v146, 0, v146, vcc_hi -; GFX11-NEXT: v_cmp_class_f64_e64 s12, v[38:39], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s14, v[48:49], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s16, v[50:51], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s18, v[52:53], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s20, v[54:55], 32 -; GFX11-NEXT: v_cmp_class_f64_e64 s21, v[64:65], 32 -; GFX11-NEXT: v_cmp_eq_f64_e64 s4, 0, v[96:97] -; GFX11-NEXT: v_cmp_eq_f64_e64 s5, 0, v[98:99] -; GFX11-NEXT: v_cmp_eq_f64_e64 s7, 0, v[100:101] -; GFX11-NEXT: v_cmp_eq_f64_e64 s9, 0, v[102:103] -; GFX11-NEXT: v_cmp_eq_f64_e64 s11, 0, v[112:113] -; GFX11-NEXT: v_cmp_eq_f64_e64 s13, 0, v[114:115] -; GFX11-NEXT: v_cmp_eq_f64_e64 s15, 0, v[116:117] -; GFX11-NEXT: v_cmp_eq_f64_e64 s17, 0, v[118:119] -; GFX11-NEXT: v_cmp_eq_f64_e64 s19, 0, v[128:129] -; GFX11-NEXT: v_cmp_eq_f64_e64 s22, 0, v[130:131] -; GFX11-NEXT: v_cmp_eq_f64_e64 s23, 0, v[132:133] -; GFX11-NEXT: v_cmp_eq_f64_e64 s24, 0, v[134:135] -; GFX11-NEXT: v_cmp_eq_f64_e64 s25, 0, v[84:85] -; GFX11-NEXT: v_cmp_eq_f64_e64 s26, 0, v[86:87] -; GFX11-NEXT: v_cmp_eq_f64_e64 s27, 0, v[144:145] -; GFX11-NEXT: v_cmp_eq_f64_e64 s28, 0, v[146:147] -; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v33, s6 -; GFX11-NEXT: v_dual_cndmask_b32 v7, v103, v7 :: v_dual_cndmask_b32 v6, v102, v6 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[8:9], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v32, s6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e64 v7, v7, v35, s8 -; GFX11-NEXT: v_cndmask_b32_e64 v0, v96, v0, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v2, v98, v2, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v4, v100, v4, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v1, v97, v1, s4 -; GFX11-NEXT: v_cndmask_b32_e64 v3, v99, v3, s5 -; GFX11-NEXT: v_cndmask_b32_e64 v5, v101, v5, s7 -; GFX11-NEXT: v_cndmask_b32_e64 v7, v103, v7, s9 -; GFX11-NEXT: v_dual_cndmask_b32 v9, v113, v9 :: v_dual_cndmask_b32 v8, v112, v8 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[10:11], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v34, s8 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v37, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v6, v102, v6, s9 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v9, v113, v9, s11 -; GFX11-NEXT: v_dual_cndmask_b32 v11, v115, v11 :: v_dual_cndmask_b32 v10, v114, v10 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[12:13], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v8, v8, v36, s10 -; GFX11-NEXT: v_cndmask_b32_e64 v11, v11, v39, s12 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v8, v112, v8, s11 -; GFX11-NEXT: v_cndmask_b32_e64 v11, v115, v11, s13 -; GFX11-NEXT: v_dual_cndmask_b32 v13, v117, v13 :: v_dual_cndmask_b32 v12, v116, v12 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[14:15], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v38, s12 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v13, v13, v49, s14 -; GFX11-NEXT: v_cndmask_b32_e64 v10, v114, v10, s13 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v13, v117, v13, s15 -; GFX11-NEXT: v_dual_cndmask_b32 v15, v119, v15 :: v_dual_cndmask_b32 v14, v118, v14 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[16:17], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v12, v12, v48, s14 -; GFX11-NEXT: v_cndmask_b32_e64 v15, v15, v51, s16 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v12, v116, v12, s15 -; GFX11-NEXT: v_cndmask_b32_e64 v15, v119, v15, s17 -; GFX11-NEXT: v_dual_cndmask_b32 v17, v129, v17 :: v_dual_cndmask_b32 v16, v128, v16 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[18:19], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v14, v14, v50, s16 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v17, v17, v53, s18 -; GFX11-NEXT: v_cndmask_b32_e64 v14, v118, v14, s17 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v17, v129, v17, s19 -; GFX11-NEXT: v_dual_cndmask_b32 v19, v131, v19 :: v_dual_cndmask_b32 v18, v130, v18 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[20:21], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v16, v16, v52, s18 -; GFX11-NEXT: v_cndmask_b32_e64 v19, v19, v55, s20 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v16, v128, v16, s19 -; GFX11-NEXT: v_cndmask_b32_e64 v19, v131, v19, s22 -; GFX11-NEXT: v_dual_cndmask_b32 v21, v133, v21 :: v_dual_cndmask_b32 v20, v132, v20 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[22:23], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v18, v18, v54, s20 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v21, v21, v65, s21 -; GFX11-NEXT: v_cndmask_b32_e64 v18, v130, v18, s22 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_cndmask_b32_e64 v21, v133, v21, s23 -; GFX11-NEXT: v_dual_cndmask_b32 v23, v135, v23 :: v_dual_cndmask_b32 v22, v134, v22 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[24:25], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v20, v20, v64, s21 -; GFX11-NEXT: v_cndmask_b32_e64 v20, v132, v20, s23 -; GFX11-NEXT: v_dual_cndmask_b32 v25, v85, v25 :: v_dual_cndmask_b32 v24, v84, v24 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[26:27], 32 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_cndmask_b32_e64 v25, v25, v69, s0 -; GFX11-NEXT: v_cndmask_b32_e64 v25, v85, v25, s25 -; GFX11-NEXT: v_dual_cndmask_b32 v27, v87, v27 :: v_dual_cndmask_b32 v26, v86, v26 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[28:29], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v24, v24, v68, s0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v27, v27, v71, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v24, v84, v24, s25 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v27, v87, v27, s26 -; GFX11-NEXT: v_dual_cndmask_b32 v29, v145, v29 :: v_dual_cndmask_b32 v28, v144, v28 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[30:31], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v26, v26, v70, s1 -; GFX11-NEXT: v_cndmask_b32_e64 v29, v29, v81, s2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v26, v86, v26, s26 -; GFX11-NEXT: v_cndmask_b32_e64 v29, v145, v29, s27 -; GFX11-NEXT: v_dual_cndmask_b32 v31, v147, v31 :: v_dual_cndmask_b32 v30, v146, v30 -; GFX11-NEXT: v_cmp_class_f64_e64 vcc_lo, v[66:67], 32 -; GFX11-NEXT: v_cndmask_b32_e64 v28, v28, v80, s2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v31, v31, v83, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v28, v144, v28, s27 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e64 v31, v147, v31, s28 -; GFX11-NEXT: v_dual_cndmask_b32 v23, v23, v67 :: v_dual_cndmask_b32 v22, v22, v66 -; GFX11-NEXT: v_cndmask_b32_e64 v30, v30, v82, s3 -; GFX11-NEXT: v_cndmask_b32_e64 v23, v135, v23, s24 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e64 v22, v134, v22, s24 -; GFX11-NEXT: v_cndmask_b32_e64 v30, v146, v30, s28 +; GFX11-NEXT: v_min_f64 v[84:85], v[30:31], v[86:87] +; GFX11-NEXT: v_cmp_u_f64_e64 s14, v[30:31], v[86:87] +; GFX11-NEXT: v_cndmask_b32_e64 v0, v96, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v97, 0x7ff80000, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v32, 0, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v3, v33, 0x7ff80000, s0 +; GFX11-NEXT: v_cndmask_b32_e64 v4, v34, 0, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v5, v35, 0x7ff80000, s1 +; GFX11-NEXT: v_cndmask_b32_e64 v6, v36, 0, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v7, v37, 0x7ff80000, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v8, v38, 0, s3 +; GFX11-NEXT: v_cndmask_b32_e64 v9, v39, 0x7ff80000, s3 +; GFX11-NEXT: v_cndmask_b32_e64 v10, v48, 0, s4 +; GFX11-NEXT: v_cndmask_b32_e64 v11, v49, 0x7ff80000, s4 +; GFX11-NEXT: v_cndmask_b32_e64 v12, v50, 0, s5 +; GFX11-NEXT: v_cndmask_b32_e64 v13, v51, 0x7ff80000, s5 +; GFX11-NEXT: v_cndmask_b32_e64 v14, v52, 0, s6 +; GFX11-NEXT: v_cndmask_b32_e64 v15, v53, 0x7ff80000, s6 +; GFX11-NEXT: v_cndmask_b32_e64 v16, v54, 0, s7 +; GFX11-NEXT: v_cndmask_b32_e64 v17, v55, 0x7ff80000, s7 +; GFX11-NEXT: v_cndmask_b32_e64 v18, v64, 0, s8 +; GFX11-NEXT: v_cndmask_b32_e64 v19, v65, 0x7ff80000, s8 +; GFX11-NEXT: v_cndmask_b32_e64 v20, v66, 0, s9 +; GFX11-NEXT: v_cndmask_b32_e64 v21, v67, 0x7ff80000, s9 +; GFX11-NEXT: v_cndmask_b32_e64 v22, v68, 0, s10 +; GFX11-NEXT: v_cndmask_b32_e64 v23, v69, 0x7ff80000, s10 +; GFX11-NEXT: v_cndmask_b32_e64 v24, v70, 0, s11 +; GFX11-NEXT: v_cndmask_b32_e64 v25, v71, 0x7ff80000, s11 +; GFX11-NEXT: v_cndmask_b32_e64 v26, v80, 0, s12 +; GFX11-NEXT: v_cndmask_b32_e64 v27, v81, 0x7ff80000, s12 +; GFX11-NEXT: v_cndmask_b32_e64 v28, v82, 0, s13 +; GFX11-NEXT: v_cndmask_b32_e64 v29, v83, 0x7ff80000, s13 +; GFX11-NEXT: v_cndmask_b32_e64 v30, v84, 0, s14 +; GFX11-NEXT: v_cndmask_b32_e64 v31, v85, 0x7ff80000, s14 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_minimum_v16f64: diff --git a/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll index 1782e5252870..55b86cadfe30 100644 --- a/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll +++ b/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll @@ -28,35 +28,17 @@ define half @test_fminimum(half %x, half %y) { define <8 x half> @test_fminimum_scalarize(<8 x half> %x, <8 x half> %y) "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" { ; CHECK-LABEL: test_fminimum_scalarize: ; CHECK: # %bb.0: -; CHECK-NEXT: vpsrldq {{.*#+}} xmm2 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vminsh %xmm2, %xmm3, %xmm2 -; CHECK-NEXT: vshufps {{.*#+}} xmm3 = xmm1[3,3,3,3] -; CHECK-NEXT: vshufps {{.*#+}} xmm4 = xmm0[3,3,3,3] -; CHECK-NEXT: vminsh %xmm3, %xmm4, %xmm3 -; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; CHECK-NEXT: vpsrldq {{.*#+}} xmm3 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vpsrldq {{.*#+}} xmm4 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vminsh %xmm3, %xmm4, %xmm3 -; CHECK-NEXT: vshufpd {{.*#+}} xmm4 = xmm1[1,0] -; CHECK-NEXT: vshufpd {{.*#+}} xmm5 = xmm0[1,0] -; CHECK-NEXT: vminsh %xmm4, %xmm5, %xmm4 -; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3] -; CHECK-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; CHECK-NEXT: vpsrlq $48, %xmm1, %xmm3 -; CHECK-NEXT: vpsrlq $48, %xmm0, %xmm4 -; CHECK-NEXT: vminsh %xmm3, %xmm4, %xmm3 -; CHECK-NEXT: vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3] -; CHECK-NEXT: vmovshdup {{.*#+}} xmm5 = xmm0[1,1,3,3] -; CHECK-NEXT: vminsh %xmm4, %xmm5, %xmm4 -; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3] -; CHECK-NEXT: vminsh %xmm1, %xmm0, %xmm4 -; CHECK-NEXT: vpsrld $16, %xmm1, %xmm1 -; CHECK-NEXT: vpsrld $16, %xmm0, %xmm0 -; CHECK-NEXT: vminsh %xmm1, %xmm0, %xmm0 -; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3] -; CHECK-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1] -; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] +; CHECK-NEXT: vcmpltph %xmm1, %xmm0, %k1 +; CHECK-NEXT: vpblendmw %xmm0, %xmm1, %xmm2 {%k1} +; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm3 = [32768,32768,32768,32768,32768,32768,32768,32768] +; CHECK-NEXT: vpcmpeqw %xmm3, %xmm0, %k1 +; CHECK-NEXT: vpblendmw %xmm0, %xmm2, %xmm0 {%k1} +; CHECK-NEXT: vpcmpeqw %xmm3, %xmm1, %k1 +; CHECK-NEXT: vmovdqu16 %xmm1, %xmm0 {%k1} +; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1 +; CHECK-NEXT: vcmpeqph %xmm1, %xmm2, %k1 +; CHECK-NEXT: vmovdqu16 %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovdqa %xmm2, %xmm0 ; CHECK-NEXT: retq %r = call <8 x half> @llvm.minimum.v8f16(<8 x half> %x, <8 x half> %y) ret <8 x half> %r @@ -134,35 +116,16 @@ define half @test_fmaximum(half %x, half %y) { define <8 x half> @test_fmaximum_scalarize(<8 x half> %x, <8 x half> %y) "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" { ; CHECK-LABEL: test_fmaximum_scalarize: ; CHECK: # %bb.0: -; CHECK-NEXT: vpsrldq {{.*#+}} xmm2 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vmaxsh %xmm2, %xmm3, %xmm2 -; CHECK-NEXT: vshufps {{.*#+}} xmm3 = xmm1[3,3,3,3] -; CHECK-NEXT: vshufps {{.*#+}} xmm4 = xmm0[3,3,3,3] -; CHECK-NEXT: vmaxsh %xmm3, %xmm4, %xmm3 -; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; CHECK-NEXT: vpsrldq {{.*#+}} xmm3 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vpsrldq {{.*#+}} xmm4 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; CHECK-NEXT: vmaxsh %xmm3, %xmm4, %xmm3 -; CHECK-NEXT: vshufpd {{.*#+}} xmm4 = xmm1[1,0] -; CHECK-NEXT: vshufpd {{.*#+}} xmm5 = xmm0[1,0] -; CHECK-NEXT: vmaxsh %xmm4, %xmm5, %xmm4 -; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3] -; CHECK-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; CHECK-NEXT: vpsrlq $48, %xmm1, %xmm3 -; CHECK-NEXT: vpsrlq $48, %xmm0, %xmm4 -; CHECK-NEXT: vmaxsh %xmm3, %xmm4, %xmm3 -; CHECK-NEXT: vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3] -; CHECK-NEXT: vmovshdup {{.*#+}} xmm5 = xmm0[1,1,3,3] -; CHECK-NEXT: vmaxsh %xmm4, %xmm5, %xmm4 -; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3] -; CHECK-NEXT: vmaxsh %xmm1, %xmm0, %xmm4 -; CHECK-NEXT: vpsrld $16, %xmm1, %xmm1 -; CHECK-NEXT: vpsrld $16, %xmm0, %xmm0 -; CHECK-NEXT: vmaxsh %xmm1, %xmm0, %xmm0 -; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3] -; CHECK-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1] -; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] +; CHECK-NEXT: vcmpltph %xmm0, %xmm1, %k1 +; CHECK-NEXT: vpblendmw %xmm0, %xmm1, %xmm2 {%k1} +; CHECK-NEXT: vptestnmw %xmm0, %xmm0, %k1 +; CHECK-NEXT: vpblendmw %xmm0, %xmm2, %xmm0 {%k1} +; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1 +; CHECK-NEXT: vmovdqu16 %xmm1, %xmm0 {%k1} +; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1 +; CHECK-NEXT: vcmpeqph %xmm1, %xmm2, %k1 +; CHECK-NEXT: vmovdqu16 %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovdqa %xmm2, %xmm0 ; CHECK-NEXT: retq %r = call <8 x half> @llvm.maximum.v8f16(<8 x half> %x, <8 x half> %y) ret <8 x half> %r -- GitLab From 429e5be768c21d208ab688f8dfa1399c04ec5626 Mon Sep 17 00:00:00 2001 From: Piotr Zegar Date: Thu, 6 Jun 2024 19:03:37 +0200 Subject: [PATCH 132/462] [clang-tidy] Fix crash in readability-container-size-empty (#94527) Fixed crash caused by call to getCookedLiteral on template user defined literal. Fix base on assert in getCookedLiteral method. Closes #94454 --- .../clang-tidy/readability/ContainerSizeEmptyCheck.cpp | 9 +++++++-- clang-tools-extra/docs/ReleaseNotes.rst | 1 + .../checkers/readability/container-size-empty.cpp | 6 ++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp b/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp index bbc1b47b97ae..bf7a847dff10 100644 --- a/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp @@ -96,9 +96,14 @@ AST_MATCHER(QualType, isIntegralType) { AST_MATCHER_P(UserDefinedLiteral, hasLiteral, clang::ast_matchers::internal::Matcher, InnerMatcher) { - if (const Expr *CookedLiteral = Node.getCookedLiteral()) { + const UserDefinedLiteral::LiteralOperatorKind LOK = + Node.getLiteralOperatorKind(); + if (LOK == UserDefinedLiteral::LOK_Template || + LOK == UserDefinedLiteral::LOK_Raw) + return false; + + if (const Expr *CookedLiteral = Node.getCookedLiteral()) return InnerMatcher.matches(*CookedLiteral, Finder, Builder); - } return false; } diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 6947cf06f6e5..661b2b1620d0 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -376,6 +376,7 @@ Changes in existing checks - Improved :doc:`readability-container-size-empty ` check to prevent false positives when utilizing ``size`` or ``length`` methods that accept parameter. + Fixed crash when facing template user defined literals. - Improved :doc:`readability-duplicate-include ` check by excluding include diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp index ecaf97fa348c..46755270b48e 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp @@ -889,3 +889,9 @@ namespace PR88203 { // CHECK-FIXES: {{^ }}if (s.empty()) {}{{$}} } } + +namespace PR94454 { + template + int operator""_ci() { return 0; } + auto eq = 0_ci == 0; +} -- GitLab From e8500a70540a04adfb8e102d3cfa6b9d95bc3ba6 Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Thu, 6 Jun 2024 10:10:00 -0700 Subject: [PATCH 133/462] fixup cuda-builtin-vars.cu broken in IntrRange change (#94639) --- clang/test/CodeGenCUDA/cuda-builtin-vars.cu | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/clang/test/CodeGenCUDA/cuda-builtin-vars.cu b/clang/test/CodeGenCUDA/cuda-builtin-vars.cu index dba0a76af21d..7880a8036f8c 100644 --- a/clang/test/CodeGenCUDA/cuda-builtin-vars.cu +++ b/clang/test/CodeGenCUDA/cuda-builtin-vars.cu @@ -6,21 +6,21 @@ __attribute__((global)) void kernel(int *out) { int i = 0; - out[i++] = threadIdx.x; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.tid.x() - out[i++] = threadIdx.y; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.tid.y() - out[i++] = threadIdx.z; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.tid.z() + out[i++] = threadIdx.x; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.tid.x() + out[i++] = threadIdx.y; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.tid.y() + out[i++] = threadIdx.z; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.tid.z() - out[i++] = blockIdx.x; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() - out[i++] = blockIdx.y; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.ctaid.y() - out[i++] = blockIdx.z; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.ctaid.z() + out[i++] = blockIdx.x; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() + out[i++] = blockIdx.y; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.ctaid.y() + out[i++] = blockIdx.z; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.ctaid.z() - out[i++] = blockDim.x; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.ntid.x() - out[i++] = blockDim.y; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.ntid.y() - out[i++] = blockDim.z; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.ntid.z() + out[i++] = blockDim.x; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.ntid.x() + out[i++] = blockDim.y; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.ntid.y() + out[i++] = blockDim.z; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.ntid.z() - out[i++] = gridDim.x; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.nctaid.x() - out[i++] = gridDim.y; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.nctaid.y() - out[i++] = gridDim.z; // CHECK: call noundef {{.*}} i32 @llvm.nvvm.read.ptx.sreg.nctaid.z() + out[i++] = gridDim.x; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.nctaid.x() + out[i++] = gridDim.y; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.nctaid.y() + out[i++] = gridDim.z; // CHECK: call noundef{{.*}} i32 @llvm.nvvm.read.ptx.sreg.nctaid.z() out[i++] = warpSize; // CHECK: store i32 32, -- GitLab From 39d38d66ec1cde07cfb959d1cf94b0adc6eb16ef Mon Sep 17 00:00:00 2001 From: aaryanshukla <53713108+aaryanshukla@users.noreply.github.com> Date: Thu, 6 Jun 2024 17:21:44 +0000 Subject: [PATCH 134/462] [libc] at_quick_exit function implemented (#94317) - added at_quick_exit function - used helper file exit_handler which reuses code from atexit - atexit now calls helper functions from exit_handler - test cases and dependencies are added --------- Co-authored-by: Aaryan Shukla --- libc/config/linux/x86_64/entrypoints.txt | 3 +- libc/hdr/types/CMakeLists.txt | 9 +++ libc/hdr/types/atexithandler_t.h | 22 +++++ libc/spec/stdc.td | 3 +- libc/src/stdlib/CMakeLists.txt | 32 ++++++-- libc/src/stdlib/at_quick_exit.cpp | 22 +++++ libc/src/stdlib/at_quick_exit.h | 20 +++++ libc/src/stdlib/atexit.cpp | 79 ++---------------- libc/src/stdlib/atexit.h | 7 +- libc/src/stdlib/exit_handler.cpp | 42 ++++++++++ libc/src/stdlib/exit_handler.h | 53 ++++++++++++ libc/src/stdlib/quick_exit.cpp | 6 +- libc/test/src/stdlib/CMakeLists.txt | 15 +++- libc/test/src/stdlib/at_quick_exit_test.cpp | 90 +++++++++++++++++++++ 14 files changed, 315 insertions(+), 88 deletions(-) create mode 100644 libc/hdr/types/atexithandler_t.h create mode 100644 libc/src/stdlib/at_quick_exit.cpp create mode 100644 libc/src/stdlib/at_quick_exit.h create mode 100644 libc/src/stdlib/exit_handler.cpp create mode 100644 libc/src/stdlib/exit_handler.h create mode 100644 libc/test/src/stdlib/at_quick_exit_test.cpp diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index ebacb1c59cee..e3ca544ae018 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -173,7 +173,6 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.atoll libc.src.stdlib.bsearch libc.src.stdlib.div - libc.src.stdlib.quick_exit libc.src.stdlib.labs libc.src.stdlib.ldiv libc.src.stdlib.llabs @@ -770,9 +769,11 @@ if(LLVM_LIBC_FULL_BUILD) # stdlib.h entrypoints libc.src.stdlib._Exit libc.src.stdlib.abort + libc.src.stdlib.at_quick_exit libc.src.stdlib.atexit libc.src.stdlib.exit libc.src.stdlib.getenv + libc.src.stdlib.quick_exit # signal.h entrypoints libc.src.signal.raise diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt index 8e87642f6608..9b3373a0ca39 100644 --- a/libc/hdr/types/CMakeLists.txt +++ b/libc/hdr/types/CMakeLists.txt @@ -117,3 +117,12 @@ add_proxy_header_library( libc.include.llvm-libc-types.pid_t libc.include.sys_types ) + +add_proxy_header_library( + atexithandler_t + HDRS + atexithandler_t.h + FULL_BUILD_DEPENDS + libc.include.llvm-libc-types.atexithandler_t + libc.include.stdlib +) diff --git a/libc/hdr/types/atexithandler_t.h b/libc/hdr/types/atexithandler_t.h new file mode 100644 index 000000000000..4275e4407367 --- /dev/null +++ b/libc/hdr/types/atexithandler_t.h @@ -0,0 +1,22 @@ +//===-- Definition of macros from atexithandler_t.h -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_ATEXITHANDLER_T_H +#define LLVM_LIBC_HDR_ATEXITHANDLER_T_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-types/__atexithandler_t.h" + +#else // overlay mode + +#error // type not available in overlay mode + +#endif // LLVM_LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_ATEXITHANDLER_T_H diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 210f2a132516..9a436c8ae38d 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -1107,8 +1107,9 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"free", RetValSpec, [ArgSpec]>, FunctionSpec<"_Exit", RetValSpec, [ArgSpec]>, - FunctionSpec<"exit", RetValSpec, [ArgSpec]>, + FunctionSpec<"at_quick_exit", RetValSpec, [ArgSpec]>, FunctionSpec<"atexit", RetValSpec, [ArgSpec]>, + FunctionSpec<"exit", RetValSpec, [ArgSpec]>, FunctionSpec<"quick_exit", RetValSpec, [ArgSpec]>, ] >; diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt index e0bff5198b59..219c85dda675 100644 --- a/libc/src/stdlib/CMakeLists.txt +++ b/libc/src/stdlib/CMakeLists.txt @@ -50,6 +50,7 @@ add_entrypoint_object( quick_exit.h DEPENDS libc.src.__support.OSUtil.osutil + .exit_handler ) add_entrypoint_object( @@ -415,14 +416,14 @@ add_entrypoint_object( libc.src.__support.OSUtil.osutil ) -add_entrypoint_object( - atexit +add_object_library( + exit_handler SRCS - atexit.cpp + exit_handler.cpp HDRS - atexit.h + exit_handler.h CXX_STANDARD - 20 # For constinit of the atexit callback list. + 20 # For constinit DEPENDS libc.src.__support.CPP.mutex libc.src.__support.CPP.new @@ -432,6 +433,26 @@ add_entrypoint_object( libc.src.__support.threads.mutex ) +add_entrypoint_object( + atexit + SRCS + atexit.cpp + HDRS + atexit.h + DEPENDS + .exit_handler +) + +add_entrypoint_object( + at_quick_exit + SRCS + at_quick_exit.cpp + HDRS + at_quick_exit.h + DEPENDS + .exit_handler +) + add_entrypoint_object( exit SRCS @@ -442,6 +463,7 @@ add_entrypoint_object( ._Exit .atexit libc.src.__support.OSUtil.osutil + .exit_handler ) add_entrypoint_object( diff --git a/libc/src/stdlib/at_quick_exit.cpp b/libc/src/stdlib/at_quick_exit.cpp new file mode 100644 index 000000000000..752d67e7fe44 --- /dev/null +++ b/libc/src/stdlib/at_quick_exit.cpp @@ -0,0 +1,22 @@ +//===-- Implementation of at_quick_exit -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdlib/at_quick_exit.h" +#include "hdr/types/atexithandler_t.h" +#include "src/__support/common.h" +#include "src/stdlib/exit_handler.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, at_quick_exit, (__atexithandler_t callback)) { + return add_atexit_unit( + at_quick_exit_callbacks, + {&stdc_at_exit_func, reinterpret_cast(callback)}); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdlib/at_quick_exit.h b/libc/src/stdlib/at_quick_exit.h new file mode 100644 index 000000000000..c36c797088ab --- /dev/null +++ b/libc/src/stdlib/at_quick_exit.h @@ -0,0 +1,20 @@ +//===-- Implementation header for at_quick_exit -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDLIB_AT_QUICK_EXIT_H +#define LLVM_LIBC_SRC_STDLIB_AT_QUICK_EXIT_H + +#include "hdr/types/atexithandler_t.h" + +namespace LIBC_NAMESPACE { + +int at_quick_exit(__atexithandler_t); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDLIB_AT_QUICK_EXIT_H diff --git a/libc/src/stdlib/atexit.cpp b/libc/src/stdlib/atexit.cpp index 9e37c4cf256c..ca3cbfe87a88 100644 --- a/libc/src/stdlib/atexit.cpp +++ b/libc/src/stdlib/atexit.cpp @@ -7,95 +7,28 @@ //===----------------------------------------------------------------------===// #include "src/stdlib/atexit.h" -#include "src/__support/CPP/mutex.h" // lock_guard -#include "src/__support/blockstore.h" +#include "hdr/types/atexithandler_t.h" #include "src/__support/common.h" -#include "src/__support/fixedvector.h" -#include "src/__support/threads/mutex.h" +#include "src/stdlib/exit_handler.h" namespace LIBC_NAMESPACE { -namespace { - -Mutex handler_list_mtx(/*timed=*/false, /*recursive=*/false, /*robust=*/false, - /*pshared=*/false); - -using AtExitCallback = void(void *); -using StdCAtExitCallback = void(void); - -struct AtExitUnit { - AtExitCallback *callback = nullptr; - void *payload = nullptr; - constexpr AtExitUnit() = default; - constexpr AtExitUnit(AtExitCallback *c, void *p) : callback(c), payload(p) {} -}; - -#if defined(LIBC_TARGET_ARCH_IS_GPU) -// The GPU build cannot handle the potentially recursive definitions required by -// the BlockStore class. Additionally, the liklihood that someone exceeds this -// while executing on the GPU is extremely small. -// FIXME: It is not generally safe to use 'atexit' on the GPU because the -// mutexes simply passthrough. We will need a lock free stack. -using ExitCallbackList = FixedVector; -#elif defined(LIBC_COPT_PUBLIC_PACKAGING) -using ExitCallbackList = ReverseOrderBlockStore; -#else -// BlockStore uses dynamic memory allocation. To avoid dynamic memory -// allocation in tests, we use a fixed size callback list when built for -// tests. -// If we use BlockStore, then we will have to pull in malloc etc into -// the tests. While this is not bad, the problem we have currently is -// that LLVM libc' allocator is SCUDO. So, we will end up pulling SCUDO's -// deps also (some of which are not yet available in LLVM libc) into the -// integration tests. -using ExitCallbackList = FixedVector; -#endif // LIBC_COPT_PUBLIC_PACKAGING - -constinit ExitCallbackList exit_callbacks; - -void stdc_at_exit_func(void *payload) { - reinterpret_cast(payload)(); -} - -void call_exit_callbacks() { - handler_list_mtx.lock(); - while (!exit_callbacks.empty()) { - AtExitUnit &unit = exit_callbacks.back(); - exit_callbacks.pop_back(); - handler_list_mtx.unlock(); - unit.callback(unit.payload); - handler_list_mtx.lock(); - } - ExitCallbackList::destroy(&exit_callbacks); -} - -int add_atexit_unit(const AtExitUnit &unit) { - cpp::lock_guard lock(handler_list_mtx); - if (exit_callbacks.push_back(unit)) - return 0; - return -1; -} - -} // namespace - extern "C" { -// TODO: Handle the last dso handle argument. int __cxa_atexit(AtExitCallback *callback, void *payload, void *) { - return add_atexit_unit({callback, payload}); + return add_atexit_unit(atexit_callbacks, {callback, payload}); } -// TODO: Handle the dso handle argument. call_exit_callbacks should only invoke -// the callbacks from this DSO. Requires adding support for __dso_handle. void __cxa_finalize(void *dso) { if (!dso) - call_exit_callbacks(); + call_exit_callbacks(atexit_callbacks); } } // extern "C" -LLVM_LIBC_FUNCTION(int, atexit, (StdCAtExitCallback * callback)) { +LLVM_LIBC_FUNCTION(int, atexit, (__atexithandler_t callback)) { return add_atexit_unit( + atexit_callbacks, {&stdc_at_exit_func, reinterpret_cast(callback)}); } diff --git a/libc/src/stdlib/atexit.h b/libc/src/stdlib/atexit.h index 7cf9d7c92191..7faaf654247c 100644 --- a/libc/src/stdlib/atexit.h +++ b/libc/src/stdlib/atexit.h @@ -9,13 +9,10 @@ #ifndef LLVM_LIBC_SRC_STDLIB_ATEXIT_H #define LLVM_LIBC_SRC_STDLIB_ATEXIT_H -#include // For size_t - +#include "hdr/types/atexithandler_t.h" namespace LIBC_NAMESPACE { -constexpr size_t CALLBACK_LIST_SIZE_FOR_TESTS = 1024; - -int atexit(void (*function)()); +int atexit(__atexithandler_t); } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdlib/exit_handler.cpp b/libc/src/stdlib/exit_handler.cpp new file mode 100644 index 000000000000..ed41247e4a31 --- /dev/null +++ b/libc/src/stdlib/exit_handler.cpp @@ -0,0 +1,42 @@ +//===--- Implementation of exit_handler------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdlib/exit_handler.h" +#include "src/__support/CPP/mutex.h" // lock_guard + +namespace LIBC_NAMESPACE { + +constinit ExitCallbackList at_quick_exit_callbacks; +constinit ExitCallbackList atexit_callbacks; + +Mutex handler_list_mtx(false, false, false, false); + +void stdc_at_exit_func(void *payload) { + reinterpret_cast(payload)(); +} + +void call_exit_callbacks(ExitCallbackList &callbacks) { + handler_list_mtx.lock(); + while (!callbacks.empty()) { + AtExitUnit &unit = callbacks.back(); + callbacks.pop_back(); + handler_list_mtx.unlock(); + unit.callback(unit.payload); + handler_list_mtx.lock(); + } + ExitCallbackList::destroy(&callbacks); +} + +int add_atexit_unit(ExitCallbackList &callbacks, const AtExitUnit &unit) { + cpp::lock_guard lock(handler_list_mtx); + if (callbacks.push_back(unit)) + return 0; + return -1; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdlib/exit_handler.h b/libc/src/stdlib/exit_handler.h new file mode 100644 index 000000000000..8494c2f2e526 --- /dev/null +++ b/libc/src/stdlib/exit_handler.h @@ -0,0 +1,53 @@ +//===-- Implementation header for exit_handler ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDLIB_EXIT_HANDLER_H +#define LLVM_LIBC_SRC_STDLIB_EXIT_HANDLER_H + +#include "src/__support/CPP/mutex.h" // lock_guard +#include "src/__support/blockstore.h" +#include "src/__support/common.h" +#include "src/__support/fixedvector.h" +#include "src/__support/threads/mutex.h" + +namespace LIBC_NAMESPACE { + +using AtExitCallback = void(void *); +using StdCAtExitCallback = void(void); +constexpr size_t CALLBACK_LIST_SIZE_FOR_TESTS = 1024; + +struct AtExitUnit { + AtExitCallback *callback = nullptr; + void *payload = nullptr; + LIBC_INLINE constexpr AtExitUnit() = default; + LIBC_INLINE constexpr AtExitUnit(AtExitCallback *c, void *p) + : callback(c), payload(p) {} +}; + +#if defined(LIBC_TARGET_ARCH_IS_GPU) +using ExitCallbackList = FixedVector; +#elif defined(LIBC_COPT_PUBLIC_PACKAGING) +using ExitCallbackList = ReverseOrderBlockStore; +#else +using ExitCallbackList = FixedVector; +#endif + +extern ExitCallbackList atexit_callbacks; +extern ExitCallbackList at_quick_exit_callbacks; + +extern Mutex handler_list_mtx; + +void stdc_at_exit_func(void *payload); + +void call_exit_callbacks(ExitCallbackList &callbacks); + +int add_atexit_unit(ExitCallbackList &callbacks, const AtExitUnit &unit); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDLIB_EXIT_HANDLER_H diff --git a/libc/src/stdlib/quick_exit.cpp b/libc/src/stdlib/quick_exit.cpp index cf7f07bf2439..38f0a3db3e2c 100644 --- a/libc/src/stdlib/quick_exit.cpp +++ b/libc/src/stdlib/quick_exit.cpp @@ -9,13 +9,15 @@ #include "src/stdlib/quick_exit.h" #include "src/__support/OSUtil/exit.h" #include "src/__support/common.h" +#include "src/stdlib/exit_handler.h" // extern "C" void __cxa_finalize(void *); - namespace LIBC_NAMESPACE { +extern ExitCallbackList at_quick_exit_callbacks; + [[noreturn]] LLVM_LIBC_FUNCTION(void, quick_exit, (int status)) { - // __cxa_finalize(nullptr); + call_exit_callbacks(at_quick_exit_callbacks); internal::exit(status); } diff --git a/libc/test/src/stdlib/CMakeLists.txt b/libc/test/src/stdlib/CMakeLists.txt index 6a7faedece38..38488778c657 100644 --- a/libc/test/src/stdlib/CMakeLists.txt +++ b/libc/test/src/stdlib/CMakeLists.txt @@ -354,7 +354,20 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.stdlib.exit libc.src.stdlib.atexit libc.src.__support.CPP.array - libc.src.__support.CPP.utility + ) + + add_libc_test( + at_quick_exit_test + # The EXPECT_EXITS test is only availible for unit tests. + UNIT_TEST_ONLY + SUITE + libc-stdlib-tests + SRCS + at_quick_exit_test.cpp + DEPENDS + libc.src.stdlib.quick_exit + libc.src.stdlib.at_quick_exit + libc.src.__support.CPP.array ) add_libc_test( diff --git a/libc/test/src/stdlib/at_quick_exit_test.cpp b/libc/test/src/stdlib/at_quick_exit_test.cpp new file mode 100644 index 000000000000..e0a258d9fb2d --- /dev/null +++ b/libc/test/src/stdlib/at_quick_exit_test.cpp @@ -0,0 +1,90 @@ +//===-- Unittests for at_quick_exit ---------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/array.h" +#include "src/__support/CPP/utility.h" +#include "src/stdlib/at_quick_exit.h" +#include "src/stdlib/quick_exit.h" +#include "test/UnitTest/Test.h" + +static int a; +TEST(LlvmLibcAtQuickExit, Basic) { + // In case tests ever run multiple times. + a = 0; + + auto test = [] { + int status = LIBC_NAMESPACE::at_quick_exit(+[] { + if (a != 1) + __builtin_trap(); + }); + status |= LIBC_NAMESPACE::at_quick_exit(+[] { a++; }); + if (status) + __builtin_trap(); + + LIBC_NAMESPACE::quick_exit(0); + }; + EXPECT_EXITS(test, 0); +} + +TEST(LlvmLibcAtQuickExit, AtQuickExitCallsSysExit) { + auto test = [] { + LIBC_NAMESPACE::at_quick_exit(+[] { _Exit(1); }); + LIBC_NAMESPACE::quick_exit(0); + }; + EXPECT_EXITS(test, 1); +} + +static int size; +static LIBC_NAMESPACE::cpp::array arr; + +template +void register_at_quick_exit_handlers( + LIBC_NAMESPACE::cpp::integer_sequence) { + (LIBC_NAMESPACE::at_quick_exit(+[] { arr[size++] = Ts; }), ...); +} + +template constexpr auto get_test() { + return [] { + LIBC_NAMESPACE::at_quick_exit(+[] { + if (size != count) + __builtin_trap(); + for (int i = 0; i < count; i++) + if (arr[i] != count - 1 - i) + __builtin_trap(); + }); + register_at_quick_exit_handlers( + LIBC_NAMESPACE::cpp::make_integer_sequence{}); + LIBC_NAMESPACE::quick_exit(0); + }; +} + +TEST(LlvmLibcAtQuickExit, ReverseOrder) { + // In case tests ever run multiple times. + size = 0; + + auto test = get_test<32>(); + EXPECT_EXITS(test, 0); +} + +TEST(LlvmLibcAtQuickExit, Many) { + // In case tests ever run multiple times. + size = 0; + + auto test = get_test<256>(); + EXPECT_EXITS(test, 0); +} + +TEST(LlvmLibcAtQuickExit, HandlerCallsAtQuickExit) { + auto test = [] { + LIBC_NAMESPACE::at_quick_exit(+[] { + LIBC_NAMESPACE::at_quick_exit(+[] { LIBC_NAMESPACE::quick_exit(1); }); + }); + LIBC_NAMESPACE::quick_exit(0); + }; + EXPECT_EXITS(test, 1); +} -- GitLab From 7eab68026d931860e9c750e8b8b29a2076370d38 Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Thu, 6 Jun 2024 13:22:07 -0400 Subject: [PATCH 135/462] [AMDGPU] Fix GFX1152 ELF arch --- llvm/include/llvm/BinaryFormat/ELF.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index fbfba515fa5f..dfba18014991 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -785,7 +785,6 @@ enum : unsigned { EF_AMDGPU_MACH_AMDGCN_GFX1200 = 0x048, EF_AMDGPU_MACH_AMDGCN_RESERVED_0X49 = 0x049, EF_AMDGPU_MACH_AMDGCN_GFX1151 = 0x04a, - EF_AMDGPU_MACH_AMDGCN_GFX1152 = 0x055, EF_AMDGPU_MACH_AMDGCN_GFX941 = 0x04b, EF_AMDGPU_MACH_AMDGCN_GFX942 = 0x04c, EF_AMDGPU_MACH_AMDGCN_RESERVED_0X4D = 0x04d, @@ -796,7 +795,7 @@ enum : unsigned { EF_AMDGPU_MACH_AMDGCN_GFX10_1_GENERIC = 0x052, EF_AMDGPU_MACH_AMDGCN_GFX10_3_GENERIC = 0x053, EF_AMDGPU_MACH_AMDGCN_GFX11_GENERIC = 0x054, - EF_AMDGPU_MACH_AMDGCN_RESERVED_0X55 = 0x055, + EF_AMDGPU_MACH_AMDGCN_GFX1152 = 0x055, EF_AMDGPU_MACH_AMDGCN_RESERVED_0X56 = 0x056, EF_AMDGPU_MACH_AMDGCN_RESERVED_0X57 = 0x057, EF_AMDGPU_MACH_AMDGCN_RESERVED_0X58 = 0x058, -- GitLab From 088b98a458b449a633e15d98ae3cead65807534e Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Thu, 6 Jun 2024 10:26:12 -0700 Subject: [PATCH 136/462] [HWASan] add optimization remarks for ignoreAccess (#94551) --- .../Instrumentation/HWAddressSanitizer.cpp | 56 +++++++++++++------ .../stack-safety-analysis.ll | 23 +++++++- 2 files changed, 62 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index 2aa21759d56e..a0e63bf12400 100644 --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -337,13 +337,17 @@ private: unsigned AccessSizeIndex, Instruction *InsertBefore, DomTreeUpdater &DTU, LoopInfo *LI); - bool ignoreMemIntrinsic(MemIntrinsic *MI); + bool ignoreMemIntrinsic(OptimizationRemarkEmitter &ORE, MemIntrinsic *MI); void instrumentMemIntrinsic(MemIntrinsic *MI); bool instrumentMemAccess(InterestingMemoryOperand &O, DomTreeUpdater &DTU, LoopInfo *LI); - bool ignoreAccess(Instruction *Inst, Value *Ptr); + bool ignoreAccessWithoutRemark(Instruction *Inst, Value *Ptr); + bool ignoreAccess(OptimizationRemarkEmitter &ORE, Instruction *Inst, + Value *Ptr); + void getInterestingMemoryOperands( - Instruction *I, const TargetLibraryInfo &TLI, + OptimizationRemarkEmitter &ORE, Instruction *I, + const TargetLibraryInfo &TLI, SmallVectorImpl &Interesting); void tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size); @@ -765,7 +769,8 @@ Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) { return IRB.CreateLoad(PtrTy, GlobalDynamicAddress); } -bool HWAddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) { +bool HWAddressSanitizer::ignoreAccessWithoutRemark(Instruction *Inst, + Value *Ptr) { // Do not instrument accesses from different address spaces; we cannot deal // with them. Type *PtrTy = cast(Ptr->getType()->getScalarType()); @@ -795,8 +800,23 @@ bool HWAddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) { return false; } +bool HWAddressSanitizer::ignoreAccess(OptimizationRemarkEmitter &ORE, + Instruction *Inst, Value *Ptr) { + bool Ignored = ignoreAccessWithoutRemark(Inst, Ptr); + if (Ignored) { + ORE.emit( + [&]() { return OptimizationRemark(DEBUG_TYPE, "ignoreAccess", Inst); }); + } else { + ORE.emit([&]() { + return OptimizationRemarkMissed(DEBUG_TYPE, "ignoreAccess", Inst); + }); + } + return Ignored; +} + void HWAddressSanitizer::getInterestingMemoryOperands( - Instruction *I, const TargetLibraryInfo &TLI, + OptimizationRemarkEmitter &ORE, Instruction *I, + const TargetLibraryInfo &TLI, SmallVectorImpl &Interesting) { // Skip memory accesses inserted by another instrumentation. if (I->hasMetadata(LLVMContext::MD_nosanitize)) @@ -807,22 +827,22 @@ void HWAddressSanitizer::getInterestingMemoryOperands( return; if (LoadInst *LI = dyn_cast(I)) { - if (!ClInstrumentReads || ignoreAccess(I, LI->getPointerOperand())) + if (!ClInstrumentReads || ignoreAccess(ORE, I, LI->getPointerOperand())) return; Interesting.emplace_back(I, LI->getPointerOperandIndex(), false, LI->getType(), LI->getAlign()); } else if (StoreInst *SI = dyn_cast(I)) { - if (!ClInstrumentWrites || ignoreAccess(I, SI->getPointerOperand())) + if (!ClInstrumentWrites || ignoreAccess(ORE, I, SI->getPointerOperand())) return; Interesting.emplace_back(I, SI->getPointerOperandIndex(), true, SI->getValueOperand()->getType(), SI->getAlign()); } else if (AtomicRMWInst *RMW = dyn_cast(I)) { - if (!ClInstrumentAtomics || ignoreAccess(I, RMW->getPointerOperand())) + if (!ClInstrumentAtomics || ignoreAccess(ORE, I, RMW->getPointerOperand())) return; Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true, RMW->getValOperand()->getType(), std::nullopt); } else if (AtomicCmpXchgInst *XCHG = dyn_cast(I)) { - if (!ClInstrumentAtomics || ignoreAccess(I, XCHG->getPointerOperand())) + if (!ClInstrumentAtomics || ignoreAccess(ORE, I, XCHG->getPointerOperand())) return; Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true, XCHG->getCompareOperand()->getType(), @@ -830,7 +850,7 @@ void HWAddressSanitizer::getInterestingMemoryOperands( } else if (auto *CI = dyn_cast(I)) { for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) { if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) || - ignoreAccess(I, CI->getArgOperand(ArgNo))) + ignoreAccess(ORE, I, CI->getArgOperand(ArgNo))) continue; Type *Ty = CI->getParamByValType(ArgNo); Interesting.emplace_back(I, ArgNo, false, Ty, Align(1)); @@ -1035,13 +1055,14 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite, ->setSuccessor(0, TCI.TagMismatchTerm->getParent()); } -bool HWAddressSanitizer::ignoreMemIntrinsic(MemIntrinsic *MI) { +bool HWAddressSanitizer::ignoreMemIntrinsic(OptimizationRemarkEmitter &ORE, + MemIntrinsic *MI) { if (MemTransferInst *MTI = dyn_cast(MI)) { - return (!ClInstrumentWrites || ignoreAccess(MTI, MTI->getDest())) && - (!ClInstrumentReads || ignoreAccess(MTI, MTI->getSource())); + return (!ClInstrumentWrites || ignoreAccess(ORE, MTI, MTI->getDest())) && + (!ClInstrumentReads || ignoreAccess(ORE, MTI, MTI->getSource())); } if (isa(MI)) - return !ClInstrumentWrites || ignoreAccess(MI, MI->getDest()); + return !ClInstrumentWrites || ignoreAccess(ORE, MI, MI->getDest()); return false; } @@ -1541,6 +1562,9 @@ void HWAddressSanitizer::sanitizeFunction(Function &F, NumTotalFuncs++; + OptimizationRemarkEmitter &ORE = + FAM.getResult(F); + if (selectiveInstrumentationShouldSkip(F, FAM)) return; @@ -1562,10 +1586,10 @@ void HWAddressSanitizer::sanitizeFunction(Function &F, if (InstrumentLandingPads && isa(Inst)) LandingPadVec.push_back(&Inst); - getInterestingMemoryOperands(&Inst, TLI, OperandsToInstrument); + getInterestingMemoryOperands(ORE, &Inst, TLI, OperandsToInstrument); if (MemIntrinsic *MI = dyn_cast(&Inst)) - if (!ignoreMemIntrinsic(MI)) + if (!ignoreMemIntrinsic(ORE, MI)) IntrinToInstrument.push_back(MI); } diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/stack-safety-analysis.ll b/llvm/test/Instrumentation/HWAddressSanitizer/stack-safety-analysis.ll index dad5f8e2fc56..8610645a4ca7 100644 --- a/llvm/test/Instrumentation/HWAddressSanitizer/stack-safety-analysis.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/stack-safety-analysis.ll @@ -1,4 +1,5 @@ -; RUN: opt -mtriple=aarch64-unknown-linux-gnu -passes=hwasan -hwasan-instrument-with-calls -hwasan-use-stack-safety=1 -hwasan-generate-tags-with-calls -S < %s | FileCheck %s --check-prefixes=SAFETY,CHECK +; RUN: opt -pass-remarks-output=%t.pass-remarks -mtriple=aarch64-unknown-linux-gnu -passes=hwasan -hwasan-instrument-with-calls -hwasan-use-stack-safety=1 -hwasan-generate-tags-with-calls -S < %s | FileCheck %s --check-prefixes=SAFETY,CHECK +; RUN: cat %t.pass-remarks | FileCheck %s --check-prefixes=SAFETY-REMARKS ; RUN: opt -mtriple=aarch64-unknown-linux-gnu -passes=hwasan -hwasan-instrument-with-calls -hwasan-use-stack-safety=0 -hwasan-generate-tags-with-calls -S < %s | FileCheck %s --check-prefixes=NOSAFETY,CHECK ; RUN: opt -mtriple=aarch64-unknown-linux-gnu -passes=hwasan -hwasan-instrument-with-calls -hwasan-generate-tags-with-calls -S < %s | FileCheck %s --check-prefixes=SAFETY,CHECK ; RUN: opt -mtriple=aarch64-unknown-linux-gnu -passes=hwasan -hwasan-instrument-stack=0 -hwasan-instrument-with-calls -hwasan-generate-tags-with-calls -S < %s | FileCheck %s --check-prefixes=NOSTACK,CHECK @@ -20,6 +21,7 @@ entry: ; SAFETY-NOT: call {{.*}}__hwasan_store ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_store + ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_simple %buf.sroa.0 = alloca i8, align 4 call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0) store volatile i8 0, ptr %buf.sroa.0, align 4, !tbaa !8 @@ -37,6 +39,7 @@ entry: ; SAFETY-NOT: call {{.*}}__hwasan_store ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_store + ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_cmpxchg %buf.sroa.0 = alloca i8, align 4 call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0) %0 = cmpxchg ptr %buf.sroa.0, i8 1, i8 2 monotonic monotonic, align 4 @@ -54,6 +57,7 @@ entry: ; SAFETY-NOT: call {{.*}}__hwasan_store ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_store + ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_atomicrwm %buf.sroa.0 = alloca i8, align 4 call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0) %0 = atomicrmw add ptr %buf.sroa.0, i8 1 monotonic, align 4 @@ -71,6 +75,7 @@ entry: ; SAFETY-NOT: call {{.*}}__hwasan_store ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_store + ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_use %buf.sroa.0 = alloca i8, align 4 call void @use(ptr nonnull %buf.sroa.0) call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0) @@ -89,6 +94,7 @@ entry: ; SAFETY-NOT: call {{.*}}__hwasan_store ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_store + ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_in_range %buf.sroa.0 = alloca [10 x i8], align 4 call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0) store volatile i8 0, ptr %buf.sroa.0, align 4, !tbaa !8 @@ -106,6 +112,7 @@ entry: ; SAFETY-NOT: call {{.*}}__hwasan_store ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_store + ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_in_range2 %buf.sroa.0 = alloca [10 x i8], align 4 %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9 call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0) @@ -123,6 +130,7 @@ entry: ; SAFETY-NOT: call {{.*}}__hwasan_memset ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_memset + ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_in_range3 %buf.sroa.0 = alloca [10 x i8], align 4 %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9 call void @llvm.memset.p0.i32(ptr %ptr, i8 0, i32 1, i1 true) @@ -138,6 +146,7 @@ entry: ; SAFETY-NOT: call {{.*}}__hwasan_memmove ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_memmove + ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_in_range4 %buf.sroa.0 = alloca [10 x i8], align 4 %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9 call void @llvm.memmove.p0.p0.i32(ptr %ptr, ptr %ptr, i32 1, i1 true) @@ -153,6 +162,7 @@ entry: ; SAFETY-NOT: call {{.*}}__hwasan_memmove ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_memmove + ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_in_range5 %buf.sroa.0 = alloca [10 x i8], align 4 %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9 %buf.sroa.1 = alloca [10 x i8], align 4 @@ -171,6 +181,7 @@ entry: ; SAFETY: call {{.*}}__hwasan_store ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_store + ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range %buf.sroa.0 = alloca [10 x i8], align 4 %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 10 call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0) @@ -188,6 +199,7 @@ entry: ; SAFETY: call {{.*}}__hwasan_store ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_store + ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range2 %buf.sroa.0 = alloca [10 x i8], align 4 %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 10 call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0) @@ -205,6 +217,7 @@ entry: ; SAFETY: call {{.*}}__hwasan_memset ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_memset + ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range3 %buf.sroa.0 = alloca [10 x i8], align 4 %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9 call void @llvm.memset.p0.i32(ptr %ptr, i8 0, i32 2, i1 true) @@ -220,6 +233,7 @@ entry: ; SAFETY: call {{.*}}__hwasan_memmove ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_memmove + ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range4 %buf.sroa.0 = alloca [10 x i8], align 4 %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9 call void @llvm.memmove.p0.p0.i32(ptr %ptr, ptr %ptr, i32 2, i1 true) @@ -235,6 +249,7 @@ entry: ; SAFETY: call {{.*}}__hwasan_memmove ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_memmove + ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range5 %buf.sroa.0 = alloca [10 x i8], align 4 %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9 %buf.sroa.1 = alloca [10 x i8], align 4 @@ -256,6 +271,7 @@ entry: ; SAFETY: call {{.*}}__hwasan_store ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_store + ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range6 %buf.sroa.0 = alloca [10 x i8], align 4 %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 10 call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0) @@ -275,6 +291,7 @@ entry: ; SAFETY: call {{.*}}__hwasan_store ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_store + ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_potentially_out_of_range %buf.sroa.0 = alloca [10 x i8], align 4 %off = call i32 @getoffset() %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 %off @@ -293,6 +310,7 @@ entry: ; SAFETY: call {{.*}}__hwasan_memmove ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK: call {{.*}}__hwasan_memmove + ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_potentially_out_of_range2 %buf.sroa.0 = alloca [10 x i8], align 4 %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9 call void @llvm.memmove.p0.p0.i32(ptr %ptr, ptr %a, i32 1, i1 true) @@ -309,6 +327,7 @@ entry: ; SAFETY: call {{.*}}__hwasan_store ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK: call {{.*}}__hwasan_store + ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_unclear %buf.sroa.0 = alloca i8, align 4 %ptr = call ptr @getptr(ptr %buf.sroa.0) call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %ptr) @@ -326,6 +345,7 @@ entry: ; SAFETY: call {{.*}}__hwasan_store ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK: call {{.*}}__hwasan_store + ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_select %x = call ptr @getptr(ptr %a) %buf.sroa.0 = alloca i8, align 4 call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0) @@ -346,6 +366,7 @@ entry: ; SAFETY-NOT: call {{.*}}__hwasan_store ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag ; NOSTACK-NOT: call {{.*}}__hwasan_store + ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_retptr %buf.sroa.0 = alloca i8, align 4 call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0) %ptr = call ptr @retptr(ptr %buf.sroa.0) -- GitLab From 04acf1177ebd75b91f60edf6da67304a339cb886 Mon Sep 17 00:00:00 2001 From: Abhina Sree <69635948+abhina-sree@users.noreply.github.com> Date: Thu, 6 Jun 2024 13:30:56 -0400 Subject: [PATCH 137/462] [gtest] Enable zos for death test support (#94623) This patch implements the following change to enable zos for death test support. https://github.com/google/googletest/pull/4527 --- .../unittest/googletest/include/gtest/internal/gtest-port.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/unittest/googletest/include/gtest/internal/gtest-port.h b/third-party/unittest/googletest/include/gtest/internal/gtest-port.h index a17349e40150..02e1eb0a914f 100644 --- a/third-party/unittest/googletest/include/gtest/internal/gtest-port.h +++ b/third-party/unittest/googletest/include/gtest/internal/gtest-port.h @@ -652,7 +652,7 @@ typedef struct _RTL_CRITICAL_SECTION GTEST_CRITICAL_SECTION; // Determines whether to support death tests. // pops up a dialog window that cannot be suppressed programmatically. #if (defined(GTEST_OS_LINUX) || defined(GTEST_OS_CYGWIN) || \ - defined(GTEST_OS_SOLARIS) || \ + defined(GTEST_OS_SOLARIS) || defined(GTEST_OS_ZOS) || \ (defined(GTEST_OS_MAC) && !defined(GTEST_OS_IOS)) || \ (defined(GTEST_OS_WINDOWS_DESKTOP) && _MSC_VER) || \ defined(GTEST_OS_WINDOWS_MINGW) || defined(GTEST_OS_AIX) || \ -- GitLab From ce938fcbde547315af18a47a609c2e09ebfb4cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Thu, 6 Jun 2024 19:40:43 +0200 Subject: [PATCH 138/462] [clang][Interp] Diagnose functions without body like undefined ones We only get a "reached end of constexpr function" diagnostic otherwise. --- clang/lib/AST/Interp/Interp.cpp | 9 +++++---- clang/test/AST/Interp/cxx23.cpp | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp index 41bbaf83b11c..49015b1dd63d 100644 --- a/clang/lib/AST/Interp/Interp.cpp +++ b/clang/lib/AST/Interp/Interp.cpp @@ -538,7 +538,7 @@ bool CheckCallable(InterpState &S, CodePtr OpPC, const Function *F) { return false; } - if (!F->isConstexpr()) { + if (!F->isConstexpr() || !F->hasBody()) { const SourceLocation &Loc = S.Current->getLocation(OpPC); if (S.getLangOpts().CPlusPlus11) { const FunctionDecl *DiagDecl = F->getDecl(); @@ -572,9 +572,10 @@ bool CheckCallable(InterpState &S, CodePtr OpPC, const Function *F) { S.checkingPotentialConstantExpression()) return false; - // If the declaration is defined _and_ declared 'constexpr', the below - // diagnostic doesn't add anything useful. - if (DiagDecl->isDefined() && DiagDecl->isConstexpr()) + // If the declaration is defined, declared 'constexpr' _and_ has a body, + // the below diagnostic doesn't add anything useful. + if (DiagDecl->isDefined() && DiagDecl->isConstexpr() && + DiagDecl->hasBody()) return false; S.FFDiag(Loc, diag::note_constexpr_invalid_function, 1) diff --git a/clang/test/AST/Interp/cxx23.cpp b/clang/test/AST/Interp/cxx23.cpp index c91d52c552b1..1efd784abbbe 100644 --- a/clang/test/AST/Interp/cxx23.cpp +++ b/clang/test/AST/Interp/cxx23.cpp @@ -178,3 +178,25 @@ namespace ExplicitLambdaThis { }; static_assert(f()); } + +namespace std { + struct strong_ordering { + int n; + constexpr operator int() const { return n; } + static const strong_ordering less, equal, greater; + }; + constexpr strong_ordering strong_ordering::less = {-1}; + constexpr strong_ordering strong_ordering::equal = {0}; + constexpr strong_ordering strong_ordering::greater = {1}; +} + +namespace UndefinedThreeWay { + struct A { + friend constexpr std::strong_ordering operator<=>(const A&, const A&) = default; // all-note {{declared here}} + }; + + constexpr std::strong_ordering operator<=>(const A&, const A&) noexcept; + constexpr std::strong_ordering (*test_a_threeway)(const A&, const A&) = &operator<=>; + static_assert(!(*test_a_threeway)(A(), A())); // all-error {{static assertion expression is not an integral constant expression}} \ + // all-note {{undefined function 'operator<=>' cannot be used in a constant expression}} +} -- GitLab From 7cfad77a5e3e9145213b024b9151ff812b54de51 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 13 Sep 2023 13:46:06 -0500 Subject: [PATCH 139/462] [InstCombine] Add tests for folding multiuse `(icmp eq/ne (or X, Y), Y)`; NFC --- .../Transforms/InstCombine/icmp-of-or-x.ll | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/llvm/test/Transforms/InstCombine/icmp-of-or-x.ll b/llvm/test/Transforms/InstCombine/icmp-of-or-x.ll index 26f53cb4807e..9abcc1a7ed81 100644 --- a/llvm/test/Transforms/InstCombine/icmp-of-or-x.ll +++ b/llvm/test/Transforms/InstCombine/icmp-of-or-x.ll @@ -373,3 +373,30 @@ define i1 @pr64610(ptr %b) { %r = icmp ugt i32 %or, %s ret i1 %r } + +define i1 @icmp_eq_x_invertable_y2_todo(i8 %x, i1 %y, i8 %z) { +; CHECK-LABEL: @icmp_eq_x_invertable_y2_todo( +; CHECK-NEXT: [[ZZ:%.*]] = xor i8 [[Z:%.*]], -1 +; CHECK-NEXT: [[YY:%.*]] = select i1 [[Y:%.*]], i8 7, i8 [[ZZ]] +; CHECK-NEXT: [[OR:%.*]] = or i8 [[YY]], [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = icmp eq i8 [[YY]], [[OR]] +; CHECK-NEXT: ret i1 [[R]] +; + %zz = xor i8 %z, -1 + %yy = select i1 %y, i8 7, i8 %zz + %or = or i8 %x, %yy + %r = icmp eq i8 %yy, %or + ret i1 %r +} + +define i1 @icmp_eq_x_invertable_y2(i8 %x, i8 %y) { +; CHECK-LABEL: @icmp_eq_x_invertable_y2( +; CHECK-NEXT: [[TMP1:%.*]] = and i8 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[R:%.*]] = icmp eq i8 [[TMP1]], 0 +; CHECK-NEXT: ret i1 [[R]] +; + %yy = xor i8 %y, -1 + %or = or i8 %x, %yy + %r = icmp eq i8 %yy, %or + ret i1 %r +} -- GitLab From 3716a3c1bce4af81db6a95eee30e35b5b923e319 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 13 Sep 2023 13:46:08 -0500 Subject: [PATCH 140/462] [InstCombine] Folding multiuse `(icmp eq/ne (or X, Y), Y)` for 2 uses of `Y` The fold will replace 2 uses of `Y` we should also do fold if `Y` has 2 uses (not only oneuse). Reviewed By: nikic Differential Revision: https://reviews.llvm.org/D159062 --- llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp | 2 +- llvm/test/Transforms/InstCombine/icmp-of-or-x.ll | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp index 803475bfaba5..38c1c2644554 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -4795,7 +4795,7 @@ static Instruction *foldICmpOrXX(ICmpInst &I, const SimplifyQuery &Q, if (ICmpInst::isEquality(Pred) && Op0->hasOneUse()) { // icmp (X | Y) eq/ne Y --> (X & ~Y) eq/ne 0 if Y is freely invertible if (Value *NotOp1 = - IC.getFreelyInverted(Op1, Op1->hasOneUse(), &IC.Builder)) + IC.getFreelyInverted(Op1, !Op1->hasNUsesOrMore(3), &IC.Builder)) return new ICmpInst(Pred, IC.Builder.CreateAnd(A, NotOp1), Constant::getNullValue(Op1->getType())); // icmp (X | Y) eq/ne Y --> (~X | Y) eq/ne -1 if X is freely invertible. diff --git a/llvm/test/Transforms/InstCombine/icmp-of-or-x.ll b/llvm/test/Transforms/InstCombine/icmp-of-or-x.ll index 9abcc1a7ed81..304874645d5d 100644 --- a/llvm/test/Transforms/InstCombine/icmp-of-or-x.ll +++ b/llvm/test/Transforms/InstCombine/icmp-of-or-x.ll @@ -376,10 +376,9 @@ define i1 @pr64610(ptr %b) { define i1 @icmp_eq_x_invertable_y2_todo(i8 %x, i1 %y, i8 %z) { ; CHECK-LABEL: @icmp_eq_x_invertable_y2_todo( -; CHECK-NEXT: [[ZZ:%.*]] = xor i8 [[Z:%.*]], -1 -; CHECK-NEXT: [[YY:%.*]] = select i1 [[Y:%.*]], i8 7, i8 [[ZZ]] -; CHECK-NEXT: [[OR:%.*]] = or i8 [[YY]], [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = icmp eq i8 [[YY]], [[OR]] +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[Y:%.*]], i8 -8, i8 [[Z:%.*]] +; CHECK-NEXT: [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = icmp eq i8 [[TMP2]], 0 ; CHECK-NEXT: ret i1 [[R]] ; %zz = xor i8 %z, -1 -- GitLab From 3a95c688cc427b2971cb69abb9bd322b5a449fb4 Mon Sep 17 00:00:00 2001 From: Brandon Wu Date: Fri, 7 Jun 2024 01:59:49 +0800 Subject: [PATCH 141/462] [clang][RISCV] Update vcpop.v C interface to follow the nameing convention (#94318) We named the intrinsics by replacing "." by "_" in the instruction conventionally, so the `vcpopv_v` where the corresponding instruction is `vcpop.v` should be named `vcpop_v`. --- clang/include/clang/Basic/riscv_vector.td | 3 +- .../non-policy/non-overloaded/vcpopv.c | 264 ++++----- .../non-policy/overloaded/vcpopv.c | 264 ++++----- .../policy/non-overloaded/vcpopv.c | 528 +++++++++--------- .../policy/overloaded/vcpopv.c | 528 +++++++++--------- 5 files changed, 794 insertions(+), 793 deletions(-) diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td index cca4367751b9..a0820e2093bc 100644 --- a/clang/include/clang/Basic/riscv_vector.td +++ b/clang/include/clang/Basic/riscv_vector.td @@ -2637,7 +2637,8 @@ let UnMaskedPolicyScheme = HasPassthruOperand in { defm vbrev : RVVOutBuiltinSetZvbb; defm vclz : RVVOutBuiltinSetZvbb; defm vctz : RVVOutBuiltinSetZvbb; - defm vcpopv : RVVOutBuiltinSetZvbb; + let IRName = "vcpopv", MaskedIRName = "vcpopv_mask" in + defm vcpop : RVVOutBuiltinSetZvbb; let OverloadedName = "vwsll" in defm vwsll : RVVSignedWidenBinBuiltinSetVwsll; } diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vcpopv.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vcpopv.c index 13748be1acc1..b87b225b632a 100644 --- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vcpopv.c +++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vcpopv.c @@ -16,399 +16,399 @@ #include -// CHECK-LABEL: @test_vcpopv_v_u8mf8( +// CHECK-LABEL: @test_vcpop_v_u8mf8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf8_t test_vcpopv_v_u8mf8(vuint8mf8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf8(vs2, vl); +vuint8mf8_t test_vcpop_v_u8mf8(vuint8mf8_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf8(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf4( +// CHECK-LABEL: @test_vcpop_v_u8mf4( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf4_t test_vcpopv_v_u8mf4(vuint8mf4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf4(vs2, vl); +vuint8mf4_t test_vcpop_v_u8mf4(vuint8mf4_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf4(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf2( +// CHECK-LABEL: @test_vcpop_v_u8mf2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf2_t test_vcpopv_v_u8mf2(vuint8mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf2(vs2, vl); +vuint8mf2_t test_vcpop_v_u8mf2(vuint8mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf2(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m1( +// CHECK-LABEL: @test_vcpop_v_u8m1( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m1_t test_vcpopv_v_u8m1(vuint8m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m1(vs2, vl); +vuint8m1_t test_vcpop_v_u8m1(vuint8m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m1(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m2( +// CHECK-LABEL: @test_vcpop_v_u8m2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv16i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m2_t test_vcpopv_v_u8m2(vuint8m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m2(vs2, vl); +vuint8m2_t test_vcpop_v_u8m2(vuint8m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m2(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m4( +// CHECK-LABEL: @test_vcpop_v_u8m4( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv32i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m4_t test_vcpopv_v_u8m4(vuint8m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m4(vs2, vl); +vuint8m4_t test_vcpop_v_u8m4(vuint8m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m4(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m8( +// CHECK-LABEL: @test_vcpop_v_u8m8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv64i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m8_t test_vcpopv_v_u8m8(vuint8m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m8(vs2, vl); +vuint8m8_t test_vcpop_v_u8m8(vuint8m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m8(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf4( +// CHECK-LABEL: @test_vcpop_v_u16mf4( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i16.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf4_t test_vcpopv_v_u16mf4(vuint16mf4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16mf4(vs2, vl); +vuint16mf4_t test_vcpop_v_u16mf4(vuint16mf4_t vs2, size_t vl) { + return __riscv_vcpop_v_u16mf4(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf2( +// CHECK-LABEL: @test_vcpop_v_u16mf2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i16.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf2_t test_vcpopv_v_u16mf2(vuint16mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16mf2(vs2, vl); +vuint16mf2_t test_vcpop_v_u16mf2(vuint16mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u16mf2(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m1( +// CHECK-LABEL: @test_vcpop_v_u16m1( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i16.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m1_t test_vcpopv_v_u16m1(vuint16m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m1(vs2, vl); +vuint16m1_t test_vcpop_v_u16m1(vuint16m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m1(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m2( +// CHECK-LABEL: @test_vcpop_v_u16m2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i16.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m2_t test_vcpopv_v_u16m2(vuint16m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m2(vs2, vl); +vuint16m2_t test_vcpop_v_u16m2(vuint16m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m2(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m4( +// CHECK-LABEL: @test_vcpop_v_u16m4( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv16i16.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m4_t test_vcpopv_v_u16m4(vuint16m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m4(vs2, vl); +vuint16m4_t test_vcpop_v_u16m4(vuint16m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m4(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m8( +// CHECK-LABEL: @test_vcpop_v_u16m8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv32i16.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m8_t test_vcpopv_v_u16m8(vuint16m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m8(vs2, vl); +vuint16m8_t test_vcpop_v_u16m8(vuint16m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m8(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32mf2( +// CHECK-LABEL: @test_vcpop_v_u32mf2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i32.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32mf2_t test_vcpopv_v_u32mf2(vuint32mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32mf2(vs2, vl); +vuint32mf2_t test_vcpop_v_u32mf2(vuint32mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u32mf2(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m1( +// CHECK-LABEL: @test_vcpop_v_u32m1( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i32.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m1_t test_vcpopv_v_u32m1(vuint32m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m1(vs2, vl); +vuint32m1_t test_vcpop_v_u32m1(vuint32m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m1(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m2( +// CHECK-LABEL: @test_vcpop_v_u32m2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i32.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m2_t test_vcpopv_v_u32m2(vuint32m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m2(vs2, vl); +vuint32m2_t test_vcpop_v_u32m2(vuint32m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m2(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m4( +// CHECK-LABEL: @test_vcpop_v_u32m4( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i32.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m4_t test_vcpopv_v_u32m4(vuint32m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m4(vs2, vl); +vuint32m4_t test_vcpop_v_u32m4(vuint32m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m4(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m8( +// CHECK-LABEL: @test_vcpop_v_u32m8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv16i32.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m8_t test_vcpopv_v_u32m8(vuint32m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m8(vs2, vl); +vuint32m8_t test_vcpop_v_u32m8(vuint32m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m8(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m1( +// CHECK-LABEL: @test_vcpop_v_u64m1( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i64.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m1_t test_vcpopv_v_u64m1(vuint64m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m1(vs2, vl); +vuint64m1_t test_vcpop_v_u64m1(vuint64m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m1(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m2( +// CHECK-LABEL: @test_vcpop_v_u64m2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i64.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m2_t test_vcpopv_v_u64m2(vuint64m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m2(vs2, vl); +vuint64m2_t test_vcpop_v_u64m2(vuint64m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m2(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m4( +// CHECK-LABEL: @test_vcpop_v_u64m4( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i64.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m4_t test_vcpopv_v_u64m4(vuint64m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m4(vs2, vl); +vuint64m4_t test_vcpop_v_u64m4(vuint64m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m4(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m8( +// CHECK-LABEL: @test_vcpop_v_u64m8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i64.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m8_t test_vcpopv_v_u64m8(vuint64m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m8(vs2, vl); +vuint64m8_t test_vcpop_v_u64m8(vuint64m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m8(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf8_m( +// CHECK-LABEL: @test_vcpop_v_u8mf8_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf8_t test_vcpopv_v_u8mf8_m(vbool64_t mask, vuint8mf8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf8_m(mask, vs2, vl); +vuint8mf8_t test_vcpop_v_u8mf8_m(vbool64_t mask, vuint8mf8_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf8_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf4_m( +// CHECK-LABEL: @test_vcpop_v_u8mf4_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf4_t test_vcpopv_v_u8mf4_m(vbool32_t mask, vuint8mf4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf4_m(mask, vs2, vl); +vuint8mf4_t test_vcpop_v_u8mf4_m(vbool32_t mask, vuint8mf4_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf4_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf2_m( +// CHECK-LABEL: @test_vcpop_v_u8mf2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf2_t test_vcpopv_v_u8mf2_m(vbool16_t mask, vuint8mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf2_m(mask, vs2, vl); +vuint8mf2_t test_vcpop_v_u8mf2_m(vbool16_t mask, vuint8mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf2_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m1_m( +// CHECK-LABEL: @test_vcpop_v_u8m1_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8m1_t test_vcpopv_v_u8m1_m(vbool8_t mask, vuint8m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m1_m(mask, vs2, vl); +vuint8m1_t test_vcpop_v_u8m1_m(vbool8_t mask, vuint8m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m1_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m2_m( +// CHECK-LABEL: @test_vcpop_v_u8m2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8m2_t test_vcpopv_v_u8m2_m(vbool4_t mask, vuint8m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m2_m(mask, vs2, vl); +vuint8m2_t test_vcpop_v_u8m2_m(vbool4_t mask, vuint8m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m2_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m4_m( +// CHECK-LABEL: @test_vcpop_v_u8m4_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8m4_t test_vcpopv_v_u8m4_m(vbool2_t mask, vuint8m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m4_m(mask, vs2, vl); +vuint8m4_t test_vcpop_v_u8m4_m(vbool2_t mask, vuint8m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m4_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m8_m( +// CHECK-LABEL: @test_vcpop_v_u8m8_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv64i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8m8_t test_vcpopv_v_u8m8_m(vbool1_t mask, vuint8m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m8_m(mask, vs2, vl); +vuint8m8_t test_vcpop_v_u8m8_m(vbool1_t mask, vuint8m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m8_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf4_m( +// CHECK-LABEL: @test_vcpop_v_u16mf4_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i16.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf4_t test_vcpopv_v_u16mf4_m(vbool64_t mask, vuint16mf4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16mf4_m(mask, vs2, vl); +vuint16mf4_t test_vcpop_v_u16mf4_m(vbool64_t mask, vuint16mf4_t vs2, size_t vl) { + return __riscv_vcpop_v_u16mf4_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf2_m( +// CHECK-LABEL: @test_vcpop_v_u16mf2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i16.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf2_t test_vcpopv_v_u16mf2_m(vbool32_t mask, vuint16mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16mf2_m(mask, vs2, vl); +vuint16mf2_t test_vcpop_v_u16mf2_m(vbool32_t mask, vuint16mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u16mf2_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m1_m( +// CHECK-LABEL: @test_vcpop_v_u16m1_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i16.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint16m1_t test_vcpopv_v_u16m1_m(vbool16_t mask, vuint16m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m1_m(mask, vs2, vl); +vuint16m1_t test_vcpop_v_u16m1_m(vbool16_t mask, vuint16m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m1_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m2_m( +// CHECK-LABEL: @test_vcpop_v_u16m2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i16.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint16m2_t test_vcpopv_v_u16m2_m(vbool8_t mask, vuint16m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m2_m(mask, vs2, vl); +vuint16m2_t test_vcpop_v_u16m2_m(vbool8_t mask, vuint16m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m2_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m4_m( +// CHECK-LABEL: @test_vcpop_v_u16m4_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i16.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint16m4_t test_vcpopv_v_u16m4_m(vbool4_t mask, vuint16m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m4_m(mask, vs2, vl); +vuint16m4_t test_vcpop_v_u16m4_m(vbool4_t mask, vuint16m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m4_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m8_m( +// CHECK-LABEL: @test_vcpop_v_u16m8_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i16.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint16m8_t test_vcpopv_v_u16m8_m(vbool2_t mask, vuint16m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m8_m(mask, vs2, vl); +vuint16m8_t test_vcpop_v_u16m8_m(vbool2_t mask, vuint16m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m8_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32mf2_m( +// CHECK-LABEL: @test_vcpop_v_u32mf2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i32.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint32mf2_t test_vcpopv_v_u32mf2_m(vbool64_t mask, vuint32mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32mf2_m(mask, vs2, vl); +vuint32mf2_t test_vcpop_v_u32mf2_m(vbool64_t mask, vuint32mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u32mf2_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m1_m( +// CHECK-LABEL: @test_vcpop_v_u32m1_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i32.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint32m1_t test_vcpopv_v_u32m1_m(vbool32_t mask, vuint32m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m1_m(mask, vs2, vl); +vuint32m1_t test_vcpop_v_u32m1_m(vbool32_t mask, vuint32m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m1_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m2_m( +// CHECK-LABEL: @test_vcpop_v_u32m2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i32.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint32m2_t test_vcpopv_v_u32m2_m(vbool16_t mask, vuint32m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m2_m(mask, vs2, vl); +vuint32m2_t test_vcpop_v_u32m2_m(vbool16_t mask, vuint32m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m2_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m4_m( +// CHECK-LABEL: @test_vcpop_v_u32m4_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i32.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint32m4_t test_vcpopv_v_u32m4_m(vbool8_t mask, vuint32m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m4_m(mask, vs2, vl); +vuint32m4_t test_vcpop_v_u32m4_m(vbool8_t mask, vuint32m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m4_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m8_m( +// CHECK-LABEL: @test_vcpop_v_u32m8_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i32.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint32m8_t test_vcpopv_v_u32m8_m(vbool4_t mask, vuint32m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m8_m(mask, vs2, vl); +vuint32m8_t test_vcpop_v_u32m8_m(vbool4_t mask, vuint32m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m8_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m1_m( +// CHECK-LABEL: @test_vcpop_v_u64m1_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i64.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint64m1_t test_vcpopv_v_u64m1_m(vbool64_t mask, vuint64m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m1_m(mask, vs2, vl); +vuint64m1_t test_vcpop_v_u64m1_m(vbool64_t mask, vuint64m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m1_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m2_m( +// CHECK-LABEL: @test_vcpop_v_u64m2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i64.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint64m2_t test_vcpopv_v_u64m2_m(vbool32_t mask, vuint64m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m2_m(mask, vs2, vl); +vuint64m2_t test_vcpop_v_u64m2_m(vbool32_t mask, vuint64m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m2_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m4_m( +// CHECK-LABEL: @test_vcpop_v_u64m4_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i64.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint64m4_t test_vcpopv_v_u64m4_m(vbool16_t mask, vuint64m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m4_m(mask, vs2, vl); +vuint64m4_t test_vcpop_v_u64m4_m(vbool16_t mask, vuint64m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m4_m(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m8_m( +// CHECK-LABEL: @test_vcpop_v_u64m8_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i64.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint64m8_t test_vcpopv_v_u64m8_m(vbool8_t mask, vuint64m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m8_m(mask, vs2, vl); +vuint64m8_t test_vcpop_v_u64m8_m(vbool8_t mask, vuint64m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m8_m(mask, vs2, vl); } diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vcpopv.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vcpopv.c index adb0ac9ee5d7..5625b19f57f3 100644 --- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vcpopv.c +++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vcpopv.c @@ -16,399 +16,399 @@ #include -// CHECK-LABEL: @test_vcpopv_v_u8mf8( +// CHECK-LABEL: @test_vcpop_v_u8mf8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf8_t test_vcpopv_v_u8mf8(vuint8mf8_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint8mf8_t test_vcpop_v_u8mf8(vuint8mf8_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf4( +// CHECK-LABEL: @test_vcpop_v_u8mf4( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf4_t test_vcpopv_v_u8mf4(vuint8mf4_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint8mf4_t test_vcpop_v_u8mf4(vuint8mf4_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf2( +// CHECK-LABEL: @test_vcpop_v_u8mf2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf2_t test_vcpopv_v_u8mf2(vuint8mf2_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint8mf2_t test_vcpop_v_u8mf2(vuint8mf2_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m1( +// CHECK-LABEL: @test_vcpop_v_u8m1( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m1_t test_vcpopv_v_u8m1(vuint8m1_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint8m1_t test_vcpop_v_u8m1(vuint8m1_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m2( +// CHECK-LABEL: @test_vcpop_v_u8m2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv16i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m2_t test_vcpopv_v_u8m2(vuint8m2_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint8m2_t test_vcpop_v_u8m2(vuint8m2_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m4( +// CHECK-LABEL: @test_vcpop_v_u8m4( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv32i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m4_t test_vcpopv_v_u8m4(vuint8m4_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint8m4_t test_vcpop_v_u8m4(vuint8m4_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m8( +// CHECK-LABEL: @test_vcpop_v_u8m8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv64i8.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m8_t test_vcpopv_v_u8m8(vuint8m8_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint8m8_t test_vcpop_v_u8m8(vuint8m8_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf4( +// CHECK-LABEL: @test_vcpop_v_u16mf4( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i16.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf4_t test_vcpopv_v_u16mf4(vuint16mf4_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint16mf4_t test_vcpop_v_u16mf4(vuint16mf4_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf2( +// CHECK-LABEL: @test_vcpop_v_u16mf2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i16.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf2_t test_vcpopv_v_u16mf2(vuint16mf2_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint16mf2_t test_vcpop_v_u16mf2(vuint16mf2_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m1( +// CHECK-LABEL: @test_vcpop_v_u16m1( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i16.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m1_t test_vcpopv_v_u16m1(vuint16m1_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint16m1_t test_vcpop_v_u16m1(vuint16m1_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m2( +// CHECK-LABEL: @test_vcpop_v_u16m2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i16.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m2_t test_vcpopv_v_u16m2(vuint16m2_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint16m2_t test_vcpop_v_u16m2(vuint16m2_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m4( +// CHECK-LABEL: @test_vcpop_v_u16m4( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv16i16.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m4_t test_vcpopv_v_u16m4(vuint16m4_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint16m4_t test_vcpop_v_u16m4(vuint16m4_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m8( +// CHECK-LABEL: @test_vcpop_v_u16m8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv32i16.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m8_t test_vcpopv_v_u16m8(vuint16m8_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint16m8_t test_vcpop_v_u16m8(vuint16m8_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32mf2( +// CHECK-LABEL: @test_vcpop_v_u32mf2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i32.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32mf2_t test_vcpopv_v_u32mf2(vuint32mf2_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint32mf2_t test_vcpop_v_u32mf2(vuint32mf2_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m1( +// CHECK-LABEL: @test_vcpop_v_u32m1( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i32.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m1_t test_vcpopv_v_u32m1(vuint32m1_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint32m1_t test_vcpop_v_u32m1(vuint32m1_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m2( +// CHECK-LABEL: @test_vcpop_v_u32m2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i32.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m2_t test_vcpopv_v_u32m2(vuint32m2_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint32m2_t test_vcpop_v_u32m2(vuint32m2_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m4( +// CHECK-LABEL: @test_vcpop_v_u32m4( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i32.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m4_t test_vcpopv_v_u32m4(vuint32m4_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint32m4_t test_vcpop_v_u32m4(vuint32m4_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m8( +// CHECK-LABEL: @test_vcpop_v_u32m8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv16i32.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m8_t test_vcpopv_v_u32m8(vuint32m8_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint32m8_t test_vcpop_v_u32m8(vuint32m8_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m1( +// CHECK-LABEL: @test_vcpop_v_u64m1( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i64.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m1_t test_vcpopv_v_u64m1(vuint64m1_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint64m1_t test_vcpop_v_u64m1(vuint64m1_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m2( +// CHECK-LABEL: @test_vcpop_v_u64m2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i64.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m2_t test_vcpopv_v_u64m2(vuint64m2_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint64m2_t test_vcpop_v_u64m2(vuint64m2_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m4( +// CHECK-LABEL: @test_vcpop_v_u64m4( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i64.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m4_t test_vcpopv_v_u64m4(vuint64m4_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint64m4_t test_vcpop_v_u64m4(vuint64m4_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m8( +// CHECK-LABEL: @test_vcpop_v_u64m8( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i64.i64( poison, [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m8_t test_vcpopv_v_u64m8(vuint64m8_t vs2, size_t vl) { - return __riscv_vcpopv(vs2, vl); +vuint64m8_t test_vcpop_v_u64m8(vuint64m8_t vs2, size_t vl) { + return __riscv_vcpop(vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf8_m( +// CHECK-LABEL: @test_vcpop_v_u8mf8_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf8_t test_vcpopv_v_u8mf8_m(vbool64_t mask, vuint8mf8_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint8mf8_t test_vcpop_v_u8mf8_m(vbool64_t mask, vuint8mf8_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf4_m( +// CHECK-LABEL: @test_vcpop_v_u8mf4_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf4_t test_vcpopv_v_u8mf4_m(vbool32_t mask, vuint8mf4_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint8mf4_t test_vcpop_v_u8mf4_m(vbool32_t mask, vuint8mf4_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf2_m( +// CHECK-LABEL: @test_vcpop_v_u8mf2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf2_t test_vcpopv_v_u8mf2_m(vbool16_t mask, vuint8mf2_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint8mf2_t test_vcpop_v_u8mf2_m(vbool16_t mask, vuint8mf2_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m1_m( +// CHECK-LABEL: @test_vcpop_v_u8m1_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8m1_t test_vcpopv_v_u8m1_m(vbool8_t mask, vuint8m1_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint8m1_t test_vcpop_v_u8m1_m(vbool8_t mask, vuint8m1_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m2_m( +// CHECK-LABEL: @test_vcpop_v_u8m2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8m2_t test_vcpopv_v_u8m2_m(vbool4_t mask, vuint8m2_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint8m2_t test_vcpop_v_u8m2_m(vbool4_t mask, vuint8m2_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m4_m( +// CHECK-LABEL: @test_vcpop_v_u8m4_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8m4_t test_vcpopv_v_u8m4_m(vbool2_t mask, vuint8m4_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint8m4_t test_vcpop_v_u8m4_m(vbool2_t mask, vuint8m4_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m8_m( +// CHECK-LABEL: @test_vcpop_v_u8m8_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv64i8.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint8m8_t test_vcpopv_v_u8m8_m(vbool1_t mask, vuint8m8_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint8m8_t test_vcpop_v_u8m8_m(vbool1_t mask, vuint8m8_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf4_m( +// CHECK-LABEL: @test_vcpop_v_u16mf4_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i16.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf4_t test_vcpopv_v_u16mf4_m(vbool64_t mask, vuint16mf4_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint16mf4_t test_vcpop_v_u16mf4_m(vbool64_t mask, vuint16mf4_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf2_m( +// CHECK-LABEL: @test_vcpop_v_u16mf2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i16.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf2_t test_vcpopv_v_u16mf2_m(vbool32_t mask, vuint16mf2_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint16mf2_t test_vcpop_v_u16mf2_m(vbool32_t mask, vuint16mf2_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m1_m( +// CHECK-LABEL: @test_vcpop_v_u16m1_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i16.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint16m1_t test_vcpopv_v_u16m1_m(vbool16_t mask, vuint16m1_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint16m1_t test_vcpop_v_u16m1_m(vbool16_t mask, vuint16m1_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m2_m( +// CHECK-LABEL: @test_vcpop_v_u16m2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i16.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint16m2_t test_vcpopv_v_u16m2_m(vbool8_t mask, vuint16m2_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint16m2_t test_vcpop_v_u16m2_m(vbool8_t mask, vuint16m2_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m4_m( +// CHECK-LABEL: @test_vcpop_v_u16m4_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i16.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint16m4_t test_vcpopv_v_u16m4_m(vbool4_t mask, vuint16m4_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint16m4_t test_vcpop_v_u16m4_m(vbool4_t mask, vuint16m4_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m8_m( +// CHECK-LABEL: @test_vcpop_v_u16m8_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i16.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint16m8_t test_vcpopv_v_u16m8_m(vbool2_t mask, vuint16m8_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint16m8_t test_vcpop_v_u16m8_m(vbool2_t mask, vuint16m8_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32mf2_m( +// CHECK-LABEL: @test_vcpop_v_u32mf2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i32.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint32mf2_t test_vcpopv_v_u32mf2_m(vbool64_t mask, vuint32mf2_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint32mf2_t test_vcpop_v_u32mf2_m(vbool64_t mask, vuint32mf2_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m1_m( +// CHECK-LABEL: @test_vcpop_v_u32m1_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i32.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint32m1_t test_vcpopv_v_u32m1_m(vbool32_t mask, vuint32m1_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint32m1_t test_vcpop_v_u32m1_m(vbool32_t mask, vuint32m1_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m2_m( +// CHECK-LABEL: @test_vcpop_v_u32m2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i32.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint32m2_t test_vcpopv_v_u32m2_m(vbool16_t mask, vuint32m2_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint32m2_t test_vcpop_v_u32m2_m(vbool16_t mask, vuint32m2_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m4_m( +// CHECK-LABEL: @test_vcpop_v_u32m4_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i32.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint32m4_t test_vcpopv_v_u32m4_m(vbool8_t mask, vuint32m4_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint32m4_t test_vcpop_v_u32m4_m(vbool8_t mask, vuint32m4_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m8_m( +// CHECK-LABEL: @test_vcpop_v_u32m8_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i32.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint32m8_t test_vcpopv_v_u32m8_m(vbool4_t mask, vuint32m8_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint32m8_t test_vcpop_v_u32m8_m(vbool4_t mask, vuint32m8_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m1_m( +// CHECK-LABEL: @test_vcpop_v_u64m1_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i64.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint64m1_t test_vcpopv_v_u64m1_m(vbool64_t mask, vuint64m1_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint64m1_t test_vcpop_v_u64m1_m(vbool64_t mask, vuint64m1_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m2_m( +// CHECK-LABEL: @test_vcpop_v_u64m2_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i64.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint64m2_t test_vcpopv_v_u64m2_m(vbool32_t mask, vuint64m2_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint64m2_t test_vcpop_v_u64m2_m(vbool32_t mask, vuint64m2_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m4_m( +// CHECK-LABEL: @test_vcpop_v_u64m4_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i64.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint64m4_t test_vcpopv_v_u64m4_m(vbool16_t mask, vuint64m4_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint64m4_t test_vcpop_v_u64m4_m(vbool16_t mask, vuint64m4_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m8_m( +// CHECK-LABEL: @test_vcpop_v_u64m8_m( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i64.i64( poison, [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 3) // CHECK-NEXT: ret [[TMP0]] // -vuint64m8_t test_vcpopv_v_u64m8_m(vbool8_t mask, vuint64m8_t vs2, size_t vl) { - return __riscv_vcpopv(mask, vs2, vl); +vuint64m8_t test_vcpop_v_u64m8_m(vbool8_t mask, vuint64m8_t vs2, size_t vl) { + return __riscv_vcpop(mask, vs2, vl); } diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vcpopv.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vcpopv.c index 8a1f2e1beec1..3a110339b5f1 100644 --- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vcpopv.c +++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vcpopv.c @@ -16,795 +16,795 @@ #include -// CHECK-LABEL: @test_vcpopv_v_u8mf8_tu( +// CHECK-LABEL: @test_vcpop_v_u8mf8_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf8_t test_vcpopv_v_u8mf8_tu(vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf8_tu(maskedoff, vs2, vl); +vuint8mf8_t test_vcpop_v_u8mf8_tu(vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf8_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf4_tu( +// CHECK-LABEL: @test_vcpop_v_u8mf4_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf4_t test_vcpopv_v_u8mf4_tu(vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf4_tu(maskedoff, vs2, vl); +vuint8mf4_t test_vcpop_v_u8mf4_tu(vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf4_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf2_tu( +// CHECK-LABEL: @test_vcpop_v_u8mf2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf2_t test_vcpopv_v_u8mf2_tu(vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf2_tu(maskedoff, vs2, vl); +vuint8mf2_t test_vcpop_v_u8mf2_tu(vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf2_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m1_tu( +// CHECK-LABEL: @test_vcpop_v_u8m1_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m1_t test_vcpopv_v_u8m1_tu(vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m1_tu(maskedoff, vs2, vl); +vuint8m1_t test_vcpop_v_u8m1_tu(vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m1_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m2_tu( +// CHECK-LABEL: @test_vcpop_v_u8m2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv16i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m2_t test_vcpopv_v_u8m2_tu(vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m2_tu(maskedoff, vs2, vl); +vuint8m2_t test_vcpop_v_u8m2_tu(vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m2_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m4_tu( +// CHECK-LABEL: @test_vcpop_v_u8m4_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv32i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m4_t test_vcpopv_v_u8m4_tu(vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m4_tu(maskedoff, vs2, vl); +vuint8m4_t test_vcpop_v_u8m4_tu(vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m4_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m8_tu( +// CHECK-LABEL: @test_vcpop_v_u8m8_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv64i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m8_t test_vcpopv_v_u8m8_tu(vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m8_tu(maskedoff, vs2, vl); +vuint8m8_t test_vcpop_v_u8m8_tu(vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m8_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf4_tu( +// CHECK-LABEL: @test_vcpop_v_u16mf4_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf4_t test_vcpopv_v_u16mf4_tu(vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16mf4_tu(maskedoff, vs2, vl); +vuint16mf4_t test_vcpop_v_u16mf4_tu(vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { + return __riscv_vcpop_v_u16mf4_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf2_tu( +// CHECK-LABEL: @test_vcpop_v_u16mf2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf2_t test_vcpopv_v_u16mf2_tu(vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16mf2_tu(maskedoff, vs2, vl); +vuint16mf2_t test_vcpop_v_u16mf2_tu(vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u16mf2_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m1_tu( +// CHECK-LABEL: @test_vcpop_v_u16m1_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m1_t test_vcpopv_v_u16m1_tu(vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m1_tu(maskedoff, vs2, vl); +vuint16m1_t test_vcpop_v_u16m1_tu(vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m1_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m2_tu( +// CHECK-LABEL: @test_vcpop_v_u16m2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m2_t test_vcpopv_v_u16m2_tu(vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m2_tu(maskedoff, vs2, vl); +vuint16m2_t test_vcpop_v_u16m2_tu(vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m2_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m4_tu( +// CHECK-LABEL: @test_vcpop_v_u16m4_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv16i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m4_t test_vcpopv_v_u16m4_tu(vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m4_tu(maskedoff, vs2, vl); +vuint16m4_t test_vcpop_v_u16m4_tu(vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m4_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m8_tu( +// CHECK-LABEL: @test_vcpop_v_u16m8_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv32i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m8_t test_vcpopv_v_u16m8_tu(vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m8_tu(maskedoff, vs2, vl); +vuint16m8_t test_vcpop_v_u16m8_tu(vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m8_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32mf2_tu( +// CHECK-LABEL: @test_vcpop_v_u32mf2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32mf2_t test_vcpopv_v_u32mf2_tu(vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32mf2_tu(maskedoff, vs2, vl); +vuint32mf2_t test_vcpop_v_u32mf2_tu(vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u32mf2_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m1_tu( +// CHECK-LABEL: @test_vcpop_v_u32m1_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m1_t test_vcpopv_v_u32m1_tu(vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m1_tu(maskedoff, vs2, vl); +vuint32m1_t test_vcpop_v_u32m1_tu(vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m1_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m2_tu( +// CHECK-LABEL: @test_vcpop_v_u32m2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m2_t test_vcpopv_v_u32m2_tu(vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m2_tu(maskedoff, vs2, vl); +vuint32m2_t test_vcpop_v_u32m2_tu(vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m2_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m4_tu( +// CHECK-LABEL: @test_vcpop_v_u32m4_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m4_t test_vcpopv_v_u32m4_tu(vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m4_tu(maskedoff, vs2, vl); +vuint32m4_t test_vcpop_v_u32m4_tu(vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m4_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m8_tu( +// CHECK-LABEL: @test_vcpop_v_u32m8_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv16i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m8_t test_vcpopv_v_u32m8_tu(vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m8_tu(maskedoff, vs2, vl); +vuint32m8_t test_vcpop_v_u32m8_tu(vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m8_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m1_tu( +// CHECK-LABEL: @test_vcpop_v_u64m1_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m1_t test_vcpopv_v_u64m1_tu(vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m1_tu(maskedoff, vs2, vl); +vuint64m1_t test_vcpop_v_u64m1_tu(vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m1_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m2_tu( +// CHECK-LABEL: @test_vcpop_v_u64m2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m2_t test_vcpopv_v_u64m2_tu(vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m2_tu(maskedoff, vs2, vl); +vuint64m2_t test_vcpop_v_u64m2_tu(vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m2_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m4_tu( +// CHECK-LABEL: @test_vcpop_v_u64m4_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m4_t test_vcpopv_v_u64m4_tu(vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m4_tu(maskedoff, vs2, vl); +vuint64m4_t test_vcpop_v_u64m4_tu(vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m4_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m8_tu( +// CHECK-LABEL: @test_vcpop_v_u64m8_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m8_t test_vcpopv_v_u64m8_tu(vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m8_tu(maskedoff, vs2, vl); +vuint64m8_t test_vcpop_v_u64m8_tu(vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m8_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf8_tum( +// CHECK-LABEL: @test_vcpop_v_u8mf8_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf8_t test_vcpopv_v_u8mf8_tum(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf8_tum(mask, maskedoff, vs2, vl); +vuint8mf8_t test_vcpop_v_u8mf8_tum(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf8_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf4_tum( +// CHECK-LABEL: @test_vcpop_v_u8mf4_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf4_t test_vcpopv_v_u8mf4_tum(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf4_tum(mask, maskedoff, vs2, vl); +vuint8mf4_t test_vcpop_v_u8mf4_tum(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf4_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf2_tum( +// CHECK-LABEL: @test_vcpop_v_u8mf2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf2_t test_vcpopv_v_u8mf2_tum(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf2_tum(mask, maskedoff, vs2, vl); +vuint8mf2_t test_vcpop_v_u8mf2_tum(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf2_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m1_tum( +// CHECK-LABEL: @test_vcpop_v_u8m1_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8m1_t test_vcpopv_v_u8m1_tum(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m1_tum(mask, maskedoff, vs2, vl); +vuint8m1_t test_vcpop_v_u8m1_tum(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m1_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m2_tum( +// CHECK-LABEL: @test_vcpop_v_u8m2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8m2_t test_vcpopv_v_u8m2_tum(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m2_tum(mask, maskedoff, vs2, vl); +vuint8m2_t test_vcpop_v_u8m2_tum(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m2_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m4_tum( +// CHECK-LABEL: @test_vcpop_v_u8m4_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8m4_t test_vcpopv_v_u8m4_tum(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m4_tum(mask, maskedoff, vs2, vl); +vuint8m4_t test_vcpop_v_u8m4_tum(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m4_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m8_tum( +// CHECK-LABEL: @test_vcpop_v_u8m8_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv64i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8m8_t test_vcpopv_v_u8m8_tum(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m8_tum(mask, maskedoff, vs2, vl); +vuint8m8_t test_vcpop_v_u8m8_tum(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m8_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf4_tum( +// CHECK-LABEL: @test_vcpop_v_u16mf4_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf4_t test_vcpopv_v_u16mf4_tum(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16mf4_tum(mask, maskedoff, vs2, vl); +vuint16mf4_t test_vcpop_v_u16mf4_tum(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { + return __riscv_vcpop_v_u16mf4_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf2_tum( +// CHECK-LABEL: @test_vcpop_v_u16mf2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf2_t test_vcpopv_v_u16mf2_tum(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16mf2_tum(mask, maskedoff, vs2, vl); +vuint16mf2_t test_vcpop_v_u16mf2_tum(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u16mf2_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m1_tum( +// CHECK-LABEL: @test_vcpop_v_u16m1_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint16m1_t test_vcpopv_v_u16m1_tum(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m1_tum(mask, maskedoff, vs2, vl); +vuint16m1_t test_vcpop_v_u16m1_tum(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m1_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m2_tum( +// CHECK-LABEL: @test_vcpop_v_u16m2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint16m2_t test_vcpopv_v_u16m2_tum(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m2_tum(mask, maskedoff, vs2, vl); +vuint16m2_t test_vcpop_v_u16m2_tum(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m2_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m4_tum( +// CHECK-LABEL: @test_vcpop_v_u16m4_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint16m4_t test_vcpopv_v_u16m4_tum(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m4_tum(mask, maskedoff, vs2, vl); +vuint16m4_t test_vcpop_v_u16m4_tum(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m4_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m8_tum( +// CHECK-LABEL: @test_vcpop_v_u16m8_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint16m8_t test_vcpopv_v_u16m8_tum(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m8_tum(mask, maskedoff, vs2, vl); +vuint16m8_t test_vcpop_v_u16m8_tum(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m8_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32mf2_tum( +// CHECK-LABEL: @test_vcpop_v_u32mf2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint32mf2_t test_vcpopv_v_u32mf2_tum(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32mf2_tum(mask, maskedoff, vs2, vl); +vuint32mf2_t test_vcpop_v_u32mf2_tum(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u32mf2_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m1_tum( +// CHECK-LABEL: @test_vcpop_v_u32m1_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint32m1_t test_vcpopv_v_u32m1_tum(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m1_tum(mask, maskedoff, vs2, vl); +vuint32m1_t test_vcpop_v_u32m1_tum(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m1_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m2_tum( +// CHECK-LABEL: @test_vcpop_v_u32m2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint32m2_t test_vcpopv_v_u32m2_tum(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m2_tum(mask, maskedoff, vs2, vl); +vuint32m2_t test_vcpop_v_u32m2_tum(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m2_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m4_tum( +// CHECK-LABEL: @test_vcpop_v_u32m4_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint32m4_t test_vcpopv_v_u32m4_tum(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m4_tum(mask, maskedoff, vs2, vl); +vuint32m4_t test_vcpop_v_u32m4_tum(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m4_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m8_tum( +// CHECK-LABEL: @test_vcpop_v_u32m8_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint32m8_t test_vcpopv_v_u32m8_tum(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m8_tum(mask, maskedoff, vs2, vl); +vuint32m8_t test_vcpop_v_u32m8_tum(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m8_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m1_tum( +// CHECK-LABEL: @test_vcpop_v_u64m1_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint64m1_t test_vcpopv_v_u64m1_tum(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m1_tum(mask, maskedoff, vs2, vl); +vuint64m1_t test_vcpop_v_u64m1_tum(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m1_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m2_tum( +// CHECK-LABEL: @test_vcpop_v_u64m2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint64m2_t test_vcpopv_v_u64m2_tum(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m2_tum(mask, maskedoff, vs2, vl); +vuint64m2_t test_vcpop_v_u64m2_tum(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m2_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m4_tum( +// CHECK-LABEL: @test_vcpop_v_u64m4_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint64m4_t test_vcpopv_v_u64m4_tum(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m4_tum(mask, maskedoff, vs2, vl); +vuint64m4_t test_vcpop_v_u64m4_tum(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m4_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m8_tum( +// CHECK-LABEL: @test_vcpop_v_u64m8_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint64m8_t test_vcpopv_v_u64m8_tum(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m8_tum(mask, maskedoff, vs2, vl); +vuint64m8_t test_vcpop_v_u64m8_tum(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m8_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf8_tumu( +// CHECK-LABEL: @test_vcpop_v_u8mf8_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf8_t test_vcpopv_v_u8mf8_tumu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf8_tumu(mask, maskedoff, vs2, vl); +vuint8mf8_t test_vcpop_v_u8mf8_tumu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf8_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf4_tumu( +// CHECK-LABEL: @test_vcpop_v_u8mf4_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf4_t test_vcpopv_v_u8mf4_tumu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf4_tumu(mask, maskedoff, vs2, vl); +vuint8mf4_t test_vcpop_v_u8mf4_tumu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf4_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf2_tumu( +// CHECK-LABEL: @test_vcpop_v_u8mf2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf2_t test_vcpopv_v_u8mf2_tumu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf2_tumu(mask, maskedoff, vs2, vl); +vuint8mf2_t test_vcpop_v_u8mf2_tumu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf2_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m1_tumu( +// CHECK-LABEL: @test_vcpop_v_u8m1_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8m1_t test_vcpopv_v_u8m1_tumu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m1_tumu(mask, maskedoff, vs2, vl); +vuint8m1_t test_vcpop_v_u8m1_tumu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m1_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m2_tumu( +// CHECK-LABEL: @test_vcpop_v_u8m2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8m2_t test_vcpopv_v_u8m2_tumu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m2_tumu(mask, maskedoff, vs2, vl); +vuint8m2_t test_vcpop_v_u8m2_tumu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m2_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m4_tumu( +// CHECK-LABEL: @test_vcpop_v_u8m4_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8m4_t test_vcpopv_v_u8m4_tumu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m4_tumu(mask, maskedoff, vs2, vl); +vuint8m4_t test_vcpop_v_u8m4_tumu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m4_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m8_tumu( +// CHECK-LABEL: @test_vcpop_v_u8m8_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv64i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8m8_t test_vcpopv_v_u8m8_tumu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m8_tumu(mask, maskedoff, vs2, vl); +vuint8m8_t test_vcpop_v_u8m8_tumu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m8_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf4_tumu( +// CHECK-LABEL: @test_vcpop_v_u16mf4_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf4_t test_vcpopv_v_u16mf4_tumu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16mf4_tumu(mask, maskedoff, vs2, vl); +vuint16mf4_t test_vcpop_v_u16mf4_tumu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { + return __riscv_vcpop_v_u16mf4_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf2_tumu( +// CHECK-LABEL: @test_vcpop_v_u16mf2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf2_t test_vcpopv_v_u16mf2_tumu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16mf2_tumu(mask, maskedoff, vs2, vl); +vuint16mf2_t test_vcpop_v_u16mf2_tumu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u16mf2_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m1_tumu( +// CHECK-LABEL: @test_vcpop_v_u16m1_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint16m1_t test_vcpopv_v_u16m1_tumu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m1_tumu(mask, maskedoff, vs2, vl); +vuint16m1_t test_vcpop_v_u16m1_tumu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m1_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m2_tumu( +// CHECK-LABEL: @test_vcpop_v_u16m2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint16m2_t test_vcpopv_v_u16m2_tumu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m2_tumu(mask, maskedoff, vs2, vl); +vuint16m2_t test_vcpop_v_u16m2_tumu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m2_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m4_tumu( +// CHECK-LABEL: @test_vcpop_v_u16m4_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint16m4_t test_vcpopv_v_u16m4_tumu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m4_tumu(mask, maskedoff, vs2, vl); +vuint16m4_t test_vcpop_v_u16m4_tumu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m4_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m8_tumu( +// CHECK-LABEL: @test_vcpop_v_u16m8_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint16m8_t test_vcpopv_v_u16m8_tumu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m8_tumu(mask, maskedoff, vs2, vl); +vuint16m8_t test_vcpop_v_u16m8_tumu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m8_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32mf2_tumu( +// CHECK-LABEL: @test_vcpop_v_u32mf2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint32mf2_t test_vcpopv_v_u32mf2_tumu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32mf2_tumu(mask, maskedoff, vs2, vl); +vuint32mf2_t test_vcpop_v_u32mf2_tumu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u32mf2_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m1_tumu( +// CHECK-LABEL: @test_vcpop_v_u32m1_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint32m1_t test_vcpopv_v_u32m1_tumu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m1_tumu(mask, maskedoff, vs2, vl); +vuint32m1_t test_vcpop_v_u32m1_tumu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m1_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m2_tumu( +// CHECK-LABEL: @test_vcpop_v_u32m2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint32m2_t test_vcpopv_v_u32m2_tumu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m2_tumu(mask, maskedoff, vs2, vl); +vuint32m2_t test_vcpop_v_u32m2_tumu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m2_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m4_tumu( +// CHECK-LABEL: @test_vcpop_v_u32m4_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint32m4_t test_vcpopv_v_u32m4_tumu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m4_tumu(mask, maskedoff, vs2, vl); +vuint32m4_t test_vcpop_v_u32m4_tumu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m4_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m8_tumu( +// CHECK-LABEL: @test_vcpop_v_u32m8_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint32m8_t test_vcpopv_v_u32m8_tumu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m8_tumu(mask, maskedoff, vs2, vl); +vuint32m8_t test_vcpop_v_u32m8_tumu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m8_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m1_tumu( +// CHECK-LABEL: @test_vcpop_v_u64m1_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint64m1_t test_vcpopv_v_u64m1_tumu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m1_tumu(mask, maskedoff, vs2, vl); +vuint64m1_t test_vcpop_v_u64m1_tumu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m1_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m2_tumu( +// CHECK-LABEL: @test_vcpop_v_u64m2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint64m2_t test_vcpopv_v_u64m2_tumu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m2_tumu(mask, maskedoff, vs2, vl); +vuint64m2_t test_vcpop_v_u64m2_tumu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m2_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m4_tumu( +// CHECK-LABEL: @test_vcpop_v_u64m4_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint64m4_t test_vcpopv_v_u64m4_tumu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m4_tumu(mask, maskedoff, vs2, vl); +vuint64m4_t test_vcpop_v_u64m4_tumu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m4_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m8_tumu( +// CHECK-LABEL: @test_vcpop_v_u64m8_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint64m8_t test_vcpopv_v_u64m8_tumu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m8_tumu(mask, maskedoff, vs2, vl); +vuint64m8_t test_vcpop_v_u64m8_tumu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m8_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf8_mu( +// CHECK-LABEL: @test_vcpop_v_u8mf8_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf8_t test_vcpopv_v_u8mf8_mu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf8_mu(mask, maskedoff, vs2, vl); +vuint8mf8_t test_vcpop_v_u8mf8_mu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf8_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf4_mu( +// CHECK-LABEL: @test_vcpop_v_u8mf4_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf4_t test_vcpopv_v_u8mf4_mu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf4_mu(mask, maskedoff, vs2, vl); +vuint8mf4_t test_vcpop_v_u8mf4_mu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf4_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf2_mu( +// CHECK-LABEL: @test_vcpop_v_u8mf2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf2_t test_vcpopv_v_u8mf2_mu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8mf2_mu(mask, maskedoff, vs2, vl); +vuint8mf2_t test_vcpop_v_u8mf2_mu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u8mf2_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m1_mu( +// CHECK-LABEL: @test_vcpop_v_u8m1_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8m1_t test_vcpopv_v_u8m1_mu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m1_mu(mask, maskedoff, vs2, vl); +vuint8m1_t test_vcpop_v_u8m1_mu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m1_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m2_mu( +// CHECK-LABEL: @test_vcpop_v_u8m2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8m2_t test_vcpopv_v_u8m2_mu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m2_mu(mask, maskedoff, vs2, vl); +vuint8m2_t test_vcpop_v_u8m2_mu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m2_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m4_mu( +// CHECK-LABEL: @test_vcpop_v_u8m4_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8m4_t test_vcpopv_v_u8m4_mu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m4_mu(mask, maskedoff, vs2, vl); +vuint8m4_t test_vcpop_v_u8m4_mu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m4_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m8_mu( +// CHECK-LABEL: @test_vcpop_v_u8m8_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv64i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8m8_t test_vcpopv_v_u8m8_mu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u8m8_mu(mask, maskedoff, vs2, vl); +vuint8m8_t test_vcpop_v_u8m8_mu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u8m8_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf4_mu( +// CHECK-LABEL: @test_vcpop_v_u16mf4_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf4_t test_vcpopv_v_u16mf4_mu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16mf4_mu(mask, maskedoff, vs2, vl); +vuint16mf4_t test_vcpop_v_u16mf4_mu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { + return __riscv_vcpop_v_u16mf4_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf2_mu( +// CHECK-LABEL: @test_vcpop_v_u16mf2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf2_t test_vcpopv_v_u16mf2_mu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16mf2_mu(mask, maskedoff, vs2, vl); +vuint16mf2_t test_vcpop_v_u16mf2_mu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u16mf2_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m1_mu( +// CHECK-LABEL: @test_vcpop_v_u16m1_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint16m1_t test_vcpopv_v_u16m1_mu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m1_mu(mask, maskedoff, vs2, vl); +vuint16m1_t test_vcpop_v_u16m1_mu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m1_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m2_mu( +// CHECK-LABEL: @test_vcpop_v_u16m2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint16m2_t test_vcpopv_v_u16m2_mu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m2_mu(mask, maskedoff, vs2, vl); +vuint16m2_t test_vcpop_v_u16m2_mu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m2_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m4_mu( +// CHECK-LABEL: @test_vcpop_v_u16m4_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint16m4_t test_vcpopv_v_u16m4_mu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m4_mu(mask, maskedoff, vs2, vl); +vuint16m4_t test_vcpop_v_u16m4_mu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m4_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m8_mu( +// CHECK-LABEL: @test_vcpop_v_u16m8_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint16m8_t test_vcpopv_v_u16m8_mu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u16m8_mu(mask, maskedoff, vs2, vl); +vuint16m8_t test_vcpop_v_u16m8_mu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u16m8_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32mf2_mu( +// CHECK-LABEL: @test_vcpop_v_u32mf2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint32mf2_t test_vcpopv_v_u32mf2_mu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32mf2_mu(mask, maskedoff, vs2, vl); +vuint32mf2_t test_vcpop_v_u32mf2_mu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { + return __riscv_vcpop_v_u32mf2_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m1_mu( +// CHECK-LABEL: @test_vcpop_v_u32m1_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint32m1_t test_vcpopv_v_u32m1_mu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m1_mu(mask, maskedoff, vs2, vl); +vuint32m1_t test_vcpop_v_u32m1_mu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m1_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m2_mu( +// CHECK-LABEL: @test_vcpop_v_u32m2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint32m2_t test_vcpopv_v_u32m2_mu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m2_mu(mask, maskedoff, vs2, vl); +vuint32m2_t test_vcpop_v_u32m2_mu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m2_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m4_mu( +// CHECK-LABEL: @test_vcpop_v_u32m4_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint32m4_t test_vcpopv_v_u32m4_mu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m4_mu(mask, maskedoff, vs2, vl); +vuint32m4_t test_vcpop_v_u32m4_mu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m4_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m8_mu( +// CHECK-LABEL: @test_vcpop_v_u32m8_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint32m8_t test_vcpopv_v_u32m8_mu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u32m8_mu(mask, maskedoff, vs2, vl); +vuint32m8_t test_vcpop_v_u32m8_mu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u32m8_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m1_mu( +// CHECK-LABEL: @test_vcpop_v_u64m1_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint64m1_t test_vcpopv_v_u64m1_mu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m1_mu(mask, maskedoff, vs2, vl); +vuint64m1_t test_vcpop_v_u64m1_mu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m1_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m2_mu( +// CHECK-LABEL: @test_vcpop_v_u64m2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint64m2_t test_vcpopv_v_u64m2_mu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m2_mu(mask, maskedoff, vs2, vl); +vuint64m2_t test_vcpop_v_u64m2_mu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m2_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m4_mu( +// CHECK-LABEL: @test_vcpop_v_u64m4_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint64m4_t test_vcpopv_v_u64m4_mu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m4_mu(mask, maskedoff, vs2, vl); +vuint64m4_t test_vcpop_v_u64m4_mu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m4_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m8_mu( +// CHECK-LABEL: @test_vcpop_v_u64m8_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint64m8_t test_vcpopv_v_u64m8_mu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { - return __riscv_vcpopv_v_u64m8_mu(mask, maskedoff, vs2, vl); +vuint64m8_t test_vcpop_v_u64m8_mu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { + return __riscv_vcpop_v_u64m8_mu(mask, maskedoff, vs2, vl); } diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vcpopv.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vcpopv.c index 02a499d4b67d..953ccac133c3 100644 --- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vcpopv.c +++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vcpopv.c @@ -16,795 +16,795 @@ #include -// CHECK-LABEL: @test_vcpopv_v_u8mf8_tu( +// CHECK-LABEL: @test_vcpop_v_u8mf8_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf8_t test_vcpopv_v_u8mf8_tu(vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint8mf8_t test_vcpop_v_u8mf8_tu(vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf4_tu( +// CHECK-LABEL: @test_vcpop_v_u8mf4_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf4_t test_vcpopv_v_u8mf4_tu(vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint8mf4_t test_vcpop_v_u8mf4_tu(vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf2_tu( +// CHECK-LABEL: @test_vcpop_v_u8mf2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf2_t test_vcpopv_v_u8mf2_tu(vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint8mf2_t test_vcpop_v_u8mf2_tu(vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m1_tu( +// CHECK-LABEL: @test_vcpop_v_u8m1_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m1_t test_vcpopv_v_u8m1_tu(vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint8m1_t test_vcpop_v_u8m1_tu(vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m2_tu( +// CHECK-LABEL: @test_vcpop_v_u8m2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv16i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m2_t test_vcpopv_v_u8m2_tu(vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint8m2_t test_vcpop_v_u8m2_tu(vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m4_tu( +// CHECK-LABEL: @test_vcpop_v_u8m4_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv32i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m4_t test_vcpopv_v_u8m4_tu(vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint8m4_t test_vcpop_v_u8m4_tu(vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m8_tu( +// CHECK-LABEL: @test_vcpop_v_u8m8_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv64i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint8m8_t test_vcpopv_v_u8m8_tu(vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint8m8_t test_vcpop_v_u8m8_tu(vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf4_tu( +// CHECK-LABEL: @test_vcpop_v_u16mf4_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf4_t test_vcpopv_v_u16mf4_tu(vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint16mf4_t test_vcpop_v_u16mf4_tu(vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf2_tu( +// CHECK-LABEL: @test_vcpop_v_u16mf2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf2_t test_vcpopv_v_u16mf2_tu(vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint16mf2_t test_vcpop_v_u16mf2_tu(vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m1_tu( +// CHECK-LABEL: @test_vcpop_v_u16m1_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m1_t test_vcpopv_v_u16m1_tu(vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint16m1_t test_vcpop_v_u16m1_tu(vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m2_tu( +// CHECK-LABEL: @test_vcpop_v_u16m2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m2_t test_vcpopv_v_u16m2_tu(vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint16m2_t test_vcpop_v_u16m2_tu(vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m4_tu( +// CHECK-LABEL: @test_vcpop_v_u16m4_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv16i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m4_t test_vcpopv_v_u16m4_tu(vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint16m4_t test_vcpop_v_u16m4_tu(vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m8_tu( +// CHECK-LABEL: @test_vcpop_v_u16m8_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv32i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint16m8_t test_vcpopv_v_u16m8_tu(vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint16m8_t test_vcpop_v_u16m8_tu(vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32mf2_tu( +// CHECK-LABEL: @test_vcpop_v_u32mf2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32mf2_t test_vcpopv_v_u32mf2_tu(vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint32mf2_t test_vcpop_v_u32mf2_tu(vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m1_tu( +// CHECK-LABEL: @test_vcpop_v_u32m1_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m1_t test_vcpopv_v_u32m1_tu(vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint32m1_t test_vcpop_v_u32m1_tu(vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m2_tu( +// CHECK-LABEL: @test_vcpop_v_u32m2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m2_t test_vcpopv_v_u32m2_tu(vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint32m2_t test_vcpop_v_u32m2_tu(vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m4_tu( +// CHECK-LABEL: @test_vcpop_v_u32m4_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m4_t test_vcpopv_v_u32m4_tu(vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint32m4_t test_vcpop_v_u32m4_tu(vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m8_tu( +// CHECK-LABEL: @test_vcpop_v_u32m8_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv16i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint32m8_t test_vcpopv_v_u32m8_tu(vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint32m8_t test_vcpop_v_u32m8_tu(vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m1_tu( +// CHECK-LABEL: @test_vcpop_v_u64m1_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv1i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m1_t test_vcpopv_v_u64m1_tu(vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint64m1_t test_vcpop_v_u64m1_tu(vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m2_tu( +// CHECK-LABEL: @test_vcpop_v_u64m2_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv2i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m2_t test_vcpopv_v_u64m2_tu(vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint64m2_t test_vcpop_v_u64m2_tu(vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m4_tu( +// CHECK-LABEL: @test_vcpop_v_u64m4_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv4i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m4_t test_vcpopv_v_u64m4_tu(vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint64m4_t test_vcpop_v_u64m4_tu(vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m8_tu( +// CHECK-LABEL: @test_vcpop_v_u64m8_tu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.nxv8i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], i64 [[VL:%.*]]) // CHECK-NEXT: ret [[TMP0]] // -vuint64m8_t test_vcpopv_v_u64m8_tu(vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { - return __riscv_vcpopv_tu(maskedoff, vs2, vl); +vuint64m8_t test_vcpop_v_u64m8_tu(vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { + return __riscv_vcpop_tu(maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf8_tum( +// CHECK-LABEL: @test_vcpop_v_u8mf8_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf8_t test_vcpopv_v_u8mf8_tum(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint8mf8_t test_vcpop_v_u8mf8_tum(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf4_tum( +// CHECK-LABEL: @test_vcpop_v_u8mf4_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf4_t test_vcpopv_v_u8mf4_tum(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint8mf4_t test_vcpop_v_u8mf4_tum(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf2_tum( +// CHECK-LABEL: @test_vcpop_v_u8mf2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf2_t test_vcpopv_v_u8mf2_tum(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint8mf2_t test_vcpop_v_u8mf2_tum(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m1_tum( +// CHECK-LABEL: @test_vcpop_v_u8m1_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8m1_t test_vcpopv_v_u8m1_tum(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint8m1_t test_vcpop_v_u8m1_tum(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m2_tum( +// CHECK-LABEL: @test_vcpop_v_u8m2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8m2_t test_vcpopv_v_u8m2_tum(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint8m2_t test_vcpop_v_u8m2_tum(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m4_tum( +// CHECK-LABEL: @test_vcpop_v_u8m4_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8m4_t test_vcpopv_v_u8m4_tum(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint8m4_t test_vcpop_v_u8m4_tum(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m8_tum( +// CHECK-LABEL: @test_vcpop_v_u8m8_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv64i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint8m8_t test_vcpopv_v_u8m8_tum(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint8m8_t test_vcpop_v_u8m8_tum(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf4_tum( +// CHECK-LABEL: @test_vcpop_v_u16mf4_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf4_t test_vcpopv_v_u16mf4_tum(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint16mf4_t test_vcpop_v_u16mf4_tum(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf2_tum( +// CHECK-LABEL: @test_vcpop_v_u16mf2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf2_t test_vcpopv_v_u16mf2_tum(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint16mf2_t test_vcpop_v_u16mf2_tum(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m1_tum( +// CHECK-LABEL: @test_vcpop_v_u16m1_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint16m1_t test_vcpopv_v_u16m1_tum(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint16m1_t test_vcpop_v_u16m1_tum(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m2_tum( +// CHECK-LABEL: @test_vcpop_v_u16m2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint16m2_t test_vcpopv_v_u16m2_tum(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint16m2_t test_vcpop_v_u16m2_tum(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m4_tum( +// CHECK-LABEL: @test_vcpop_v_u16m4_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint16m4_t test_vcpopv_v_u16m4_tum(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint16m4_t test_vcpop_v_u16m4_tum(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m8_tum( +// CHECK-LABEL: @test_vcpop_v_u16m8_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint16m8_t test_vcpopv_v_u16m8_tum(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint16m8_t test_vcpop_v_u16m8_tum(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32mf2_tum( +// CHECK-LABEL: @test_vcpop_v_u32mf2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint32mf2_t test_vcpopv_v_u32mf2_tum(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint32mf2_t test_vcpop_v_u32mf2_tum(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m1_tum( +// CHECK-LABEL: @test_vcpop_v_u32m1_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint32m1_t test_vcpopv_v_u32m1_tum(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint32m1_t test_vcpop_v_u32m1_tum(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m2_tum( +// CHECK-LABEL: @test_vcpop_v_u32m2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint32m2_t test_vcpopv_v_u32m2_tum(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint32m2_t test_vcpop_v_u32m2_tum(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m4_tum( +// CHECK-LABEL: @test_vcpop_v_u32m4_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint32m4_t test_vcpopv_v_u32m4_tum(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint32m4_t test_vcpop_v_u32m4_tum(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m8_tum( +// CHECK-LABEL: @test_vcpop_v_u32m8_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint32m8_t test_vcpopv_v_u32m8_tum(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint32m8_t test_vcpop_v_u32m8_tum(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m1_tum( +// CHECK-LABEL: @test_vcpop_v_u64m1_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint64m1_t test_vcpopv_v_u64m1_tum(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint64m1_t test_vcpop_v_u64m1_tum(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m2_tum( +// CHECK-LABEL: @test_vcpop_v_u64m2_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint64m2_t test_vcpopv_v_u64m2_tum(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint64m2_t test_vcpop_v_u64m2_tum(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m4_tum( +// CHECK-LABEL: @test_vcpop_v_u64m4_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint64m4_t test_vcpopv_v_u64m4_tum(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint64m4_t test_vcpop_v_u64m4_tum(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m8_tum( +// CHECK-LABEL: @test_vcpop_v_u64m8_tum( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 2) // CHECK-NEXT: ret [[TMP0]] // -vuint64m8_t test_vcpopv_v_u64m8_tum(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { - return __riscv_vcpopv_tum(mask, maskedoff, vs2, vl); +vuint64m8_t test_vcpop_v_u64m8_tum(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { + return __riscv_vcpop_tum(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf8_tumu( +// CHECK-LABEL: @test_vcpop_v_u8mf8_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf8_t test_vcpopv_v_u8mf8_tumu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint8mf8_t test_vcpop_v_u8mf8_tumu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf4_tumu( +// CHECK-LABEL: @test_vcpop_v_u8mf4_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf4_t test_vcpopv_v_u8mf4_tumu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint8mf4_t test_vcpop_v_u8mf4_tumu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf2_tumu( +// CHECK-LABEL: @test_vcpop_v_u8mf2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf2_t test_vcpopv_v_u8mf2_tumu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint8mf2_t test_vcpop_v_u8mf2_tumu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m1_tumu( +// CHECK-LABEL: @test_vcpop_v_u8m1_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8m1_t test_vcpopv_v_u8m1_tumu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint8m1_t test_vcpop_v_u8m1_tumu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m2_tumu( +// CHECK-LABEL: @test_vcpop_v_u8m2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8m2_t test_vcpopv_v_u8m2_tumu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint8m2_t test_vcpop_v_u8m2_tumu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m4_tumu( +// CHECK-LABEL: @test_vcpop_v_u8m4_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8m4_t test_vcpopv_v_u8m4_tumu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint8m4_t test_vcpop_v_u8m4_tumu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m8_tumu( +// CHECK-LABEL: @test_vcpop_v_u8m8_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv64i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint8m8_t test_vcpopv_v_u8m8_tumu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint8m8_t test_vcpop_v_u8m8_tumu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf4_tumu( +// CHECK-LABEL: @test_vcpop_v_u16mf4_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf4_t test_vcpopv_v_u16mf4_tumu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint16mf4_t test_vcpop_v_u16mf4_tumu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf2_tumu( +// CHECK-LABEL: @test_vcpop_v_u16mf2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf2_t test_vcpopv_v_u16mf2_tumu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint16mf2_t test_vcpop_v_u16mf2_tumu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m1_tumu( +// CHECK-LABEL: @test_vcpop_v_u16m1_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint16m1_t test_vcpopv_v_u16m1_tumu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint16m1_t test_vcpop_v_u16m1_tumu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m2_tumu( +// CHECK-LABEL: @test_vcpop_v_u16m2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint16m2_t test_vcpopv_v_u16m2_tumu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint16m2_t test_vcpop_v_u16m2_tumu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m4_tumu( +// CHECK-LABEL: @test_vcpop_v_u16m4_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint16m4_t test_vcpopv_v_u16m4_tumu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint16m4_t test_vcpop_v_u16m4_tumu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m8_tumu( +// CHECK-LABEL: @test_vcpop_v_u16m8_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint16m8_t test_vcpopv_v_u16m8_tumu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint16m8_t test_vcpop_v_u16m8_tumu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32mf2_tumu( +// CHECK-LABEL: @test_vcpop_v_u32mf2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint32mf2_t test_vcpopv_v_u32mf2_tumu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint32mf2_t test_vcpop_v_u32mf2_tumu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m1_tumu( +// CHECK-LABEL: @test_vcpop_v_u32m1_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint32m1_t test_vcpopv_v_u32m1_tumu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint32m1_t test_vcpop_v_u32m1_tumu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m2_tumu( +// CHECK-LABEL: @test_vcpop_v_u32m2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint32m2_t test_vcpopv_v_u32m2_tumu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint32m2_t test_vcpop_v_u32m2_tumu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m4_tumu( +// CHECK-LABEL: @test_vcpop_v_u32m4_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint32m4_t test_vcpopv_v_u32m4_tumu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint32m4_t test_vcpop_v_u32m4_tumu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m8_tumu( +// CHECK-LABEL: @test_vcpop_v_u32m8_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint32m8_t test_vcpopv_v_u32m8_tumu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint32m8_t test_vcpop_v_u32m8_tumu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m1_tumu( +// CHECK-LABEL: @test_vcpop_v_u64m1_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint64m1_t test_vcpopv_v_u64m1_tumu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint64m1_t test_vcpop_v_u64m1_tumu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m2_tumu( +// CHECK-LABEL: @test_vcpop_v_u64m2_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint64m2_t test_vcpopv_v_u64m2_tumu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint64m2_t test_vcpop_v_u64m2_tumu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m4_tumu( +// CHECK-LABEL: @test_vcpop_v_u64m4_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint64m4_t test_vcpopv_v_u64m4_tumu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint64m4_t test_vcpop_v_u64m4_tumu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m8_tumu( +// CHECK-LABEL: @test_vcpop_v_u64m8_tumu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) // CHECK-NEXT: ret [[TMP0]] // -vuint64m8_t test_vcpopv_v_u64m8_tumu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { - return __riscv_vcpopv_tumu(mask, maskedoff, vs2, vl); +vuint64m8_t test_vcpop_v_u64m8_tumu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { + return __riscv_vcpop_tumu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf8_mu( +// CHECK-LABEL: @test_vcpop_v_u8mf8_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf8_t test_vcpopv_v_u8mf8_mu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint8mf8_t test_vcpop_v_u8mf8_mu(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf4_mu( +// CHECK-LABEL: @test_vcpop_v_u8mf4_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf4_t test_vcpopv_v_u8mf4_mu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint8mf4_t test_vcpop_v_u8mf4_mu(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8mf2_mu( +// CHECK-LABEL: @test_vcpop_v_u8mf2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8mf2_t test_vcpopv_v_u8mf2_mu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint8mf2_t test_vcpop_v_u8mf2_mu(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m1_mu( +// CHECK-LABEL: @test_vcpop_v_u8m1_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8m1_t test_vcpopv_v_u8m1_mu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint8m1_t test_vcpop_v_u8m1_mu(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m2_mu( +// CHECK-LABEL: @test_vcpop_v_u8m2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8m2_t test_vcpopv_v_u8m2_mu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint8m2_t test_vcpop_v_u8m2_mu(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m4_mu( +// CHECK-LABEL: @test_vcpop_v_u8m4_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8m4_t test_vcpopv_v_u8m4_mu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint8m4_t test_vcpop_v_u8m4_mu(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u8m8_mu( +// CHECK-LABEL: @test_vcpop_v_u8m8_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv64i8.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint8m8_t test_vcpopv_v_u8m8_mu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint8m8_t test_vcpop_v_u8m8_mu(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf4_mu( +// CHECK-LABEL: @test_vcpop_v_u16mf4_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf4_t test_vcpopv_v_u16mf4_mu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint16mf4_t test_vcpop_v_u16mf4_mu(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16mf2_mu( +// CHECK-LABEL: @test_vcpop_v_u16mf2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint16mf2_t test_vcpopv_v_u16mf2_mu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint16mf2_t test_vcpop_v_u16mf2_mu(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m1_mu( +// CHECK-LABEL: @test_vcpop_v_u16m1_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint16m1_t test_vcpopv_v_u16m1_mu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint16m1_t test_vcpop_v_u16m1_mu(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m2_mu( +// CHECK-LABEL: @test_vcpop_v_u16m2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint16m2_t test_vcpopv_v_u16m2_mu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint16m2_t test_vcpop_v_u16m2_mu(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m4_mu( +// CHECK-LABEL: @test_vcpop_v_u16m4_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint16m4_t test_vcpopv_v_u16m4_mu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint16m4_t test_vcpop_v_u16m4_mu(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u16m8_mu( +// CHECK-LABEL: @test_vcpop_v_u16m8_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv32i16.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint16m8_t test_vcpopv_v_u16m8_mu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint16m8_t test_vcpop_v_u16m8_mu(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32mf2_mu( +// CHECK-LABEL: @test_vcpop_v_u32mf2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint32mf2_t test_vcpopv_v_u32mf2_mu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint32mf2_t test_vcpop_v_u32mf2_mu(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m1_mu( +// CHECK-LABEL: @test_vcpop_v_u32m1_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint32m1_t test_vcpopv_v_u32m1_mu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint32m1_t test_vcpop_v_u32m1_mu(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m2_mu( +// CHECK-LABEL: @test_vcpop_v_u32m2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint32m2_t test_vcpopv_v_u32m2_mu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint32m2_t test_vcpop_v_u32m2_mu(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m4_mu( +// CHECK-LABEL: @test_vcpop_v_u32m4_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint32m4_t test_vcpopv_v_u32m4_mu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint32m4_t test_vcpop_v_u32m4_mu(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u32m8_mu( +// CHECK-LABEL: @test_vcpop_v_u32m8_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv16i32.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint32m8_t test_vcpopv_v_u32m8_mu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint32m8_t test_vcpop_v_u32m8_mu(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m1_mu( +// CHECK-LABEL: @test_vcpop_v_u64m1_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv1i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint64m1_t test_vcpopv_v_u64m1_mu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint64m1_t test_vcpop_v_u64m1_mu(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m2_mu( +// CHECK-LABEL: @test_vcpop_v_u64m2_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv2i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint64m2_t test_vcpopv_v_u64m2_mu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint64m2_t test_vcpop_v_u64m2_mu(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m4_mu( +// CHECK-LABEL: @test_vcpop_v_u64m4_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv4i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint64m4_t test_vcpopv_v_u64m4_mu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint64m4_t test_vcpop_v_u64m4_mu(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -// CHECK-LABEL: @test_vcpopv_v_u64m8_mu( +// CHECK-LABEL: @test_vcpop_v_u64m8_mu( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vcpopv.mask.nxv8i64.i64( [[MASKEDOFF:%.*]], [[VS2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) // CHECK-NEXT: ret [[TMP0]] // -vuint64m8_t test_vcpopv_v_u64m8_mu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { - return __riscv_vcpopv_mu(mask, maskedoff, vs2, vl); +vuint64m8_t test_vcpop_v_u64m8_mu(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t vs2, size_t vl) { + return __riscv_vcpop_mu(mask, maskedoff, vs2, vl); } -- GitLab From 9eac38a000d510e9b162d6996816ef298019b553 Mon Sep 17 00:00:00 2001 From: Teresa Johnson Date: Thu, 6 Jun 2024 11:04:45 -0700 Subject: [PATCH 142/462] [MemProf] Remove context id set from nodes and recompute on demand (#94415) The ContextIds set on the ContextNode struct is not technically needed as we can compute it from either the callee or caller edge context ids. Remove it and add a helper to recompute from the edges on demand. Also add helpers to compute the node allocation type and whether the context ids are empty from the edges without needing to first compute the node's context id set, to minimize the runtime cost increase. This yielded a 20% reduction in peak memory for a large thin link, for about a 2% time increase (which is more than offset by some other recent time efficiency improvements). --- .../IPO/MemProfContextDisambiguation.cpp | 251 +++++++++++------- 1 file changed, 155 insertions(+), 96 deletions(-) diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp index 03923b83cf34..f033d2b0d6d0 100644 --- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp +++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp @@ -262,8 +262,70 @@ public: // TODO: Should this be a map (from Caller node) for more efficient lookup? std::vector> CallerEdges; - // The set of IDs for contexts including this node. - DenseSet ContextIds; + // Get the list of edges from which we can compute allocation information + // such as the context ids and allocation type of this node. + const std::vector> * + getEdgesWithAllocInfo() const { + // If node has any callees, compute from those, otherwise compute from + // callers (i.e. if this is the leaf allocation node). + if (!CalleeEdges.empty()) + return &CalleeEdges; + if (!CallerEdges.empty()) { + // A node with caller edges but no callee edges must be the allocation + // node. + assert(IsAllocation); + return &CallerEdges; + } + return nullptr; + } + + // Compute the context ids for this node from the union of its edge context + // ids. + DenseSet getContextIds() const { + DenseSet ContextIds; + auto *Edges = getEdgesWithAllocInfo(); + if (!Edges) + return {}; + unsigned Count = 0; + for (auto &Edge : *Edges) + Count += Edge->getContextIds().size(); + ContextIds.reserve(Count); + for (auto &Edge : *Edges) + ContextIds.insert(Edge->getContextIds().begin(), + Edge->getContextIds().end()); + return ContextIds; + } + + // Compute the allocation type for this node from the OR of its edge + // allocation types. + uint8_t computeAllocType() const { + auto *Edges = getEdgesWithAllocInfo(); + if (!Edges) + return (uint8_t)AllocationType::None; + uint8_t BothTypes = + (uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold; + uint8_t AllocType = (uint8_t)AllocationType::None; + for (auto &Edge : *Edges) { + AllocType |= Edge->AllocTypes; + // Bail early if alloc type reached both, no further refinement. + if (AllocType == BothTypes) + return AllocType; + } + return AllocType; + } + + // The context ids set for this node is empty if its edge context ids are + // also all empty. + bool emptyContextIds() const { + auto *Edges = getEdgesWithAllocInfo(); + if (!Edges) + return true; + for (auto &Edge : *Edges) { + if (!Edge->getContextIds().empty()) + return false; + } + return true; + } // List of clones of this ContextNode, initially empty. std::vector Clones; @@ -308,15 +370,11 @@ public: void printCall(raw_ostream &OS) const { Call.print(OS); } // True if this node was effectively removed from the graph, in which case - // its context id set, caller edges, and callee edges should all be empty. + // it should have an allocation type of None and empty context ids. bool isRemoved() const { - // Note that we can have non-empty context ids with empty caller and - // callee edges if the graph ends up with a single node. - if (ContextIds.empty()) - assert(CalleeEdges.empty() && CallerEdges.empty() && - "Context ids empty but at least one of callee and caller edges " - "were not!"); - return ContextIds.empty(); + assert((AllocTypes == (uint8_t)AllocationType::None) == + emptyContextIds()); + return AllocTypes == (uint8_t)AllocationType::None; } void dump() const; @@ -429,7 +487,8 @@ private: /// else to its callers. Also updates OrigNode's edges to remove any context /// ids moved to the newly created edge. void connectNewNode(ContextNode *NewNode, ContextNode *OrigNode, - bool TowardsCallee); + bool TowardsCallee, + DenseSet RemainingContextIds); /// Get the stack id corresponding to the given Id or Index (for IR this will /// return itself, for a summary index this will return the id recorded in the @@ -958,7 +1017,6 @@ void CallsiteContextGraph::addStackNodesForMIB( // Update alloc type and context ids for this MIB. AllocNode->AllocTypes |= (uint8_t)AllocType; - AllocNode->ContextIds.insert(LastContextId); // Now add or update nodes for each stack id in alloc's context. // Later when processing the stack ids on non-alloc callsites we will adjust @@ -983,7 +1041,6 @@ void CallsiteContextGraph::addStackNodesForMIB( auto Ins = StackIdSet.insert(StackId); if (!Ins.second) StackNode->Recursive = true; - StackNode->ContextIds.insert(LastContextId); StackNode->AllocTypes |= (uint8_t)AllocType; PrevNode->addOrUpdateCallerEdge(StackNode, AllocType, LastContextId); PrevNode = StackNode; @@ -1034,7 +1091,6 @@ void CallsiteContextGraph:: // it resulted in any added ids to NextNode. if (!NewIdsToAdd.empty()) { Edge->getContextIds().insert(NewIdsToAdd.begin(), NewIdsToAdd.end()); - NextNode->ContextIds.insert(NewIdsToAdd.begin(), NewIdsToAdd.end()); UpdateCallers(NextNode, Visited, UpdateCallers); } } @@ -1043,21 +1099,16 @@ void CallsiteContextGraph:: DenseSet Visited; for (auto &Entry : AllocationCallToContextNodeMap) { auto *Node = Entry.second; - // Update ids on the allocation nodes before calling the recursive - // update along caller edges, since this simplifies the logic during - // that traversal. - DenseSet NewIdsToAdd = GetNewIds(Node->ContextIds); - Node->ContextIds.insert(NewIdsToAdd.begin(), NewIdsToAdd.end()); UpdateCallers(Node, Visited, UpdateCallers); } } template void CallsiteContextGraph::connectNewNode( - ContextNode *NewNode, ContextNode *OrigNode, bool TowardsCallee) { - // Make a copy of the context ids, since this will be adjusted below as they - // are moved. - DenseSet RemainingContextIds = NewNode->ContextIds; + ContextNode *NewNode, ContextNode *OrigNode, bool TowardsCallee, + // This must be passed by value to make a copy since it will be adjusted + // as ids are moved. + DenseSet RemainingContextIds) { auto &OrigEdges = TowardsCallee ? OrigNode->CalleeEdges : OrigNode->CallerEdges; // Increment iterator in loop so that we can remove edges as needed. @@ -1103,6 +1154,51 @@ void CallsiteContextGraph::connectNewNode( } } +template +static void checkEdge( + const std::shared_ptr> &Edge) { + // Confirm that alloc type is not None and that we have at least one context + // id. + assert(Edge->AllocTypes != (uint8_t)AllocationType::None); + assert(!Edge->ContextIds.empty()); +} + +template +static void checkNode(const ContextNode *Node, + bool CheckEdges = true) { + if (Node->isRemoved()) + return; +#ifndef NDEBUG + // Compute node's context ids once for use in asserts. + auto NodeContextIds = Node->getContextIds(); +#endif + // Node's context ids should be the union of both its callee and caller edge + // context ids. + if (Node->CallerEdges.size()) { + DenseSet CallerEdgeContextIds( + Node->CallerEdges.front()->ContextIds); + for (const auto &Edge : llvm::drop_begin(Node->CallerEdges)) { + if (CheckEdges) + checkEdge(Edge); + set_union(CallerEdgeContextIds, Edge->ContextIds); + } + // Node can have more context ids than callers if some contexts terminate at + // node and some are longer. + assert(NodeContextIds == CallerEdgeContextIds || + set_is_subset(CallerEdgeContextIds, NodeContextIds)); + } + if (Node->CalleeEdges.size()) { + DenseSet CalleeEdgeContextIds( + Node->CalleeEdges.front()->ContextIds); + for (const auto &Edge : llvm::drop_begin(Node->CalleeEdges)) { + if (CheckEdges) + checkEdge(Edge); + set_union(CalleeEdgeContextIds, Edge->getContextIds()); + } + assert(NodeContextIds == CalleeEdgeContextIds); + } +} + template void CallsiteContextGraph:: assignStackNodesPostOrder(ContextNode *Node, @@ -1178,7 +1274,7 @@ void CallsiteContextGraph:: // duplicated context ids. We have to recompute as we might have overlap // overlap between the saved context ids for different last nodes, and // removed them already during the post order traversal. - set_intersect(SavedContextIds, FirstNode->ContextIds); + set_intersect(SavedContextIds, FirstNode->getContextIds()); ContextNode *PrevNode = nullptr; for (auto Id : Ids) { ContextNode *CurNode = getNodeForStackId(Id); @@ -1211,18 +1307,17 @@ void CallsiteContextGraph:: ContextNode *NewNode = NodeOwner.back().get(); NodeToCallingFunc[NewNode] = Func; NonAllocationCallToContextNodeMap[Call] = NewNode; - NewNode->ContextIds = SavedContextIds; - NewNode->AllocTypes = computeAllocType(NewNode->ContextIds); + NewNode->AllocTypes = computeAllocType(SavedContextIds); // Connect to callees of innermost stack frame in inlined call chain. // This updates context ids for FirstNode's callee's to reflect those // moved to NewNode. - connectNewNode(NewNode, FirstNode, /*TowardsCallee=*/true); + connectNewNode(NewNode, FirstNode, /*TowardsCallee=*/true, SavedContextIds); // Connect to callers of outermost stack frame in inlined call chain. // This updates context ids for FirstNode's caller's to reflect those // moved to NewNode. - connectNewNode(NewNode, LastNode, /*TowardsCallee=*/false); + connectNewNode(NewNode, LastNode, /*TowardsCallee=*/false, SavedContextIds); // Now we need to remove context ids from edges/nodes between First and // Last Node. @@ -1234,18 +1329,32 @@ void CallsiteContextGraph:: // Remove the context ids moved to NewNode from CurNode, and the // edge from the prior node. - set_subtract(CurNode->ContextIds, NewNode->ContextIds); if (PrevNode) { auto *PrevEdge = CurNode->findEdgeFromCallee(PrevNode); assert(PrevEdge); - set_subtract(PrevEdge->getContextIds(), NewNode->ContextIds); + set_subtract(PrevEdge->getContextIds(), SavedContextIds); if (PrevEdge->getContextIds().empty()) { PrevNode->eraseCallerEdge(PrevEdge); CurNode->eraseCalleeEdge(PrevEdge); } } + // Since we update the edges from leaf to tail, only look at the callee + // edges. This isn't an alloc node, so if there are no callee edges, the + // alloc type is None. + CurNode->AllocTypes = CurNode->CalleeEdges.empty() + ? (uint8_t)AllocationType::None + : CurNode->computeAllocType(); PrevNode = CurNode; } + if (VerifyNodes) { + checkNode(NewNode, /*CheckEdges=*/true); + for (auto Id : Ids) { + ContextNode *CurNode = getNodeForStackId(Id); + // We should only have kept stack ids that had nodes. + assert(CurNode); + checkNode(CurNode, /*CheckEdges=*/true); + } + } } } @@ -1319,7 +1428,7 @@ void CallsiteContextGraph::updateStackNodes() { // Initialize the context ids with the last node's. We will subsequently // refine the context ids by computing the intersection along all edges. - DenseSet LastNodeContextIds = LastNode->ContextIds; + DenseSet LastNodeContextIds = LastNode->getContextIds(); assert(!LastNodeContextIds.empty()); for (unsigned I = 0; I < Calls.size(); I++) { @@ -1442,6 +1551,8 @@ void CallsiteContextGraph::updateStackNodes() { DenseSet Visited; for (auto &Entry : AllocationCallToContextNodeMap) assignStackNodesPostOrder(Entry.second, Visited, StackIdToMatchingCalls); + if (VerifyCCG) + check(); } uint64_t ModuleCallsiteContextGraph::getLastStackId(Instruction *Call) { @@ -1786,8 +1897,6 @@ bool CallsiteContextGraph::calleesMatch( // First check if we have already synthesized a node for this tail call. if (TailCallToContextNodeMap.count(NewCall)) { NewNode = TailCallToContextNodeMap[NewCall]; - NewNode->ContextIds.insert(Edge->ContextIds.begin(), - Edge->ContextIds.end()); NewNode->AllocTypes |= Edge->AllocTypes; } else { FuncToCallsWithMetadata[Func].push_back({NewCall}); @@ -1797,7 +1906,6 @@ bool CallsiteContextGraph::calleesMatch( NewNode = NodeOwner.back().get(); NodeToCallingFunc[NewNode] = Func; TailCallToContextNodeMap[NewCall] = NewNode; - NewNode->ContextIds = Edge->ContextIds; NewNode->AllocTypes = Edge->AllocTypes; } @@ -2091,6 +2199,8 @@ void CallsiteContextGraph::ContextNode::print( OS << "\n"; OS << "\tAllocTypes: " << getAllocTypeString(AllocTypes) << "\n"; OS << "\tContextIds:"; + // Make a copy of the computed context ids that we can sort for stability. + auto ContextIds = getContextIds(); std::vector SortedIds(ContextIds.begin(), ContextIds.end()); std::sort(SortedIds.begin(), SortedIds.end()); for (auto Id : SortedIds) @@ -2150,53 +2260,6 @@ void CallsiteContextGraph::print( } } -template -static void checkEdge( - const std::shared_ptr> &Edge) { - // Confirm that alloc type is not None and that we have at least one context - // id. - assert(Edge->AllocTypes != (uint8_t)AllocationType::None); - assert(!Edge->ContextIds.empty()); -} - -template -static void checkNode(const ContextNode *Node, - bool CheckEdges = true) { - if (Node->isRemoved()) - return; - // Node's context ids should be the union of both its callee and caller edge - // context ids. - if (Node->CallerEdges.size()) { - auto EI = Node->CallerEdges.begin(); - auto &FirstEdge = *EI; - EI++; - DenseSet CallerEdgeContextIds(FirstEdge->ContextIds); - for (; EI != Node->CallerEdges.end(); EI++) { - const auto &Edge = *EI; - if (CheckEdges) - checkEdge(Edge); - set_union(CallerEdgeContextIds, Edge->ContextIds); - } - // Node can have more context ids than callers if some contexts terminate at - // node and some are longer. - assert(Node->ContextIds == CallerEdgeContextIds || - set_is_subset(CallerEdgeContextIds, Node->ContextIds)); - } - if (Node->CalleeEdges.size()) { - auto EI = Node->CalleeEdges.begin(); - auto &FirstEdge = *EI; - EI++; - DenseSet CalleeEdgeContextIds(FirstEdge->ContextIds); - for (; EI != Node->CalleeEdges.end(); EI++) { - const auto &Edge = *EI; - if (CheckEdges) - checkEdge(Edge); - set_union(CalleeEdgeContextIds, Edge->ContextIds); - } - assert(Node->ContextIds == CalleeEdgeContextIds); - } -} - template void CallsiteContextGraph::check() const { using GraphType = const CallsiteContextGraph *; @@ -2284,7 +2347,7 @@ struct DOTGraphTraits *> static std::string getNodeAttributes(NodeRef Node, GraphType) { std::string AttributeString = (Twine("tooltip=\"") + getNodeId(Node) + " " + - getContextIds(Node->ContextIds) + "\"") + getContextIds(Node->getContextIds()) + "\"") .str(); AttributeString += (Twine(",fillcolor=\"") + getColor(Node->AllocTypes) + "\"").str(); @@ -2443,16 +2506,6 @@ void CallsiteContextGraph:: set_subtract(Edge->ContextIds, ContextIdsToMove); Edge->AllocTypes = computeAllocType(Edge->ContextIds); } - // Now perform some updates that are common to all cases: the NewCallee gets - // the moved ids added, and we need to remove those ids from OldCallee and - // update its alloc type (NewCallee alloc type updates handled above). - NewCallee->ContextIds.insert(ContextIdsToMove.begin(), - ContextIdsToMove.end()); - set_subtract(OldCallee->ContextIds, ContextIdsToMove); - OldCallee->AllocTypes = computeAllocType(OldCallee->ContextIds); - // OldCallee alloc type should be None iff its context id set is now empty. - assert((OldCallee->AllocTypes == (uint8_t)AllocationType::None) == - OldCallee->ContextIds.empty()); // Now walk the old callee node's callee edges and move Edge's context ids // over to the corresponding edge into the clone (which is created here if // this is a newly created clone). @@ -2484,6 +2537,12 @@ void CallsiteContextGraph:: NewCallee->CalleeEdges.push_back(NewEdge); NewEdge->Callee->CallerEdges.push_back(NewEdge); } + // Recompute the node alloc type now that its callee edges have been + // updated (since we will compute from those edges). + OldCallee->AllocTypes = OldCallee->computeAllocType(); + // OldCallee alloc type should be None iff its context id set is now empty. + assert((OldCallee->AllocTypes == (uint8_t)AllocationType::None) == + OldCallee->emptyContextIds()); if (VerifyCCG) { checkNode(OldCallee, /*CheckEdges=*/false); checkNode(NewCallee, /*CheckEdges=*/false); @@ -2528,7 +2587,7 @@ void CallsiteContextGraph::identifyClones() { DenseSet Visited; for (auto &Entry : AllocationCallToContextNodeMap) { Visited.clear(); - identifyClones(Entry.second, Visited, Entry.second->ContextIds); + identifyClones(Entry.second, Visited, Entry.second->getContextIds()); } Visited.clear(); for (auto &Entry : AllocationCallToContextNodeMap) @@ -2714,7 +2773,7 @@ void CallsiteContextGraph::identifyClones( } // We should still have some context ids on the original Node. - assert(!Node->ContextIds.empty()); + assert(!Node->emptyContextIds()); // Sanity check that no alloc types on node or edges are None. assert(Node->AllocTypes != (uint8_t)AllocationType::None); @@ -2918,7 +2977,7 @@ bool CallsiteContextGraph::assignFunctions() { // find additional cloning is required. std::deque ClonesWorklist; // Ignore original Node if we moved all of its contexts to clones. - if (!Node->ContextIds.empty()) + if (!Node->emptyContextIds()) ClonesWorklist.push_back(Node); ClonesWorklist.insert(ClonesWorklist.end(), Node->Clones.begin(), Node->Clones.end()); @@ -3258,7 +3317,7 @@ bool CallsiteContextGraph::assignFunctions() { // Skip if either no call to update, or if we ended up with no context ids // (we moved all edges onto other clones). - if (!Node->hasCall() || Node->ContextIds.empty()) + if (!Node->hasCall() || Node->emptyContextIds()) return; if (Node->IsAllocation) { -- GitLab From 88cdd9905597ace5b1ac7d080df5326d3399b3f8 Mon Sep 17 00:00:00 2001 From: khaki3 <47756807+khaki3@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:16:40 -0700 Subject: [PATCH 143/462] [flang] Add reduction semantics to fir.do_loop (#93934) Derived from #92480. This PR introduces reduction semantics into loops for DO CONCURRENT REDUCE. The `fir.do_loop` operation now invisibly has the `operandSegmentsizes` attribute and takes variable-length reduction operands with their operations given as `fir.reduce_attr`. For the sake of compatibility, `fir.do_loop`'s builder has additional arguments at the end. The `iter_args` operand should be placed in front of the declaration of result types, so the new operand for reduction variables (`reduce`) is put in the middle of arguments. --- .../flang/Optimizer/Dialect/FIRAttr.td | 30 ++++++++ .../include/flang/Optimizer/Dialect/FIROps.td | 35 +++++++-- flang/lib/Optimizer/Dialect/FIRAttr.cpp | 4 +- flang/lib/Optimizer/Dialect/FIROps.cpp | 73 +++++++++++++++++-- flang/test/Fir/loop03.fir | 17 +++++ 5 files changed, 142 insertions(+), 17 deletions(-) create mode 100644 flang/test/Fir/loop03.fir diff --git a/flang/include/flang/Optimizer/Dialect/FIRAttr.td b/flang/include/flang/Optimizer/Dialect/FIRAttr.td index 0c34b640a5c9..aedb6769186e 100644 --- a/flang/include/flang/Optimizer/Dialect/FIRAttr.td +++ b/flang/include/flang/Optimizer/Dialect/FIRAttr.td @@ -67,6 +67,36 @@ def fir_BoxFieldAttr : I32EnumAttr< let cppNamespace = "fir"; } +def fir_ReduceOperationEnum : I32BitEnumAttr<"ReduceOperationEnum", + "intrinsic operations and functions supported by DO CONCURRENT REDUCE", + [ + I32BitEnumAttrCaseBit<"Add", 0, "add">, + I32BitEnumAttrCaseBit<"Multiply", 1, "multiply">, + I32BitEnumAttrCaseBit<"AND", 2, "and">, + I32BitEnumAttrCaseBit<"OR", 3, "or">, + I32BitEnumAttrCaseBit<"EQV", 4, "eqv">, + I32BitEnumAttrCaseBit<"NEQV", 5, "neqv">, + I32BitEnumAttrCaseBit<"MAX", 6, "max">, + I32BitEnumAttrCaseBit<"MIN", 7, "min">, + I32BitEnumAttrCaseBit<"IAND", 8, "iand">, + I32BitEnumAttrCaseBit<"IOR", 9, "ior">, + I32BitEnumAttrCaseBit<"EIOR", 10, "eior"> + ]> { + let separator = ", "; + let cppNamespace = "::fir"; + let printBitEnumPrimaryGroups = 1; +} + +def fir_ReduceAttr : fir_Attr<"Reduce"> { + let mnemonic = "reduce_attr"; + + let parameters = (ins + "ReduceOperationEnum":$reduce_operation + ); + + let assemblyFormat = "`<` $reduce_operation `>`"; +} + // mlir::SideEffects::Resource for modelling operations which add debugging information def DebuggingResource : Resource<"::fir::DebuggingResource">; diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td index 37fbd1f9692a..e7da3af5485c 100644 --- a/flang/include/flang/Optimizer/Dialect/FIROps.td +++ b/flang/include/flang/Optimizer/Dialect/FIROps.td @@ -2125,8 +2125,8 @@ class region_Op traits = []> : let hasVerifier = 1; } -def fir_DoLoopOp : region_Op<"do_loop", - [DeclareOpInterfaceMethods]> { let summary = "generalized loop operation"; let description = [{ @@ -2156,9 +2156,11 @@ def fir_DoLoopOp : region_Op<"do_loop", Index:$lowerBound, Index:$upperBound, Index:$step, + Variadic:$reduceOperands, Variadic:$initArgs, OptionalAttr:$unordered, - OptionalAttr:$finalValue + OptionalAttr:$finalValue, + OptionalAttr:$reduceAttrs ); let results = (outs Variadic:$results); let regions = (region SizedRegion<1>:$region); @@ -2169,6 +2171,8 @@ def fir_DoLoopOp : region_Op<"do_loop", "mlir::Value":$step, CArg<"bool", "false">:$unordered, CArg<"bool", "false">:$finalCountValue, CArg<"mlir::ValueRange", "std::nullopt">:$iterArgs, + CArg<"mlir::ValueRange", "std::nullopt">:$reduceOperands, + CArg<"llvm::ArrayRef", "{}">:$reduceAttrs, CArg<"llvm::ArrayRef", "{}">:$attributes)> ]; @@ -2181,11 +2185,12 @@ def fir_DoLoopOp : region_Op<"do_loop", return getBody()->getArguments().drop_front(); } mlir::Operation::operand_range getIterOperands() { - return getOperands().drop_front(getNumControlOperands()); + return getOperands() + .drop_front(getNumControlOperands() + getNumReduceOperands()); } llvm::MutableArrayRef getInitsMutable() { - return - getOperation()->getOpOperands().drop_front(getNumControlOperands()); + return getOperation()->getOpOperands() + .drop_front(getNumControlOperands() + getNumReduceOperands()); } void setLowerBound(mlir::Value bound) { (*this)->setOperand(0, bound); } @@ -2200,11 +2205,25 @@ def fir_DoLoopOp : region_Op<"do_loop", unsigned getNumControlOperands() { return 3; } /// Does the operation hold operands for loop-carried values bool hasIterOperands() { - return (*this)->getNumOperands() > getNumControlOperands(); + return getNumIterOperands() > 0; + } + /// Does the operation hold operands for reduction variables + bool hasReduceOperands() { + return getNumReduceOperands() > 0; + } + /// Get Number of variadic operands + unsigned getNumOperands(unsigned idx) { + auto segments = (*this)->getAttrOfType( + getOperandSegmentSizeAttr()); + return static_cast(segments[idx]); + } + // Get Number of reduction operands + unsigned getNumReduceOperands() { + return getNumOperands(3); } /// Get Number of loop-carried values unsigned getNumIterOperands() { - return (*this)->getNumOperands() - getNumControlOperands(); + return getNumOperands(4); } /// Get the body of the loop diff --git a/flang/lib/Optimizer/Dialect/FIRAttr.cpp b/flang/lib/Optimizer/Dialect/FIRAttr.cpp index 2faba63dfba0..a0202a015922 100644 --- a/flang/lib/Optimizer/Dialect/FIRAttr.cpp +++ b/flang/lib/Optimizer/Dialect/FIRAttr.cpp @@ -297,6 +297,6 @@ void fir::printFirAttribute(FIROpsDialect *dialect, mlir::Attribute attr, void FIROpsDialect::registerAttributes() { addAttributes(); + LowerBoundAttr, PointIntervalAttr, RealAttr, ReduceAttr, + SubclassAttr, UpperBoundAttr>(); } diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp index b530a9dc1bcc..75ca738211ab 100644 --- a/flang/lib/Optimizer/Dialect/FIROps.cpp +++ b/flang/lib/Optimizer/Dialect/FIROps.cpp @@ -2456,9 +2456,16 @@ void fir::DoLoopOp::build(mlir::OpBuilder &builder, mlir::OperationState &result, mlir::Value lb, mlir::Value ub, mlir::Value step, bool unordered, bool finalCountValue, mlir::ValueRange iterArgs, + mlir::ValueRange reduceOperands, + llvm::ArrayRef reduceAttrs, llvm::ArrayRef attributes) { result.addOperands({lb, ub, step}); + result.addOperands(reduceOperands); result.addOperands(iterArgs); + result.addAttribute(getOperandSegmentSizeAttr(), + builder.getDenseI32ArrayAttr( + {1, 1, 1, static_cast(reduceOperands.size()), + static_cast(iterArgs.size())})); if (finalCountValue) { result.addTypes(builder.getIndexType()); result.addAttribute(getFinalValueAttrName(result.name), @@ -2477,6 +2484,9 @@ void fir::DoLoopOp::build(mlir::OpBuilder &builder, if (unordered) result.addAttribute(getUnorderedAttrName(result.name), builder.getUnitAttr()); + if (!reduceAttrs.empty()) + result.addAttribute(getReduceAttrsAttrName(result.name), + builder.getArrayAttr(reduceAttrs)); result.addAttributes(attributes); } @@ -2502,24 +2512,51 @@ mlir::ParseResult fir::DoLoopOp::parse(mlir::OpAsmParser &parser, if (mlir::succeeded(parser.parseOptionalKeyword("unordered"))) result.addAttribute("unordered", builder.getUnitAttr()); + // Parse the reduction arguments. + llvm::SmallVector reduceOperands; + llvm::SmallVector reduceArgTypes; + if (succeeded(parser.parseOptionalKeyword("reduce"))) { + // Parse reduction attributes and variables. + llvm::SmallVector attributes; + if (failed(parser.parseCommaSeparatedList( + mlir::AsmParser::Delimiter::Paren, [&]() { + if (parser.parseAttribute(attributes.emplace_back()) || + parser.parseArrow() || + parser.parseOperand(reduceOperands.emplace_back()) || + parser.parseColonType(reduceArgTypes.emplace_back())) + return mlir::failure(); + return mlir::success(); + }))) + return mlir::failure(); + // Resolve input operands. + for (auto operand_type : llvm::zip(reduceOperands, reduceArgTypes)) + if (parser.resolveOperand(std::get<0>(operand_type), + std::get<1>(operand_type), result.operands)) + return mlir::failure(); + llvm::SmallVector arrayAttr(attributes.begin(), + attributes.end()); + result.addAttribute(getReduceAttrsAttrName(result.name), + builder.getArrayAttr(arrayAttr)); + } + // Parse the optional initial iteration arguments. llvm::SmallVector regionArgs; - llvm::SmallVector operands; + llvm::SmallVector iterOperands; llvm::SmallVector argTypes; bool prependCount = false; regionArgs.push_back(inductionVariable); if (succeeded(parser.parseOptionalKeyword("iter_args"))) { // Parse assignment list and results type list. - if (parser.parseAssignmentList(regionArgs, operands) || + if (parser.parseAssignmentList(regionArgs, iterOperands) || parser.parseArrowTypeList(result.types)) return mlir::failure(); - if (result.types.size() == operands.size() + 1) + if (result.types.size() == iterOperands.size() + 1) prependCount = true; // Resolve input operands. llvm::ArrayRef resTypes = result.types; - for (auto operand_type : - llvm::zip(operands, prependCount ? resTypes.drop_front() : resTypes)) + for (auto operand_type : llvm::zip( + iterOperands, prependCount ? resTypes.drop_front() : resTypes)) if (parser.resolveOperand(std::get<0>(operand_type), std::get<1>(operand_type), result.operands)) return mlir::failure(); @@ -2530,6 +2567,12 @@ mlir::ParseResult fir::DoLoopOp::parse(mlir::OpAsmParser &parser, prependCount = true; } + // Set the operandSegmentSizes attribute + result.addAttribute(getOperandSegmentSizeAttr(), + builder.getDenseI32ArrayAttr( + {1, 1, 1, static_cast(reduceOperands.size()), + static_cast(iterOperands.size())})); + if (parser.parseOptionalAttrDictWithKeyword(result.attributes)) return mlir::failure(); @@ -2606,6 +2649,10 @@ mlir::LogicalResult fir::DoLoopOp::verify() { i++; } + auto reduceAttrs = getReduceAttrsAttr(); + if (getNumReduceOperands() != (reduceAttrs ? reduceAttrs.size() : 0)) + return emitOpError( + "mismatch in number of reduction variables and reduction attributes"); return mlir::success(); } @@ -2615,6 +2662,17 @@ void fir::DoLoopOp::print(mlir::OpAsmPrinter &p) { << getUpperBound() << " step " << getStep(); if (getUnordered()) p << " unordered"; + if (hasReduceOperands()) { + p << " reduce("; + auto attrs = getReduceAttrsAttr(); + auto operands = getReduceOperands(); + llvm::interleaveComma(llvm::zip(attrs, operands), p, [&](auto it) { + p << std::get<0>(it) << " -> " << std::get<1>(it) << " : " + << std::get<1>(it).getType(); + }); + p << ')'; + printBlockTerminators = true; + } if (hasIterOperands()) { p << " iter_args("; auto regionArgs = getRegionIterArgs(); @@ -2628,8 +2686,9 @@ void fir::DoLoopOp::print(mlir::OpAsmPrinter &p) { p << " -> " << getResultTypes(); printBlockTerminators = true; } - p.printOptionalAttrDictWithKeyword((*this)->getAttrs(), - {"unordered", "finalValue"}); + p.printOptionalAttrDictWithKeyword( + (*this)->getAttrs(), + {"unordered", "finalValue", "reduceAttrs", "operandSegmentSizes"}); p << ' '; p.printRegion(getRegion(), /*printEntryBlockArgs=*/false, printBlockTerminators); diff --git a/flang/test/Fir/loop03.fir b/flang/test/Fir/loop03.fir new file mode 100644 index 000000000000..b88dcaf8639b --- /dev/null +++ b/flang/test/Fir/loop03.fir @@ -0,0 +1,17 @@ +// Test the reduction semantics of fir.do_loop +// RUN: fir-opt %s | FileCheck %s + +func.func @reduction() { + %bound = arith.constant 10 : index + %step = arith.constant 1 : index + %sum = fir.alloca i32 +// CHECK: %[[VAL_0:.*]] = fir.alloca i32 +// CHECK: fir.do_loop %[[VAL_1:.*]] = %[[VAL_2:.*]] to %[[VAL_3:.*]] step %[[VAL_4:.*]] unordered reduce(#fir.reduce_attr -> %[[VAL_0]] : !fir.ref) { + fir.do_loop %iv = %step to %bound step %step unordered reduce(#fir.reduce_attr -> %sum : !fir.ref) { + %index = fir.convert %iv : (index) -> i32 + %1 = fir.load %sum : !fir.ref + %2 = arith.addi %index, %1 : i32 + fir.store %2 to %sum : !fir.ref + } + return +} -- GitLab From 649edb8eb25e82e3ac6fce4788f51759636229ec Mon Sep 17 00:00:00 2001 From: PiJoules <6019989+PiJoules@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:25:16 -0700 Subject: [PATCH 144/462] [libc][FixedVector] Add more helper methods (#94278) This adds: - A ctor accepting a start and end iterator - A ctor accepting a count and const T& - size() - subscript operators - begin() and end() iterators --- libc/src/__support/fixedvector.h | 18 ++++++++++++- libc/test/src/__support/CMakeLists.txt | 1 + libc/test/src/__support/fixedvector_test.cpp | 27 ++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/libc/src/__support/fixedvector.h b/libc/src/__support/fixedvector.h index 6aeb4d56363e..ddd0993a9527 100644 --- a/libc/src/__support/fixedvector.h +++ b/libc/src/__support/fixedvector.h @@ -24,6 +24,17 @@ template class FixedVector { public: constexpr FixedVector() = default; + using iterator = typename cpp::array::iterator; + constexpr FixedVector(iterator begin, iterator end) { + for (; begin != end; ++begin) + push_back(*begin); + } + + constexpr FixedVector(size_t count, const T &value) { + for (size_t i = 0; i < count; ++i) + push_back(value); + } + bool push_back(const T &obj) { if (item_count == CAPACITY) return false; @@ -43,8 +54,14 @@ public: return true; } + T &operator[](size_t idx) { return store[idx]; } + + const T &operator[](size_t idx) const { return store[idx]; } + bool empty() const { return item_count == 0; } + size_t size() const { return item_count; } + // Empties the store for all practical purposes. void reset() { item_count = 0; } @@ -64,7 +81,6 @@ public: } LIBC_INLINE constexpr reverse_iterator rend() { return store.rend(); } - using iterator = typename cpp::array::iterator; LIBC_INLINE constexpr iterator begin() { return store.begin(); } LIBC_INLINE constexpr iterator end() { return iterator{&store[item_count]}; } }; diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt index 5afc4173f61a..d05377eca8a8 100644 --- a/libc/test/src/__support/CMakeLists.txt +++ b/libc/test/src/__support/CMakeLists.txt @@ -132,6 +132,7 @@ add_libc_test( SRCS fixedvector_test.cpp DEPENDS + libc.src.__support.CPP.array libc.src.__support.fixedvector ) diff --git a/libc/test/src/__support/fixedvector_test.cpp b/libc/test/src/__support/fixedvector_test.cpp index e9ffdd0203c2..212e1aed20f7 100644 --- a/libc/test/src/__support/fixedvector_test.cpp +++ b/libc/test/src/__support/fixedvector_test.cpp @@ -6,6 +6,7 @@ // //===----------------------------------------------------------------------===// +#include "src/__support/CPP/array.h" #include "src/__support/fixedvector.h" #include "test/UnitTest/Test.h" @@ -69,3 +70,29 @@ TEST(LlvmLibcFixedVectorTest, Iteration) { for (int &x : v) ASSERT_GE(x, 0); } + +TEST(LlvmLibcFixedVectorTest, ConstructionFromIterators) { + LIBC_NAMESPACE::cpp::array arr{1, 2, 3, 4}; + LIBC_NAMESPACE::FixedVector vec(arr.begin(), arr.end()); + ASSERT_EQ(vec.size(), arr.size()); + for (size_t i = 0; i < arr.size(); ++i) + ASSERT_EQ(vec[i], arr[i]); +} + +TEST(LlvmLibcFixedVectorTest, ConstructionFromCountAndValue) { + constexpr int kVal = 10; + LIBC_NAMESPACE::FixedVector vec(4, kVal); + ASSERT_EQ(vec.size(), size_t(4)); + for (size_t i = 0; i < vec.size(); ++i) + ASSERT_EQ(vec[i], kVal); +} + +TEST(LlvmLibcFixedVectorTest, ForwardIteration) { + LIBC_NAMESPACE::cpp::array arr{1, 2, 3, 4}; + LIBC_NAMESPACE::FixedVector vec(arr.begin(), arr.end()); + ASSERT_EQ(vec.size(), arr.size()); + for (auto it = vec.begin(); it != vec.end(); ++it) { + auto idx = it - vec.begin(); + ASSERT_EQ(*it, arr[idx]); + } +} -- GitLab From f0785484c8ebf42ae0284cd608ad5f78eae20e95 Mon Sep 17 00:00:00 2001 From: Yuxuan Chen Date: Thu, 6 Jun 2024 11:32:19 -0700 Subject: [PATCH 145/462] [clang][NFC] fix name lookup for llvm::json::Value in SymbolGraphSerializer (#94511) This code uses namespaces `llvm` and `llvm::json`. However, we have both `llvm::Value` and `llvm::json::Value`. Whenever any of the headers declare or include `llvm::Value`, the lookup becomes ambiguous. Fixing this by qualifying the `Value` type. --- clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp index 08e711cafae2..6e56ee5b573f 100644 --- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp +++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp @@ -31,7 +31,6 @@ using namespace clang; using namespace clang::extractapi; using namespace llvm; -using namespace llvm::json; namespace { @@ -1036,9 +1035,9 @@ void SymbolGraphSerializer::serializeGraphToStream( ExtendedModule &&EM) { Object Root = serializeGraph(ModuleName, std::move(EM)); if (Options.Compact) - OS << formatv("{0}", Value(std::move(Root))) << "\n"; + OS << formatv("{0}", json::Value(std::move(Root))) << "\n"; else - OS << formatv("{0:2}", Value(std::move(Root))) << "\n"; + OS << formatv("{0:2}", json::Value(std::move(Root))) << "\n"; } void SymbolGraphSerializer::serializeMainSymbolGraph( -- GitLab From d55e235b2384281a5d1d982094fb2f819999885b Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 6 Jun 2024 12:01:38 -0700 Subject: [PATCH 146/462] [memprof] Use std::unique_ptr instead of std::optional (#94655) Changing the type of Frame::SymbolName from std::optional to std::unique reduces sizeof(Frame) from 64 to 32. The smaller type reduces the cycle and instruction counts by 23% and 4.4%, respectively, with "llvm-profdata show" modified to deserialize all MemProfRecords in a MemProf V2 profile. The peak memory usage is cut down nearly by half. --- llvm/include/llvm/ProfileData/MemProf.h | 14 +++++++++----- llvm/lib/ProfileData/MemProfReader.cpp | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h index 406144d9db1e..528abe1ad3d4 100644 --- a/llvm/include/llvm/ProfileData/MemProf.h +++ b/llvm/include/llvm/ProfileData/MemProf.h @@ -199,7 +199,7 @@ struct Frame { GlobalValue::GUID Function; // The symbol name for the function. Only populated in the Frame by the reader // if requested during initialization. This field should not be serialized. - std::optional SymbolName; + std::unique_ptr SymbolName; // The source line offset of the call from the beginning of parent function. uint32_t LineOffset; // The source column number of the call to help distinguish multiple calls @@ -210,7 +210,9 @@ struct Frame { Frame(const Frame &Other) { Function = Other.Function; - SymbolName = Other.SymbolName; + SymbolName = Other.SymbolName + ? std::make_unique(*Other.SymbolName) + : nullptr; LineOffset = Other.LineOffset; Column = Other.Column; IsInlineFrame = Other.IsInlineFrame; @@ -228,7 +230,9 @@ struct Frame { Frame &operator=(const Frame &Other) { Function = Other.Function; - SymbolName = Other.SymbolName; + SymbolName = Other.SymbolName + ? std::make_unique(*Other.SymbolName) + : nullptr; LineOffset = Other.LineOffset; Column = Other.Column; IsInlineFrame = Other.IsInlineFrame; @@ -237,10 +241,10 @@ struct Frame { bool operator!=(const Frame &Other) const { return !operator==(Other); } - bool hasSymbolName() const { return SymbolName.has_value(); } + bool hasSymbolName() const { return !!SymbolName; } StringRef getSymbolName() const { - assert(SymbolName.has_value()); + assert(hasSymbolName()); return *SymbolName; } diff --git a/llvm/lib/ProfileData/MemProfReader.cpp b/llvm/lib/ProfileData/MemProfReader.cpp index fc3be716087e..693897f874a2 100644 --- a/llvm/lib/ProfileData/MemProfReader.cpp +++ b/llvm/lib/ProfileData/MemProfReader.cpp @@ -690,7 +690,7 @@ Error RawMemProfReader::readNextRecord( return F; auto Iter = this->GuidToSymbolName.find(F.Function); assert(Iter != this->GuidToSymbolName.end()); - F.SymbolName = Iter->getSecond(); + F.SymbolName = std::make_unique(Iter->getSecond()); return F; }; return MemProfReader::readNextRecord(GuidRecord, IdToFrameCallback); -- GitLab From c771b670eabbd38867d43475dacd35a1b572e9b5 Mon Sep 17 00:00:00 2001 From: Stanislav Mekhanoshin Date: Thu, 6 Jun 2024 12:05:51 -0700 Subject: [PATCH 147/462] [AMDGPU] Promote immediate offset to atomics (#94043) --- .../Target/AMDGPU/SILoadStoreOptimizer.cpp | 7 - .../AMDGPU/promote-constOffset-to-imm.mir | 165 ++++++++++++++++++ 2 files changed, 165 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp index c47eea20563d..8b42d4a1dee7 100644 --- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp +++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp @@ -2052,9 +2052,6 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm( MemInfoMap &Visited, SmallPtrSet &AnchorList) const { - if (!(MI.mayLoad() ^ MI.mayStore())) - return false; - if (!STM->hasFlatInstOffsets() || !SIInstrInfo::isFLAT(MI)) return false; @@ -2065,10 +2062,6 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm( unsigned AS = SIInstrInfo::isFLATGlobal(MI) ? AMDGPUAS::GLOBAL_ADDRESS : AMDGPUAS::FLAT_ADDRESS; - if (MI.mayLoad() && - TII->getNamedOperand(MI, AMDGPU::OpName::vdata) != nullptr) - return false; - if (AnchorList.count(&MI)) return false; diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.mir b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.mir index 4ecce2842455..6dda1fe1f39d 100644 --- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.mir +++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.mir @@ -282,3 +282,168 @@ body: | %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1 FLAT_STORE_DWORD %13, %0.sub1, 0, 0, implicit $exec, implicit $flat_scr ... + +--- +# GCN-LABEL: name: diffoporder_add_global_atomic_cmpswap +# GFX9: GLOBAL_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 1000, 0, +# GFX9: GLOBAL_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 0, 0, + +# GFX8: GLOBAL_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 0, 0, +# GFX8: GLOBAL_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 0, 0, + +name: diffoporder_add_global_atomic_cmpswap +body: | + bb.0.entry: + + %0:vreg_64 = COPY $vgpr0_vgpr1 + + %1:sgpr_32 = S_MOV_B32 4000 + %2:vgpr_32, %3:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %1, 0, implicit $exec + %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %3, 0, implicit $exec + %6:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %4, %subreg.sub1 + GLOBAL_ATOMIC_CMPSWAP %6:vreg_64, %0:vreg_64, 0, 0, implicit $exec + + %8:sgpr_32 = S_MOV_B32 3000 + %9:vgpr_32, %10:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %8, 0, implicit $exec + %11:vgpr_32, dead %12:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %10, 0, implicit $exec + %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1 + GLOBAL_ATOMIC_CMPSWAP %13:vreg_64, %0:vreg_64, 0, 0, implicit $exec +... + +--- +# GCN-LABEL: name: diffoporder_add_flat_atomic_cmpswap +# GFX9: FLAT_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 1000, 0, +# GFX9: FLAT_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 0, 0, + +# GFX8: FLAT_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 0, 0, +# GFX8: FLAT_ATOMIC_CMPSWAP %{{[0-9]+}}, %0, 0, 0, + +name: diffoporder_add_flat_atomic_cmpswap +body: | + bb.0.entry: + + %0:vreg_64 = COPY $vgpr0_vgpr1 + + %1:sgpr_32 = S_MOV_B32 4000 + %2:vgpr_32, %3:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %1, 0, implicit $exec + %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %3, 0, implicit $exec + %6:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %4, %subreg.sub1 + FLAT_ATOMIC_CMPSWAP %6:vreg_64, %0:vreg_64, 0, 0, implicit $exec, implicit $flat_scr + + + %8:sgpr_32 = S_MOV_B32 3000 + %9:vgpr_32, %10:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %8, 0, implicit $exec + %11:vgpr_32, dead %12:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %10, 0, implicit $exec + %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1 + FLAT_ATOMIC_CMPSWAP %13:vreg_64, %0:vreg_64, 0, 0, implicit $exec, implicit $flat_scr +... + +--- +# GCN-LABEL: name: diffoporder_add_global_atomic_add +# GFX9: GLOBAL_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 1000, 0, +# GFX9: GLOBAL_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 0, 0, + +# GFX8: GLOBAL_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 0, 0, +# GFX8: GLOBAL_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 0, 0, + +name: diffoporder_add_global_atomic_add +body: | + bb.0.entry: + + %0:vreg_64 = COPY $vgpr0_vgpr1 + + %1:sgpr_32 = S_MOV_B32 4000 + %2:vgpr_32, %3:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %1, 0, implicit $exec + %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %3, 0, implicit $exec + %6:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %4, %subreg.sub1 + GLOBAL_ATOMIC_ADD %6:vreg_64, %0.sub0, 0, 0, implicit $exec + + %8:sgpr_32 = S_MOV_B32 3000 + %9:vgpr_32, %10:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %8, 0, implicit $exec + %11:vgpr_32, dead %12:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %10, 0, implicit $exec + %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1 + GLOBAL_ATOMIC_ADD %13:vreg_64, %0.sub0, 0, 0, implicit $exec +... + +--- +# GCN-LABEL: name: diffoporder_add_flat_atomic_add +# GFX9: FLAT_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 1000, 0, +# GFX9: FLAT_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 0, 0, + +# GFX8: FLAT_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 0, 0, +# GFX8: FLAT_ATOMIC_ADD %{{[0-9]+}}, %0.sub0, 0, 0, + +name: diffoporder_add_flat_atomic_add +body: | + bb.0.entry: + + %0:vreg_64 = COPY $vgpr0_vgpr1 + + %1:sgpr_32 = S_MOV_B32 4000 + %2:vgpr_32, %3:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %1, 0, implicit $exec + %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %3, 0, implicit $exec + %6:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %4, %subreg.sub1 + FLAT_ATOMIC_ADD %6:vreg_64, %0.sub0, 0, 0, implicit $exec, implicit $flat_scr + + + %8:sgpr_32 = S_MOV_B32 3000 + %9:vgpr_32, %10:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %8, 0, implicit $exec + %11:vgpr_32, dead %12:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %10, 0, implicit $exec + %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1 + FLAT_ATOMIC_ADD %13:vreg_64, %0.sub0, 0, 0, implicit $exec, implicit $flat_scr +... + +--- +# GCN-LABEL: name: diffoporder_add_global_atomic_add_rtn +# GFX9: GLOBAL_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 1000, 0, +# GFX9: GLOBAL_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 0, 0, + +# GFX8: GLOBAL_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 0, 0, +# GFX8: GLOBAL_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 0, 0, + +name: diffoporder_add_global_atomic_add_rtn +body: | + bb.0.entry: + + %0:vreg_64 = COPY $vgpr0_vgpr1 + + %1:sgpr_32 = S_MOV_B32 4000 + %2:vgpr_32, %3:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %1, 0, implicit $exec + %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %3, 0, implicit $exec + %6:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %4, %subreg.sub1 + %14:vgpr_32 = GLOBAL_ATOMIC_ADD_RTN %6:vreg_64, %0.sub0, 0, 0, implicit $exec + + %8:sgpr_32 = S_MOV_B32 3000 + %9:vgpr_32, %10:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %8, 0, implicit $exec + %11:vgpr_32, dead %12:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %10, 0, implicit $exec + %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1 + %15:vgpr_32 = GLOBAL_ATOMIC_ADD_RTN %13:vreg_64, %0.sub0, 0, 0, implicit $exec +... + +--- +# GCN-LABEL: name: diffoporder_add_flat_atomic_add_rtn +# GFX9: FLAT_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 1000, 0, +# GFX9: FLAT_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 0, 0, + +# GFX8: FLAT_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 0, 0, +# GFX8: FLAT_ATOMIC_ADD_RTN %{{[0-9]+}}, %0.sub0, 0, 0, + +name: diffoporder_add_flat_atomic_add_rtn +body: | + bb.0.entry: + + %0:vreg_64 = COPY $vgpr0_vgpr1 + + %1:sgpr_32 = S_MOV_B32 4000 + %2:vgpr_32, %3:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %1, 0, implicit $exec + %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %3, 0, implicit $exec + %6:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %4, %subreg.sub1 + %14:vgpr_32 = FLAT_ATOMIC_ADD_RTN %6:vreg_64, %0.sub0, 0, 0, implicit $exec, implicit $flat_scr + + + %8:sgpr_32 = S_MOV_B32 3000 + %9:vgpr_32, %10:sreg_64_xexec = V_ADD_CO_U32_e64 %0.sub0, %8, 0, implicit $exec + %11:vgpr_32, dead %12:sreg_64_xexec = V_ADDC_U32_e64 %0.sub1, 0, %10, 0, implicit $exec + %13:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1 + %15:vgpr_32 = FLAT_ATOMIC_ADD_RTN %13:vreg_64, %0.sub0, 0, 0, implicit $exec, implicit $flat_scr +... -- GitLab From 9ad0175ea0099351050e1dfb0074d9938f469404 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 6 Jun 2024 12:13:19 -0700 Subject: [PATCH 148/462] [ELF] Keep non-alloc orphan sections at the end https://reviews.llvm.org/D85867 changed the way we assign file offsets (alloc sections first, then non-alloc sections). It also removed a non-alloc special case from `findOrphanPos`. Looking at the memory-nonalloc-no-warn.test change, which would be needed by #93761, it makes sense to restore the previous behavior: when placing non-alloc orphan sections, keep these sections at the end so that the section index order matches the file offset order. This change is cosmetic. In sections-nonalloc.s, GNU ld places the orphan `other3` in the middle and the orphan .symtab/.shstrtab/.strtab at the end. Pull Request: https://github.com/llvm/llvm-project/pull/94519 --- lld/ELF/Writer.cpp | 4 +++ .../linkerscript/memory-nonalloc-no-warn.test | 22 +++++++-------- lld/test/ELF/linkerscript/sections-nonalloc.s | 28 +++++++++---------- lld/test/ELF/linkerscript/sections.s | 4 +-- 4 files changed, 31 insertions(+), 27 deletions(-) diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index 0295a656b070..544db201d1ff 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -921,7 +921,11 @@ static bool shouldSkip(SectionCommand *cmd) { static SmallVectorImpl::iterator findOrphanPos(SmallVectorImpl::iterator b, SmallVectorImpl::iterator e) { + // Place non-alloc orphan sections at the end. This matches how we assign file + // offsets to non-alloc sections. OutputSection *sec = &cast(*e)->osec; + if (!(sec->flags & SHF_ALLOC)) + return e; // As a special case, place .relro_padding before the SymbolAssignment using // DATA_SEGMENT_RELRO_END, if present. diff --git a/lld/test/ELF/linkerscript/memory-nonalloc-no-warn.test b/lld/test/ELF/linkerscript/memory-nonalloc-no-warn.test index 2dcd0f8d6ce2..eabdf75fcf93 100644 --- a/lld/test/ELF/linkerscript/memory-nonalloc-no-warn.test +++ b/lld/test/ELF/linkerscript/memory-nonalloc-no-warn.test @@ -16,20 +16,20 @@ ## The output file must include all sections. # RUN: llvm-readelf -S %t/a.elf | FileCheck %s -# CHECK: There are 12 section headers, starting at offset 0x2140: +# CHECK: There are 12 section headers, starting at offset 0x2138: # CHECK: [Nr] Name Type Address Off Size ES Flg Lk Inf Al # CHECK-NEXT: [ 0] NULL 0000000000000000 000000 000000 00 0 0 0 # CHECK-NEXT: [ 1] .nonalloc PROGBITS 0000000000000000 001064 001000 00 W 0 0 1 -# CHECK-NEXT: [ 2] .comment PROGBITS 0000000000000000 {{.*}} {{.*}} 01 MS 0 0 1 -# CHECK-NEXT: [ 3] .symtab SYMTAB 0000000000000000 {{.*}} {{.*}} 18 5 1 8 -# CHECK-NEXT: [ 4] .shstrtab STRTAB 0000000000000000 {{.*}} {{.*}} 00 0 0 1 -# CHECK-NEXT: [ 5] .strtab STRTAB 0000000000000000 {{.*}} {{.*}} 00 0 0 1 -# CHECK-NEXT: [ 6] .dat PROGBITS 0000000000000000 002137 000004 00 W 0 0 1 -# CHECK-NEXT: [ 7] .intvec0_out PROGBITS 0000000000000000 00213b 000000 00 W 0 0 1 -# CHECK-NEXT: [ 8] .intvec1_out PROGBITS 0000000000000000 00213b 000000 00 W 0 0 1 -# CHECK-NEXT: [ 9] .intvec2_out PROGBITS 0000000000000000 00213b 000000 00 W 0 0 1 -# CHECK-NEXT: [10] .intvec3_out PROGBITS 00000000803fe060 001060 000004 00 AX 0 0 1 -# CHECK-NEXT: [11] .text PROGBITS 00000000803fe064 001064 000000 00 AX 0 0 4 +# CHECK-NEXT: [ 2] .dat PROGBITS 0000000000000000 002064 000004 00 W 0 0 1 +# CHECK-NEXT: [ 3] .intvec0_out PROGBITS 0000000000000000 002068 000000 00 W 0 0 1 +# CHECK-NEXT: [ 4] .intvec1_out PROGBITS 0000000000000000 002068 000000 00 W 0 0 1 +# CHECK-NEXT: [ 5] .intvec2_out PROGBITS 0000000000000000 002068 000000 00 W 0 0 1 +# CHECK-NEXT: [ 6] .intvec3_out PROGBITS 00000000803fe060 001060 000004 00 AX 0 0 1 +# CHECK-NEXT: [ 7] .text PROGBITS 00000000803fe064 001064 000000 00 AX 0 0 4 +# CHECK-NEXT: [ 8] .comment PROGBITS 0000000000000000 {{.*}} {{.*}} 01 MS 0 0 1 +# CHECK-NEXT: [ 9] .symtab SYMTAB 0000000000000000 {{.*}} {{.*}} 18 11 1 8 +# CHECK-NEXT: [10] .shstrtab STRTAB 0000000000000000 {{.*}} {{.*}} 00 0 0 1 +# CHECK-NEXT: [11] .strtab STRTAB 0000000000000000 {{.*}} {{.*}} 00 0 0 1 #--- a.s diff --git a/lld/test/ELF/linkerscript/sections-nonalloc.s b/lld/test/ELF/linkerscript/sections-nonalloc.s index 79765d32dfff..b4fab8c24cfb 100644 --- a/lld/test/ELF/linkerscript/sections-nonalloc.s +++ b/lld/test/ELF/linkerscript/sections-nonalloc.s @@ -16,15 +16,15 @@ # CHECK-NEXT: [ 2] data1 PROGBITS 0000000000000001 001001 000001 00 WA 0 # CHECK-NEXT: [ 3] other1 PROGBITS 0000000000000000 001008 000001 00 0 # CHECK-NEXT: [ 4] other2 PROGBITS 0000000000000000 001010 000001 00 0 -## Orphan placement places other3, .symtab, .shstrtab and .strtab after other2. -# CHECK-NEXT: [ 5] other3 PROGBITS 0000000000000000 001020 000001 00 0 -# CHECK-NEXT: [ 6] .symtab SYMTAB 0000000000000000 001028 000030 18 8 -# CHECK-NEXT: [ 7] .shstrtab STRTAB 0000000000000000 001058 00004d 00 0 -# CHECK-NEXT: [ 8] .strtab STRTAB 0000000000000000 0010a5 000008 00 0 -# CHECK-NEXT: [ 9] data2 PROGBITS 0000000000000002 001002 000001 00 WA 0 +# CHECK-NEXT: [ 5] data2 PROGBITS 0000000000000002 001002 000001 00 WA 0 ## max{sortRank(data1),sortRank(data2)} <= sortRank(data3). data3 is placed after the latter. -# CHECK-NEXT: [10] data3 PROGBITS 0000000000000003 001003 000001 00 WA 0 -# CHECK-NEXT: [11] .text PROGBITS 0000000000000004 001004 000001 00 AX 0 +# CHECK-NEXT: [ 6] data3 PROGBITS 0000000000000003 001003 000001 00 WA 0 +# CHECK-NEXT: [ 7] .text PROGBITS 0000000000000004 001004 000001 00 AX 0 +## Non-alloc orphan sections other3, .symtab, .shstrtab and .strtab are placed at the end. +# CHECK-NEXT: [ 8] other3 PROGBITS 0000000000000000 001020 000001 00 0 +# CHECK-NEXT: [ 9] .symtab SYMTAB 0000000000000000 001028 000030 18 11 +# CHECK-NEXT: [10] .shstrtab STRTAB 0000000000000000 001058 00004d 00 0 +# CHECK-NEXT: [11] .strtab STRTAB 0000000000000000 0010a5 000008 00 0 # CHECK: Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align # CHECK-NEXT: LOAD 0x001000 0x0000000000000000 0x0000000000000000 0x000004 0x000004 RW 0x1000 @@ -41,12 +41,12 @@ # CHECK1-NEXT: [ 3] data1 PROGBITS 00000000000000b2 0000b2 000001 00 WA 0 # CHECK1-NEXT: [ 4] other1 PROGBITS 0000000000000000 0000b8 000001 00 0 # CHECK1-NEXT: [ 5] other2 PROGBITS 0000000000000000 0000c0 000001 00 0 -# CHECK1-NEXT: [ 6] other3 PROGBITS 0000000000000000 0000d0 000001 00 0 -# CHECK1-NEXT: [ 7] .symtab SYMTAB 0000000000000000 0000d8 000030 18 9 -# CHECK1-NEXT: [ 8] .shstrtab STRTAB 0000000000000000 000108 00004d 00 0 -# CHECK1-NEXT: [ 9] .strtab STRTAB 0000000000000000 000155 000008 00 0 -# CHECK1-NEXT: [10] data2 PROGBITS 00000000000000b3 0000b3 000001 00 WA 0 -# CHECK1-NEXT: [11] data3 PROGBITS 00000000000000b4 0000b4 000001 00 WA 0 +# CHECK1-NEXT: [ 6] data2 PROGBITS 00000000000000b3 0000b3 000001 00 WA 0 +# CHECK1-NEXT: [ 7] data3 PROGBITS 00000000000000b4 0000b4 000001 00 WA 0 +# CHECK1-NEXT: [ 8] other3 PROGBITS 0000000000000000 0000d0 000001 00 0 +# CHECK1-NEXT: [ 9] .symtab SYMTAB 0000000000000000 0000d8 000030 18 11 +# CHECK1-NEXT: [10] .shstrtab STRTAB 0000000000000000 000108 00004d 00 0 +# CHECK1-NEXT: [11] .strtab STRTAB 0000000000000000 000155 000008 00 0 # CHECK1: Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align # CHECK1-NEXT: LOAD 0x000000 0x0000000000000000 0x0000000000000000 0x0000b5 0x0000b5 RWE 0x1000 # CHECK1-NEXT: 0x60000000 0x0000b8 0x0000000000000000 0x0000000000000000 0x000009 0x000001 0x8 diff --git a/lld/test/ELF/linkerscript/sections.s b/lld/test/ELF/linkerscript/sections.s index 5d6cc1f3bd0d..fc03af8402df 100644 --- a/lld/test/ELF/linkerscript/sections.s +++ b/lld/test/ELF/linkerscript/sections.s @@ -79,8 +79,8 @@ # SEP-BY-NONALLOC: [ 1] .text PROGBITS 0000000000000000 001000 00000e 00 AX # SEP-BY-NONALLOC-NEXT: [ 2] .data PROGBITS 000000000000000e 00100e 000020 00 WA # SEP-BY-NONALLOC-NEXT: [ 3] .comment PROGBITS 0000000000000000 001031 000008 01 MS -# SEP-BY-NONALLOC: [ 7] other PROGBITS 000000000000002e 00102e 000003 00 WA -# SEP-BY-NONALLOC-NEXT: [ 8] .bss NOBITS 0000000000000031 001031 000002 00 WA +# SEP-BY-NONALLOC: [ 4] other PROGBITS 000000000000002e 00102e 000003 00 WA +# SEP-BY-NONALLOC-NEXT: [ 5] .bss NOBITS 0000000000000031 001031 000002 00 WA # SEP-BY-NONALLOC: Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align # SEP-BY-NONALLOC-NEXT: LOAD 0x001000 0x0000000000000000 0x0000000000000000 0x00000e 0x00000e R E 0x1000 -- GitLab From 5d0308f3930887a1610b6965b6c8cec10733cdb2 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Thu, 6 Jun 2024 21:14:38 +0200 Subject: [PATCH 149/462] [bazel] Port for 649edb8eb25e82e3ac6fce4788f51759636229ec --- .../llvm-project-overlay/libc/test/src/__support/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel index 3980ef60c197..c8001fe1e581 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel @@ -99,6 +99,7 @@ libc_test( name = "fixedvector_test", srcs = ["fixedvector_test.cpp"], deps = [ + "//libc:__support_cpp_array", "//libc:__support_fixedvector", ], ) -- GitLab From d09231a422f052d6f6f44913fad610728a7c266b Mon Sep 17 00:00:00 2001 From: Med Ismail Bennani Date: Thu, 6 Jun 2024 12:17:00 -0700 Subject: [PATCH 150/462] [lldb/crashlog] Remove aarch64 requirement on crashlog tests (#94553) This PR removes the `target-aarch64` requirement on the crashlog tests to exercice them on Intel bots and make image loading single-threaded temporarily while implementing a fix for a deadlock issue when loading the images in parallel. Signed-off-by: Med Ismail Bennani --- lldb/examples/python/crashlog.py | 3 +++ .../Python/Crashlog/app_specific_backtrace_crashlog.test | 2 +- .../Python/Crashlog/interactive_crashlog_invalid_target.test | 2 +- .../Python/Crashlog/interactive_crashlog_json.test | 2 +- .../Python/Crashlog/interactive_crashlog_legacy.test | 2 +- .../Python/Crashlog/last_exception_backtrace_crashlog.test | 2 +- .../test/Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg | 3 +++ .../Python/Crashlog/skipped_status_interactive_crashlog.test | 2 +- 8 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lldb/examples/python/crashlog.py b/lldb/examples/python/crashlog.py index 5281d6d949ba..1c0d717ce455 100755 --- a/lldb/examples/python/crashlog.py +++ b/lldb/examples/python/crashlog.py @@ -1814,6 +1814,9 @@ def SymbolicateCrashLogs(debugger, command_args, result, is_command): ) ) + if "NO_PARALLEL_IMG_LOADING" in os.environ: + options.no_parallel_image_loading = True + if options.version: print(debugger.GetVersionString()) return diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/app_specific_backtrace_crashlog.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/app_specific_backtrace_crashlog.test index 430febb09625..9c0510c34cca 100644 --- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/app_specific_backtrace_crashlog.test +++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/app_specific_backtrace_crashlog.test @@ -1,4 +1,4 @@ -# REQUIRES: python, native && target-aarch64 && system-darwin +# REQUIRES: python, native && system-darwin # RUN: mkdir -p %t.dir # RUN: yaml2obj %S/Inputs/application_specific_info/asi.yaml > %t.dir/asi diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_invalid_target.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_invalid_target.test index abd1e7c3da53..eb1f5f456a2d 100644 --- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_invalid_target.test +++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_invalid_target.test @@ -1,4 +1,4 @@ -# REQUIRES: python, native && target-aarch64 && system-darwin +# REQUIRES: python, native && system-darwin # RUN: %lldb -o 'command script import lldb.macosx.crashlog' \ # RUN: -o 'crashlog -V' \ diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_json.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_json.test index fccd71ce31f7..684be2846f78 100644 --- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_json.test +++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_json.test @@ -1,4 +1,4 @@ -# REQUIRES: python, native && target-aarch64 && system-darwin +# REQUIRES: python, native && system-darwin # RUN: mkdir -p %t.dir # RUN: yaml2obj %S/Inputs/interactive_crashlog/multithread-test.yaml > %t.dir/multithread-test diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_legacy.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_legacy.test index 6e2826e88aed..271a4c2aa90f 100644 --- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_legacy.test +++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_legacy.test @@ -1,4 +1,4 @@ -# REQUIRES: python, native && target-aarch64 && system-darwin +# REQUIRES: python, native && system-darwin # RUN: mkdir -p %t.dir # RUN: yaml2obj %S/Inputs/interactive_crashlog/multithread-test.yaml > %t.dir/multithread-test diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/last_exception_backtrace_crashlog.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/last_exception_backtrace_crashlog.test index fa857ac0e84f..a17b7ac18a62 100644 --- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/last_exception_backtrace_crashlog.test +++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/last_exception_backtrace_crashlog.test @@ -1,4 +1,4 @@ -# REQUIRES: python, native && target-aarch64 && system-darwin +# REQUIRES: python, native && system-darwin # RUN: mkdir -p %t.dir # RUN: yaml2obj %S/Inputs/application_specific_info/asi.yaml > %t.dir/asi diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg index 3da9265b3553..b72b29419893 100644 --- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg +++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/lit.local.cfg @@ -5,3 +5,6 @@ if 'lldb-repro' in config.available_features: config.unsupported = True config.environment["LLDB_APPLE_DSYMFORUUID_EXECUTABLE"] = "" + +# Temporary parallel image loading deadlock workaround +config.environment["NO_PARALLEL_IMG_LOADING"] = "" diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/skipped_status_interactive_crashlog.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/skipped_status_interactive_crashlog.test index 81e06868eaee..64cd0904371a 100644 --- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/skipped_status_interactive_crashlog.test +++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/skipped_status_interactive_crashlog.test @@ -1,4 +1,4 @@ -# REQUIRES: python, native && target-aarch64 && system-darwin +# REQUIRES: python, native && system-darwin # RUN: mkdir -p %t.dir # RUN: yaml2obj %S/Inputs/interactive_crashlog/multithread-test.yaml > %t.dir/multithread-test -- GitLab From 2cc16442998786f7716ee7093c4f29f1b9b07cd5 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Thu, 6 Jun 2024 14:36:34 -0500 Subject: [PATCH 151/462] [Offload] Fix missing `abs` function for test Summary: We don't have the abs function to link against, just use the builtin. --- offload/test/offloading/ompx_bare_shfl_down_sync.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offload/test/offloading/ompx_bare_shfl_down_sync.cpp b/offload/test/offloading/ompx_bare_shfl_down_sync.cpp index d2569a5b0266..c9246894b089 100644 --- a/offload/test/offloading/ompx_bare_shfl_down_sync.cpp +++ b/offload/test/offloading/ompx_bare_shfl_down_sync.cpp @@ -23,7 +23,7 @@ bool equal(T LHS, T RHS) { template ::value, bool> = true> bool equal(T LHS, T RHS) { - return std::abs(LHS - RHS) < std::numeric_limits::epsilon(); + return __builtin_fabs(LHS - RHS) < std::numeric_limits::epsilon(); } template void test() { -- GitLab From 878deaeeadb940c892908cb51356b60000626db8 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Thu, 6 Jun 2024 20:50:46 +0100 Subject: [PATCH 152/462] [LLVM] Do not require shell for some tests (#94595) Remove `REQUIRES: shell` from some tests that seem fine without it. Tested on Windows and with LIT_USE_INTERNAL_SHELL=1 on Linux. --- llvm/test/DebugInfo/symbolize-gnu-debuglink-no-realpath.test | 1 - llvm/test/Other/can-execute.txt | 1 - llvm/test/Other/lit-unicode.txt | 1 - llvm/test/tools/llvm-cov/gcov/intermediate-format.test | 2 -- llvm/test/tools/llvm-rc/windres-prefix.test | 2 -- llvm/test/tools/split-file/output-is-special.test | 1 - 6 files changed, 8 deletions(-) diff --git a/llvm/test/DebugInfo/symbolize-gnu-debuglink-no-realpath.test b/llvm/test/DebugInfo/symbolize-gnu-debuglink-no-realpath.test index 5141ff6ce322..9e46570783c9 100644 --- a/llvm/test/DebugInfo/symbolize-gnu-debuglink-no-realpath.test +++ b/llvm/test/DebugInfo/symbolize-gnu-debuglink-no-realpath.test @@ -1,4 +1,3 @@ -# REQUIRES: shell # Ensure that no realpath assumptions are made about .gnu_debuglink paths. # Copy inputs to some other location with arbitrary names, with the original diff --git a/llvm/test/Other/can-execute.txt b/llvm/test/Other/can-execute.txt index 46791cb892a2..37626e7357b2 100644 --- a/llvm/test/Other/can-execute.txt +++ b/llvm/test/Other/can-execute.txt @@ -1,5 +1,4 @@ REQUIRES: can-execute -REQUIRES: shell This tests that we abstract two peculiarities of unix in can_execute: diff --git a/llvm/test/Other/lit-unicode.txt b/llvm/test/Other/lit-unicode.txt index 2f4000145168..b375fc505b73 100644 --- a/llvm/test/Other/lit-unicode.txt +++ b/llvm/test/Other/lit-unicode.txt @@ -1,5 +1,4 @@ FIXME: See if we can fix this in lit by using Unicode strings. -REQUIRES: shell RUN: echo "ようこそ" | FileCheck %s CHECK: {{^}}ようこそ{{$}} diff --git a/llvm/test/tools/llvm-cov/gcov/intermediate-format.test b/llvm/test/tools/llvm-cov/gcov/intermediate-format.test index 583e670c2d3f..a3f4695c3204 100644 --- a/llvm/test/tools/llvm-cov/gcov/intermediate-format.test +++ b/llvm/test/tools/llvm-cov/gcov/intermediate-format.test @@ -1,5 +1,3 @@ -REQUIRES: shell - RUN: rm -rf %t && mkdir %t && cd %t RUN: cp %S/Inputs/test.gcno %S/Inputs/test.gcda . diff --git a/llvm/test/tools/llvm-rc/windres-prefix.test b/llvm/test/tools/llvm-rc/windres-prefix.test index 4c53fdfc3db6..7dda51d06352 100644 --- a/llvm/test/tools/llvm-rc/windres-prefix.test +++ b/llvm/test/tools/llvm-rc/windres-prefix.test @@ -1,5 +1,3 @@ -; REQUIRES: shell - ; RUN: rm -rf %t && mkdir %t ; Check that a triple prefix on the executable gets picked up as target triple. diff --git a/llvm/test/tools/split-file/output-is-special.test b/llvm/test/tools/split-file/output-is-special.test index 98bb4d36a4ff..0b1e0f786c4d 100644 --- a/llvm/test/tools/split-file/output-is-special.test +++ b/llvm/test/tools/split-file/output-is-special.test @@ -1,5 +1,4 @@ # UNSUPPORTED: system-windows -# REQUIRES: shell ## Don't delete the output if it is special, otherwise root may accidentally ## remove important special files. -- GitLab From f543dfd1d70d2e5d9a746dd77450304c47cd7c15 Mon Sep 17 00:00:00 2001 From: Christopher Bate Date: Thu, 6 Jun 2024 14:11:52 -0600 Subject: [PATCH 153/462] NFC: resolve TODO in LLVM dialect conversions (#91497) Relaxes restriction that certain public utility functions only apply to the builtin ModuleOp. --- .../mlir/Dialect/LLVMIR/FunctionCallUtils.h | 38 +++++++-------- .../MemRefToLLVM/AllocLikeConversion.cpp | 16 +++---- .../Dialect/LLVMIR/IR/FunctionCallUtils.cpp | 46 ++++++++++--------- 3 files changed, 51 insertions(+), 49 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h b/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h index 123ce36cb0a7..852490cf7428 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h +++ b/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h @@ -33,36 +33,36 @@ class LLVMFuncOp; /// external C function calls. The list of functions provided here must be /// implemented separately (e.g. as part of a support runtime library or as part /// of the libc). -LLVM::LLVMFuncOp lookupOrCreatePrintI64Fn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreatePrintU64Fn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreatePrintF16Fn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreatePrintBF16Fn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreatePrintF32Fn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreatePrintF64Fn(ModuleOp moduleOp); +LLVM::LLVMFuncOp lookupOrCreatePrintI64Fn(Operation *moduleOp); +LLVM::LLVMFuncOp lookupOrCreatePrintU64Fn(Operation *moduleOp); +LLVM::LLVMFuncOp lookupOrCreatePrintF16Fn(Operation *moduleOp); +LLVM::LLVMFuncOp lookupOrCreatePrintBF16Fn(Operation *moduleOp); +LLVM::LLVMFuncOp lookupOrCreatePrintF32Fn(Operation *moduleOp); +LLVM::LLVMFuncOp lookupOrCreatePrintF64Fn(Operation *moduleOp); /// Declares a function to print a C-string. /// If a custom runtime function is defined via `runtimeFunctionName`, it must /// have the signature void(char const*). The default function is `printString`. LLVM::LLVMFuncOp -lookupOrCreatePrintStringFn(ModuleOp moduleOp, +lookupOrCreatePrintStringFn(Operation *moduleOp, std::optional runtimeFunctionName = {}); -LLVM::LLVMFuncOp lookupOrCreatePrintOpenFn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreatePrintCloseFn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreatePrintCommaFn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreatePrintNewlineFn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreateMallocFn(ModuleOp moduleOp, Type indexType); -LLVM::LLVMFuncOp lookupOrCreateAlignedAllocFn(ModuleOp moduleOp, +LLVM::LLVMFuncOp lookupOrCreatePrintOpenFn(Operation *moduleOp); +LLVM::LLVMFuncOp lookupOrCreatePrintCloseFn(Operation *moduleOp); +LLVM::LLVMFuncOp lookupOrCreatePrintCommaFn(Operation *moduleOp); +LLVM::LLVMFuncOp lookupOrCreatePrintNewlineFn(Operation *moduleOp); +LLVM::LLVMFuncOp lookupOrCreateMallocFn(Operation *moduleOp, Type indexType); +LLVM::LLVMFuncOp lookupOrCreateAlignedAllocFn(Operation *moduleOp, Type indexType); -LLVM::LLVMFuncOp lookupOrCreateFreeFn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreateGenericAllocFn(ModuleOp moduleOp, +LLVM::LLVMFuncOp lookupOrCreateFreeFn(Operation *moduleOp); +LLVM::LLVMFuncOp lookupOrCreateGenericAllocFn(Operation *moduleOp, Type indexType); -LLVM::LLVMFuncOp lookupOrCreateGenericAlignedAllocFn(ModuleOp moduleOp, +LLVM::LLVMFuncOp lookupOrCreateGenericAlignedAllocFn(Operation *moduleOp, Type indexType); -LLVM::LLVMFuncOp lookupOrCreateGenericFreeFn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreateMemRefCopyFn(ModuleOp moduleOp, Type indexType, +LLVM::LLVMFuncOp lookupOrCreateGenericFreeFn(Operation *moduleOp); +LLVM::LLVMFuncOp lookupOrCreateMemRefCopyFn(Operation *moduleOp, Type indexType, Type unrankedDescriptorType); /// Create a FuncOp with signature `resultType`(`paramTypes`)` and name `name`. -LLVM::LLVMFuncOp lookupOrCreateFn(ModuleOp moduleOp, StringRef name, +LLVM::LLVMFuncOp lookupOrCreateFn(Operation *moduleOp, StringRef name, ArrayRef paramTypes = {}, Type resultType = {}, bool isVarArg = false); diff --git a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp index b29abc94ce40..e48ca5180b70 100644 --- a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp +++ b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp @@ -10,18 +10,14 @@ #include "mlir/Analysis/DataLayoutAnalysis.h" #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/IR/SymbolTable.h" using namespace mlir; namespace { -// TODO: Fix the LLVM utilities for looking up functions to take Operation* -// with SymbolTable trait instead of ModuleOp and make similar change here. This -// allows call sites to use getParentWithTrait instead -// of getParentOfType to pass down the operation. LLVM::LLVMFuncOp getNotalignedAllocFn(const LLVMTypeConverter *typeConverter, - ModuleOp module, Type indexType) { + Operation *module, Type indexType) { bool useGenericFn = typeConverter->getOptions().useGenericFunctions; - if (useGenericFn) return LLVM::lookupOrCreateGenericAllocFn(module, indexType); @@ -29,7 +25,7 @@ LLVM::LLVMFuncOp getNotalignedAllocFn(const LLVMTypeConverter *typeConverter, } LLVM::LLVMFuncOp getAlignedAllocFn(const LLVMTypeConverter *typeConverter, - ModuleOp module, Type indexType) { + Operation *module, Type indexType) { bool useGenericFn = typeConverter->getOptions().useGenericFunctions; if (useGenericFn) @@ -79,7 +75,8 @@ std::tuple AllocationOpLLVMLowering::allocateBufferManuallyAlign( // Allocate the underlying buffer. Type elementPtrType = this->getElementPtrType(memRefType); LLVM::LLVMFuncOp allocFuncOp = getNotalignedAllocFn( - getTypeConverter(), op->getParentOfType(), getIndexType()); + getTypeConverter(), op->getParentWithTrait(), + getIndexType()); auto results = rewriter.create(loc, allocFuncOp, sizeBytes); Value allocatedPtr = @@ -144,7 +141,8 @@ Value AllocationOpLLVMLowering::allocateBufferAutoAlign( Type elementPtrType = this->getElementPtrType(memRefType); LLVM::LLVMFuncOp allocFuncOp = getAlignedAllocFn( - getTypeConverter(), op->getParentOfType(), getIndexType()); + getTypeConverter(), op->getParentWithTrait(), + getIndexType()); auto results = rewriter.create( loc, allocFuncOp, ValueRange({allocAlignment, sizeBytes})); diff --git a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp index 0004c2e3403e..88421a16ccf9 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp @@ -45,49 +45,53 @@ static constexpr llvm::StringRef kGenericFree = "_mlir_memref_to_llvm_free"; static constexpr llvm::StringRef kMemRefCopy = "memrefCopy"; /// Generic print function lookupOrCreate helper. -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFn(ModuleOp moduleOp, StringRef name, +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFn(Operation *moduleOp, + StringRef name, ArrayRef paramTypes, Type resultType, bool isVarArg) { - auto func = moduleOp.lookupSymbol(name); + assert(moduleOp->hasTrait() && + "expected SymbolTable operation"); + auto func = llvm::dyn_cast_or_null( + SymbolTable::lookupSymbolIn(moduleOp, name)); if (func) return func; - OpBuilder b(moduleOp.getBodyRegion()); + OpBuilder b(moduleOp->getRegion(0)); return b.create( moduleOp->getLoc(), name, LLVM::LLVMFunctionType::get(resultType, paramTypes, isVarArg)); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintI64Fn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintI64Fn(Operation *moduleOp) { return lookupOrCreateFn(moduleOp, kPrintI64, IntegerType::get(moduleOp->getContext(), 64), LLVM::LLVMVoidType::get(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintU64Fn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintU64Fn(Operation *moduleOp) { return lookupOrCreateFn(moduleOp, kPrintU64, IntegerType::get(moduleOp->getContext(), 64), LLVM::LLVMVoidType::get(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF16Fn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF16Fn(Operation *moduleOp) { return lookupOrCreateFn(moduleOp, kPrintF16, IntegerType::get(moduleOp->getContext(), 16), // bits! LLVM::LLVMVoidType::get(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintBF16Fn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintBF16Fn(Operation *moduleOp) { return lookupOrCreateFn(moduleOp, kPrintBF16, IntegerType::get(moduleOp->getContext(), 16), // bits! LLVM::LLVMVoidType::get(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF32Fn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF32Fn(Operation *moduleOp) { return lookupOrCreateFn(moduleOp, kPrintF32, Float32Type::get(moduleOp->getContext()), LLVM::LLVMVoidType::get(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF64Fn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF64Fn(Operation *moduleOp) { return lookupOrCreateFn(moduleOp, kPrintF64, Float64Type::get(moduleOp->getContext()), LLVM::LLVMVoidType::get(moduleOp->getContext())); @@ -103,72 +107,72 @@ static LLVM::LLVMPointerType getVoidPtr(MLIRContext *context) { } LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintStringFn( - ModuleOp moduleOp, std::optional runtimeFunctionName) { + Operation *moduleOp, std::optional runtimeFunctionName) { return lookupOrCreateFn(moduleOp, runtimeFunctionName.value_or(kPrintString), getCharPtr(moduleOp->getContext()), LLVM::LLVMVoidType::get(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintOpenFn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintOpenFn(Operation *moduleOp) { return lookupOrCreateFn(moduleOp, kPrintOpen, {}, LLVM::LLVMVoidType::get(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintCloseFn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintCloseFn(Operation *moduleOp) { return lookupOrCreateFn(moduleOp, kPrintClose, {}, LLVM::LLVMVoidType::get(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintCommaFn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintCommaFn(Operation *moduleOp) { return lookupOrCreateFn(moduleOp, kPrintComma, {}, LLVM::LLVMVoidType::get(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintNewlineFn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintNewlineFn(Operation *moduleOp) { return lookupOrCreateFn(moduleOp, kPrintNewline, {}, LLVM::LLVMVoidType::get(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateMallocFn(ModuleOp moduleOp, +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateMallocFn(Operation *moduleOp, Type indexType) { return LLVM::lookupOrCreateFn(moduleOp, kMalloc, indexType, getVoidPtr(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateAlignedAllocFn(ModuleOp moduleOp, +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateAlignedAllocFn(Operation *moduleOp, Type indexType) { return LLVM::lookupOrCreateFn(moduleOp, kAlignedAlloc, {indexType, indexType}, getVoidPtr(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFreeFn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFreeFn(Operation *moduleOp) { return LLVM::lookupOrCreateFn( moduleOp, kFree, getVoidPtr(moduleOp->getContext()), LLVM::LLVMVoidType::get(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericAllocFn(ModuleOp moduleOp, +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericAllocFn(Operation *moduleOp, Type indexType) { return LLVM::lookupOrCreateFn(moduleOp, kGenericAlloc, indexType, getVoidPtr(moduleOp->getContext())); } LLVM::LLVMFuncOp -mlir::LLVM::lookupOrCreateGenericAlignedAllocFn(ModuleOp moduleOp, +mlir::LLVM::lookupOrCreateGenericAlignedAllocFn(Operation *moduleOp, Type indexType) { return LLVM::lookupOrCreateFn(moduleOp, kGenericAlignedAlloc, {indexType, indexType}, getVoidPtr(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericFreeFn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericFreeFn(Operation *moduleOp) { return LLVM::lookupOrCreateFn( moduleOp, kGenericFree, getVoidPtr(moduleOp->getContext()), LLVM::LLVMVoidType::get(moduleOp->getContext())); } LLVM::LLVMFuncOp -mlir::LLVM::lookupOrCreateMemRefCopyFn(ModuleOp moduleOp, Type indexType, +mlir::LLVM::lookupOrCreateMemRefCopyFn(Operation *moduleOp, Type indexType, Type unrankedDescriptorType) { return LLVM::lookupOrCreateFn( moduleOp, kMemRefCopy, -- GitLab From 9293fc7981526eaca0a28012f2e5963fff1b830b Mon Sep 17 00:00:00 2001 From: Alex Langford Date: Thu, 6 Jun 2024 13:18:06 -0700 Subject: [PATCH 154/462] [lldb] Include memory stats in statistics summary (#94671) The summary already includes other size information, e.g. total debug info size in bytes. The only other way I can get this information is by dumping all statistics which can be quite large. Adding it to the summary seems fair. --- lldb/source/Target/Statistics.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lldb/source/Target/Statistics.cpp b/lldb/source/Target/Statistics.cpp index be0848573f81..2a5300012511 100644 --- a/lldb/source/Target/Statistics.cpp +++ b/lldb/source/Target/Statistics.cpp @@ -355,14 +355,14 @@ llvm::json::Value DebuggerStats::ReportStatistics( } global_stats.try_emplace("targets", std::move(json_targets)); + ConstStringStats const_string_stats; + json::Object json_memory{ + {"strings", const_string_stats.ToJSON()}, + }; + global_stats.try_emplace("memory", std::move(json_memory)); if (!summary_only) { - ConstStringStats const_string_stats; - json::Object json_memory{ - {"strings", const_string_stats.ToJSON()}, - }; json::Value cmd_stats = debugger.GetCommandInterpreter().GetStatistics(); global_stats.try_emplace("modules", std::move(json_modules)); - global_stats.try_emplace("memory", std::move(json_memory)); global_stats.try_emplace("commands", std::move(cmd_stats)); } -- GitLab From 9e209a4a3728a599c434bfed4fa37fd8b5907e89 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Thu, 6 Jun 2024 15:19:55 -0500 Subject: [PATCH 155/462] [Offload] Use the kernel argument size directly in AMDGPU offloading (#94667) Summary: The old COV3 implementation of HSA used to omit the implicit arguments from the kernel argument size. For COV4 and COV5 this is no longer the case so we can simply use the size reported from the symbol information. See https://github.com/ROCm/ROCR-Runtime/issues/117#issuecomment-812758161 --- offload/plugins-nextgen/amdgpu/src/rtl.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp index f088d5d1685d..663cfdc5fdf0 100644 --- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp +++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp @@ -3272,19 +3272,13 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice, if (ArgsSize < KernelArgsSize) return Plugin::error("Mismatch of kernel arguments size"); - // The args size reported by HSA may or may not contain the implicit args. - // For now, assume that HSA does not consider the implicit arguments when - // reporting the arguments of a kernel. In the worst case, we can waste - // 56 bytes per allocation. - uint32_t AllArgsSize = KernelArgsSize + ImplicitArgsSize; - AMDGPUPluginTy &AMDGPUPlugin = static_cast(GenericDevice.Plugin); AMDHostDeviceTy &HostDevice = AMDGPUPlugin.getHostDevice(); AMDGPUMemoryManagerTy &ArgsMemoryManager = HostDevice.getArgsMemoryManager(); void *AllArgs = nullptr; - if (auto Err = ArgsMemoryManager.allocate(AllArgsSize, &AllArgs)) + if (auto Err = ArgsMemoryManager.allocate(ArgsSize, &AllArgs)) return Err; // Account for user requested dynamic shared memory. -- GitLab From 06f03b806a347619f0251220baff56209abe9711 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Thu, 6 Jun 2024 13:19:12 -0700 Subject: [PATCH 156/462] [RISCV][InsertVSETVLI] Check for undef register operand directly [nfc] getVNInfoFromReg is expected to return a nullptr if-and-only-if the operand is undef. (This was asserted for.) Reverse the order of the checks to simplify an upcoming set of patches. --- llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp index a96768240a93..82358cdd45ed 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -932,11 +932,11 @@ RISCVInsertVSETVLI::getInfoForVSETVLI(const MachineInstr &MI) const { "Can't handle X0, X0 vsetvli yet"); if (AVLReg == RISCV::X0) NewInfo.setAVLVLMAX(); - else if (VNInfo *VNI = getVNInfoFromReg(AVLReg, MI, LIS)) - NewInfo.setAVLRegDef(VNI, AVLReg); - else { - assert(MI.getOperand(1).isUndef()); + else if (MI.getOperand(1).isUndef()) NewInfo.setAVLIgnored(); + else { + VNInfo *VNI = getVNInfoFromReg(AVLReg, MI, LIS); + NewInfo.setAVLRegDef(VNI, AVLReg); } } NewInfo.setVTYPE(MI.getOperand(2).getImm()); @@ -1008,11 +1008,11 @@ RISCVInsertVSETVLI::computeInfoForInstr(const MachineInstr &MI) const { } else InstrInfo.setAVLImm(Imm); - } else if (VNInfo *VNI = getVNInfoFromReg(VLOp.getReg(), MI, LIS)) { - InstrInfo.setAVLRegDef(VNI, VLOp.getReg()); - } else { - assert(VLOp.isUndef()); + } else if (VLOp.isUndef()) { InstrInfo.setAVLIgnored(); + } else { + VNInfo *VNI = getVNInfoFromReg(VLOp.getReg(), MI, LIS); + InstrInfo.setAVLRegDef(VNI, VLOp.getReg()); } } else { assert(isScalarExtractInstr(MI)); -- GitLab From fbcb92ca017ee7fbf84be808701133fbdf3b1c59 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 6 Jun 2024 13:22:47 -0700 Subject: [PATCH 157/462] [ELF] Test non-alloc orphan that does not the RF_NOT_ADDR_SET rank flag --- lld/test/ELF/linkerscript/sections-nonalloc.s | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lld/test/ELF/linkerscript/sections-nonalloc.s b/lld/test/ELF/linkerscript/sections-nonalloc.s index b4fab8c24cfb..d66e524248ce 100644 --- a/lld/test/ELF/linkerscript/sections-nonalloc.s +++ b/lld/test/ELF/linkerscript/sections-nonalloc.s @@ -34,6 +34,11 @@ # RUN: ld.lld -T %t/b.lds %t.o -o %tb # RUN: llvm-readelf -S -l %tb | FileCheck %s --check-prefix=CHECK1 +## --section-start causes the orphan other3 to be considered before .data3. +## The non-alloc other3 does not disable the placement of .data3. +# RUN: ld.lld -T %t/b.lds %t.o -o %tb --section-start=other3=0 +# RUN: llvm-readelf -S -l %tb | FileCheck %s --check-prefix=CHECK1 + # CHECK1: [Nr] Name Type Address Off Size ES Flg Lk # CHECK1-NEXT: [ 0] NULL 0000000000000000 000000 000000 00 0 # CHECK1-NEXT: [ 1] .text PROGBITS 00000000000000b0 0000b0 000001 00 AX 0 -- GitLab From 7476c20c481cbccbdb89139fb94620e083015932 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 6 Jun 2024 13:25:52 -0700 Subject: [PATCH 158/462] [ProfileData] Remove swapToHostOrder (#94665) This patch removes swapToHostOrder in favor of llvm::support::endian::readNext as swapToHostOrder is too thin a wrapper around readNext. Note that there are two variants of readNext: - readNext(ptr) - readNext(ptr, endian) swapToHostOrder uses the former, but this patch switches to the latter. While we are at it, this patch teaches readNext to default to unaligned just as I did in: commit 568368a43e5b4adb3c5d105a0eff3e0c13c0af8c Author: Kazu Hirata Date: Mon Apr 15 19:05:30 2024 -0700 --- llvm/include/llvm/Support/Endian.h | 3 ++- llvm/lib/ProfileData/InstrProf.cpp | 13 ++----------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/llvm/include/llvm/Support/Endian.h b/llvm/include/llvm/Support/Endian.h index 30e0852b972c..5831fe66a1f7 100644 --- a/llvm/include/llvm/Support/Endian.h +++ b/llvm/include/llvm/Support/Endian.h @@ -72,7 +72,8 @@ template /// Read a value of a particular endianness from a buffer, and increment the /// buffer past that value. -template +template [[nodiscard]] inline value_type readNext(const CharT *&memory, endianness endian) { value_type ret = read(memory, endian); diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index ad63086430bf..6a8f25d4d3bf 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -1176,16 +1176,6 @@ void ValueProfData::deserializeTo(InstrProfRecord &Record, } } -template -static T swapToHostOrder(const unsigned char *&D, llvm::endianness Orig) { - using namespace support; - - if (Orig == llvm::endianness::little) - return endian::readNext(D); - else - return endian::readNext(D); -} - static std::unique_ptr allocValueProfData(uint32_t TotalSize) { return std::unique_ptr(new (::operator new(TotalSize)) ValueProfData()); @@ -1224,7 +1214,8 @@ ValueProfData::getValueProfData(const unsigned char *D, return make_error(instrprof_error::truncated); const unsigned char *Header = D; - uint32_t TotalSize = swapToHostOrder(Header, Endianness); + uint32_t TotalSize = endian::readNext(Header, Endianness); + if (D + TotalSize > BufferEnd) return make_error(instrprof_error::too_large); -- GitLab From 12ccc245f195695c8bac9156c75e5b70044882fa Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 6 Jun 2024 14:22:12 -0700 Subject: [PATCH 159/462] [clang] Fix flag typo in comment Fixed for more accurate searches of the flag `-Wsystem-headers-in-module=`. --- clang/include/clang/Basic/DiagnosticOptions.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/include/clang/Basic/DiagnosticOptions.h b/clang/include/clang/Basic/DiagnosticOptions.h index 099982c3bdd5..30141c2b8f44 100644 --- a/clang/include/clang/Basic/DiagnosticOptions.h +++ b/clang/include/clang/Basic/DiagnosticOptions.h @@ -124,7 +124,7 @@ public: /// default). std::vector VerifyPrefixes; - /// The list of -Wsystem-header-in-module=... options used to override + /// The list of -Wsystem-headers-in-module=... options used to override /// whether -Wsystem-headers is enabled on a per-module basis. std::vector SystemHeaderWarningsModules; -- GitLab From 4a918f071005a2d7aba82e031f3f745270bf67da Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 6 Jun 2024 14:24:43 -0700 Subject: [PATCH 160/462] [memprof] Use std::vector instead of llvm::SmallVector (NFC) (#94432) This patch replaces llvm::SmallVector with std::vector. llvm::SmallVector sets aside one inline element. Meanwhile, when I sort all call stacks by their lengths, the length at the first percentile is already 2. That is, 99 percent of call stacks do not take advantage of the inline element. Using std::vector reduces the cycle and instruction counts by 11% and 22%, respectively, with "llvm-profdata show" modified to deserialize all MemProfRecords. --- llvm/include/llvm/ProfileData/MemProf.h | 19 +++++++++---------- llvm/lib/ProfileData/MemProf.cpp | 3 +-- .../Instrumentation/MemProfiler.cpp | 4 ++-- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h index 528abe1ad3d4..9f6f9955260d 100644 --- a/llvm/include/llvm/ProfileData/MemProf.h +++ b/llvm/include/llvm/ProfileData/MemProf.h @@ -368,7 +368,7 @@ struct IndexedAllocationInfo { // be used for temporary in-memory instances. struct AllocationInfo { // Same as IndexedAllocationInfo::CallStack with the frame contents inline. - llvm::SmallVector CallStack; + std::vector CallStack; // Same as IndexedAllocationInfo::Info; PortableMemInfoBlock Info; @@ -450,8 +450,7 @@ struct IndexedMemProfRecord { // Convert IndexedMemProfRecord to MemProfRecord. Callback is used to // translate CallStackId to call stacks with frames inline. MemProfRecord toMemProfRecord( - llvm::function_ref(const CallStackId)> Callback) - const; + llvm::function_ref(const CallStackId)> Callback) const; // Returns the GUID for the function name after canonicalization. For // memprof, we remove any .llvm suffix added by LTO. MemProfRecords are @@ -466,7 +465,7 @@ struct MemProfRecord { // Same as IndexedMemProfRecord::AllocSites with frame contents inline. llvm::SmallVector AllocSites; // Same as IndexedMemProfRecord::CallSites with frame contents inline. - llvm::SmallVector> CallSites; + llvm::SmallVector> CallSites; MemProfRecord() = default; MemProfRecord( @@ -476,7 +475,7 @@ struct MemProfRecord { AllocSites.emplace_back(IndexedAI, IdToFrameCallback); } for (const ArrayRef Site : Record.CallSites) { - llvm::SmallVector Frames; + std::vector Frames; for (const FrameId Id : Site) { Frames.push_back(IdToFrameCallback(Id)); } @@ -494,7 +493,7 @@ struct MemProfRecord { if (!CallSites.empty()) { OS << " CallSites:\n"; - for (const llvm::SmallVector &Frames : CallSites) { + for (const std::vector &Frames : CallSites) { for (const Frame &F : Frames) { OS << " -\n"; F.printYAML(OS); @@ -848,8 +847,8 @@ template struct CallStackIdConverter { CallStackIdConverter(const CallStackIdConverter &) = delete; CallStackIdConverter &operator=(const CallStackIdConverter &) = delete; - llvm::SmallVector operator()(CallStackId CSId) { - llvm::SmallVector Frames; + std::vector operator()(CallStackId CSId) { + std::vector Frames; auto CSIter = Map.find(CSId); if (CSIter == Map.end()) { LastUnmappedId = CSId; @@ -890,8 +889,8 @@ struct LinearCallStackIdConverter { std::function FrameIdToFrame) : CallStackBase(CallStackBase), FrameIdToFrame(FrameIdToFrame) {} - llvm::SmallVector operator()(LinearCallStackId LinearCSId) { - llvm::SmallVector Frames; + std::vector operator()(LinearCallStackId LinearCSId) { + std::vector Frames; const unsigned char *Ptr = CallStackBase + diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp index 1d9860e0ea7e..2b227a65c1d8 100644 --- a/llvm/lib/ProfileData/MemProf.cpp +++ b/llvm/lib/ProfileData/MemProf.cpp @@ -338,8 +338,7 @@ IndexedMemProfRecord::deserialize(const MemProfSchema &Schema, } MemProfRecord IndexedMemProfRecord::toMemProfRecord( - llvm::function_ref(const CallStackId)> Callback) - const { + llvm::function_ref(const CallStackId)> Callback) const { MemProfRecord Record; Record.AllocSites.reserve(AllocSites.size()); diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp index c0a3bf8464d2..d70c6a7a0a15 100644 --- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp @@ -759,7 +759,7 @@ static void readMemprof(Module &M, Function &F, std::map> LocHashToAllocInfo; // For the callsites we need to record the index of the associated frame in // the frame array (see comments below where the map entries are added). - std::map *, unsigned>>> + std::map *, unsigned>>> LocHashToCallSites; for (auto &AI : MemProfRec->AllocSites) { // Associate the allocation info with the leaf frame. The later matching @@ -815,7 +815,7 @@ static void readMemprof(Module &M, Function &F, // and another callsite). std::map>::iterator AllocInfoIter; - std::map *, + std::map *, unsigned>>>::iterator CallSitesIter; for (const DILocation *DIL = I.getDebugLoc(); DIL != nullptr; DIL = DIL->getInlinedAt()) { -- GitLab From ba7f52ccf42fd481a1b309fe76863729fdd18c1c Mon Sep 17 00:00:00 2001 From: Alex Langford Date: Thu, 6 Jun 2024 14:25:53 -0700 Subject: [PATCH 161/462] [lldb] Fix TestStatisticsAPI after 9293fc798152 (#94683) --- .../API/functionalities/stats_api/TestStatisticsAPI.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lldb/test/API/functionalities/stats_api/TestStatisticsAPI.py b/lldb/test/API/functionalities/stats_api/TestStatisticsAPI.py index 851097bdfecf..f06c9ae14bb7 100644 --- a/lldb/test/API/functionalities/stats_api/TestStatisticsAPI.py +++ b/lldb/test/API/functionalities/stats_api/TestStatisticsAPI.py @@ -85,14 +85,15 @@ class TestStatsAPI(TestBase): stats_summary.GetAsJSON(stream_summary) debug_stats_summary = json.loads(stream_summary.GetData()) self.assertNotIn("modules", debug_stats_summary) - self.assertNotIn("memory", debug_stats_summary) self.assertNotIn("commands", debug_stats_summary) # Summary values should be the same as in full statistics. - # Except the parse time on Mac OS X is not deterministic. + # The exceptions to this are: + # - The parse time on Mac OS X is not deterministic. + # - Memory usage may grow over time due to the use of ConstString. for key, value in debug_stats_summary.items(): self.assertIn(key, debug_stats) - if key != "targets" and not key.endswith("Time"): + if key != "memory" and key != "targets" and not key.endswith("Time"): self.assertEqual(debug_stats[key], value) def test_command_stats_api(self): -- GitLab From 81e9a3c3fc9552ef0e1191da70aa0d44d918bcc0 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Thu, 6 Jun 2024 14:32:10 -0700 Subject: [PATCH 162/462] [clang-repl] Disable new test after #89811 --- clang/test/Interpreter/pretty-print.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clang/test/Interpreter/pretty-print.c b/clang/test/Interpreter/pretty-print.c index f6158ad4ecc9..d21749a649e1 100644 --- a/clang/test/Interpreter/pretty-print.c +++ b/clang/test/Interpreter/pretty-print.c @@ -3,6 +3,9 @@ // RUN: cat %s | clang-repl -Xcc -xc | FileCheck %s // RUN: cat %s | clang-repl -Xcc -std=c++11 | FileCheck %s +// Fails with `Symbols not found: [ __clang_Interpreter_SetValueNoAlloc ]`. +// UNSUPPORTED: hwasan + const char* c_str = "Hello, world!"; c_str // CHECK: Not implement yet. -- GitLab From 855af13e8a1ded22fde3898c4e321cc5830eebf1 Mon Sep 17 00:00:00 2001 From: aaryanshukla <53713108+aaryanshukla@users.noreply.github.com> Date: Thu, 6 Jun 2024 21:35:59 +0000 Subject: [PATCH 163/462] [libc] fixed target issue with exit_handler (#94678) - addressed https://github.com/llvm/llvm-project/pull/94317#issuecomment-2153103129 - added conditional in cmake file for exit_handler object library Co-authored-by: Aaryan Shukla --- libc/src/stdlib/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt index 219c85dda675..f0091ad367c0 100644 --- a/libc/src/stdlib/CMakeLists.txt +++ b/libc/src/stdlib/CMakeLists.txt @@ -416,6 +416,9 @@ add_entrypoint_object( libc.src.__support.OSUtil.osutil ) +# TODO: Move all exit functions to linux specific + +if (TARGET libc.src.__support.threads.mutex) add_object_library( exit_handler SRCS @@ -432,6 +435,7 @@ add_object_library( libc.src.__support.fixedvector libc.src.__support.threads.mutex ) +endif() add_entrypoint_object( atexit -- GitLab From b1861429cb1843c3a3da56b1411fdb131fc33519 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 6 Jun 2024 23:48:23 +0200 Subject: [PATCH 164/462] [sanitizer] Make CHECKs in bitvector more precise (NFC) (#94630) These CHECKs are all checking indices, which must be strictly smaller than the size (otherwise they would go out of bounds). --- compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h b/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h index 07a59ab11c42..eef1e7e9d957 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h @@ -321,23 +321,23 @@ class TwoLevelBitVector { }; private: - void check(uptr idx) const { CHECK_LE(idx, size()); } + void check(uptr idx) const { CHECK_LT(idx, size()); } uptr idx0(uptr idx) const { uptr res = idx / (BV::kSize * BV::kSize); - CHECK_LE(res, kLevel1Size); + CHECK_LT(res, kLevel1Size); return res; } uptr idx1(uptr idx) const { uptr res = (idx / BV::kSize) % BV::kSize; - CHECK_LE(res, BV::kSize); + CHECK_LT(res, BV::kSize); return res; } uptr idx2(uptr idx) const { uptr res = idx % BV::kSize; - CHECK_LE(res, BV::kSize); + CHECK_LT(res, BV::kSize); return res; } -- GitLab From 19bbbcbedcbddcfc39202d7d801508a20baf83b6 Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Thu, 6 Jun 2024 14:49:12 -0700 Subject: [PATCH 165/462] =?UTF-8?q?[sanitizer=5Fcommon]=20Change=20allocat?= =?UTF-8?q?or=20base=20in=20test=20case=20for=20compatibili=E2=80=A6=20(#9?= =?UTF-8?q?3234)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …ty with high-entropy ASLR With high-entropy ASLR (e.g., 32-bits == 16TB), the allocator base of 0x700000000000 (112TB) may collide with the placement of the libraries (e.g., on Linux, the mmap base could be 128TB - 16TB == 112TB). This results in a segfault in the test case. This patch moves the allocator base below the PIE program segment, inspired by fb77ca05ffb4f8e666878f2f6718a9fb4d686839. As per that patch: 1) we are leaving the old behavior for Apple 2) since ASLR cannot be set above 32-bits for x86-64 Linux, we expect this new layout to be durable. Note that this is only changing a test case, not the behavior of sanitizers. Sanitizers have their own settings for initializing the allocator base. Reproducer: 1. ninja check-sanitizer # Just to build the test binary needed below; no need to actually run the tests here 2. sudo sysctl vm.mmap_rnd_bits=32 # Increase ASLR entropy 3. for f in `seq 1 10000`; do echo $f; GTEST_FILTER=*SizeClassAllocator64Dense ./projects/compiler-rt/lib/sanitizer_common/tests/Sanitizer-x86_64-Test > /tmp/x; if [ $? -ne 0 ]; then cat /tmp/x; fi; done --- .../sanitizer_common/tests/sanitizer_allocator_test.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cpp index 58f2c8f7b333..1a1ccce82d25 100644 --- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cpp +++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cpp @@ -69,12 +69,17 @@ const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x2000000000ULL; // 128G. static const u64 kAddressSpaceSize = 1ULL << 38; typedef VeryDenseSizeClassMap SizeClassMap; -#else +# elif SANITIZER_APPLE static const uptr kAllocatorSpace = 0x700000000000ULL; static const uptr kAllocatorSize = 0x010000000000ULL; // 1T. static const u64 kAddressSpaceSize = 1ULL << 47; typedef DefaultSizeClassMap SizeClassMap; -#endif +# else +static const uptr kAllocatorSpace = 0x500000000000ULL; +static const uptr kAllocatorSize = 0x010000000000ULL; // 1T. +static const u64 kAddressSpaceSize = 1ULL << 47; +typedef DefaultSizeClassMap SizeClassMap; +# endif template struct AP64 { // Allocator Params. Short name for shorter demangled names.. -- GitLab From 92a870888c2d19f0004d2d0bc90730559c6bf0f4 Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Thu, 6 Jun 2024 15:22:07 -0700 Subject: [PATCH 166/462] [compiler-rt] Map internal_sigaction to __sys_sigaction on FreeBSD (#84441) This function is called during very early startup and which can result in a crash on FreeBSD. The sigaction() function in libc is indirected via a table so that it can be interposed by the threading library rather than calling the syscall directly. In the crash I was observing this table had not yet been relocated, so we ended up jumping to an invalid address. To avoid this problem we can call __sys_sigaction, which calls the syscall directly and in FreeBSD 15 is part of libsys rather than libc, so does not depend on libc being fully initialized. --- .../sanitizer_common/sanitizer_linux_libcdep.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp index cccbb4d256df..6d05411222d9 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp @@ -54,6 +54,8 @@ # undef MAP_NORESERVE # define MAP_NORESERVE 0 extern const Elf_Auxinfo *__elf_aux_vector; +extern "C" int __sys_sigaction(int signum, const struct sigaction *act, + struct sigaction *oldact); # endif # if SANITIZER_NETBSD @@ -93,12 +95,22 @@ SANITIZER_WEAK_ATTRIBUTE int real_sigaction(int signum, const void *act, void *oldact); int internal_sigaction(int signum, const void *act, void *oldact) { -# if !SANITIZER_GO +# if SANITIZER_FREEBSD + // On FreeBSD, call the sigaction syscall directly (part of libsys in FreeBSD + // 15) since the libc version goes via a global interposing table. Due to + // library initialization order the table can be relocated after the call to + // InitializeDeadlySignals() which then crashes when dereferencing the + // uninitialized pointer in libc. + return __sys_sigaction(signum, (const struct sigaction *)act, + (struct sigaction *)oldact); +# else +# if !SANITIZER_GO if (&real_sigaction) return real_sigaction(signum, act, oldact); -# endif +# endif return sigaction(signum, (const struct sigaction *)act, (struct sigaction *)oldact); +# endif } void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, -- GitLab From 210692bcd6588ac4ae518436f55895a4520deb18 Mon Sep 17 00:00:00 2001 From: Med Ismail Bennani Date: Thu, 6 Jun 2024 15:49:09 -0700 Subject: [PATCH 167/462] [llvm/IR] Fix module build issue following e57308b063bb (NFC) (#94580) This patch fixes a build issue following e57308b063bb when enabling module build. With that change, we failed to build the LLVM_IR module since GEPNoWrapFlags wasn't defined prior to using it. This patch addressed that issue by including the missing header in `llvm/IR/IRBuilderFolder.h` which uses the `GEPNoWrapFlags` type. This should ensure that we can always build the `LLVM_IR` module. Signed-off-by: Med Ismail Bennani Signed-off-by: Med Ismail Bennani --- llvm/include/llvm/IR/IRBuilderFolder.h | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/include/llvm/IR/IRBuilderFolder.h b/llvm/include/llvm/IR/IRBuilderFolder.h index 3c42eb2cf2a5..921001c8a5d5 100644 --- a/llvm/include/llvm/IR/IRBuilderFolder.h +++ b/llvm/include/llvm/IR/IRBuilderFolder.h @@ -15,6 +15,7 @@ #define LLVM_IR_IRBUILDERFOLDER_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/IR/GEPNoWrapFlags.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" -- GitLab From 5c0df5fe2207562d99149bad65c04e3054181f75 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 6 Jun 2024 15:52:45 -0700 Subject: [PATCH 168/462] [memprof] Add CallStackRadixTreeBuilder (#93784) Call stacks are a huge portion of the MemProf profile, taking up 70+% of the profile file size. This patch implements a radix tree to compress call stacks, which are known to have long common prefixes. Specifically, CallStackRadixTreeBuilder, introduced in this patch, takes call stacks in the MemProf profile, sorts them in the dictionary order to maximize the common prefix between adjacent call stacks, and then encodes a radix tree into a single array that is ready for serialization. The resulting radix array is essentially a concatenation of call stack arrays, each encoded with its length followed by the payload, except that these arrays contain "instructions" like "skip 7 elements forward" to borrow common prefixes from other call stacks. This patch does not integrate with the MemProf serialization/deserialization infrastructure yet. Once integrated, the radix tree is expected to roughly halve the file size of the MemProf profile. --- llvm/include/llvm/ProfileData/MemProf.h | 107 ++++++++++++++ llvm/lib/ProfileData/MemProf.cpp | 159 +++++++++++++++++++++ llvm/unittests/ProfileData/MemProfTest.cpp | 107 ++++++++++++++ 3 files changed, 373 insertions(+) diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h index 9f6f9955260d..6b0fa6cd6541 100644 --- a/llvm/include/llvm/ProfileData/MemProf.h +++ b/llvm/include/llvm/ProfileData/MemProf.h @@ -923,6 +923,113 @@ struct IndexedMemProfData { llvm::MapVector> CallStackData; }; +// Construct a radix tree of call stacks. +// +// A set of call stacks might look like: +// +// CallStackId 1: f1 -> f2 -> f3 +// CallStackId 2: f1 -> f2 -> f4 -> f5 +// CallStackId 3: f1 -> f2 -> f4 -> f6 +// CallStackId 4: f7 -> f8 -> f9 +// +// where each fn refers to a stack frame. +// +// Since we expect a lot of common prefixes, we can compress the call stacks +// into a radix tree like: +// +// CallStackId 1: f1 -> f2 -> f3 +// | +// CallStackId 2: +---> f4 -> f5 +// | +// CallStackId 3: +---> f6 +// +// CallStackId 4: f7 -> f8 -> f9 +// +// Now, we are interested in retrieving call stacks for a given CallStackId, so +// we just need a pointer from a given call stack to its parent. For example, +// CallStackId 2 would point to CallStackId 1 as a parent. +// +// We serialize the radix tree above into a single array along with the length +// of each call stack and pointers to the parent call stacks. +// +// Index: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 +// Array: L3 f9 f8 f7 L4 f6 J3 L4 f5 f4 J3 L3 f3 f2 f1 +// ^ ^ ^ ^ +// | | | | +// CallStackId 4: 0 --+ | | | +// CallStackId 3: 4 --------------+ | | +// CallStackId 2: 7 -----------------------+ | +// CallStackId 1: 11 -----------------------------------+ +// +// - LN indicates the length of a call stack, encoded as ordinary integer N. +// +// - JN indicates a pointer to the parent, encoded as -N. +// +// The radix tree allows us to reconstruct call stacks in the leaf-to-root +// order as we scan the array from left ro right while following pointers to +// parents along the way +// +// For example, if we are decoding CallStackId 2, we start a forward traversal +// at Index 7, noting the call stack length of 4 and obtaining f5 and f4. When +// we see J3 at Index 10, we resume a forward traversal at Index 13 = 10 + 3, +// picking up f2 and f1. We are done after collecting 4 frames as indicated at +// the beginning of the traversal. +// +// On-disk IndexedMemProfRecord will refer to call stacks by their indexes into +// the radix tree array, so we do not explicitly encode mappings like: +// "CallStackId 1 -> 11". +class CallStackRadixTreeBuilder { + // The radix tree array. + std::vector RadixArray; + + // Mapping from CallStackIds to indexes into RadixArray. + llvm::DenseMap CallStackPos; + + // In build, we partition a given call stack into two parts -- the prefix + // that's common with the previously encoded call stack and the frames beyond + // the common prefix -- the unique portion. Then we want to find out where + // the common prefix is stored in RadixArray so that we can link the unique + // portion to the common prefix. Indexes, declared below, helps with our + // needs. Intuitively, Indexes tells us where each of the previously encoded + // call stack is stored in RadixArray. More formally, Indexes satisfies: + // + // RadixArray[Indexes[I]] == Prev[I] + // + // for every I, where Prev is the the call stack in the root-to-leaf order + // previously encoded by build. (Note that Prev, as passed to + // encodeCallStack, is in the leaf-to-root order.) + // + // For example, if the call stack being encoded shares 5 frames at the root of + // the call stack with the previously encoded call stack, + // RadixArray[Indexes[0]] is the root frame of the common prefix. + // RadixArray[Indexes[5 - 1]] is the last frame of the common prefix. + std::vector Indexes; + + using CSIdPair = std::pair>; + + // Encode a call stack into RadixArray. Return the starting index within + // RadixArray. + LinearCallStackId encodeCallStack( + const llvm::SmallVector *CallStack, + const llvm::SmallVector *Prev, + const llvm::DenseMap &MemProfFrameIndexes); + +public: + CallStackRadixTreeBuilder() = default; + + // Build a radix tree array. + void build(llvm::MapVector> + &&MemProfCallStackData, + const llvm::DenseMap &MemProfFrameIndexes); + + const std::vector &getRadixArray() const { return RadixArray; } + + const llvm::DenseMap & + getCallStackPos() const { + return CallStackPos; + } +}; + // Verify that each CallStackId is computed with hashCallStack. This function // is intended to help transition from CallStack to CSId in // IndexedAllocationInfo. diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp index 2b227a65c1d8..aecac2416952 100644 --- a/llvm/lib/ProfileData/MemProf.cpp +++ b/llvm/lib/ProfileData/MemProf.cpp @@ -409,6 +409,165 @@ CallStackId hashCallStack(ArrayRef CS) { return CSId; } +// Encode a call stack into RadixArray. Return the starting index within +// RadixArray. For each call stack we encode, we emit two or three components +// into RadixArray. If a given call stack doesn't have a common prefix relative +// to the previous one, we emit: +// +// - the frames in the given call stack in the root-to-leaf order +// +// - the length of the given call stack +// +// If a given call stack has a non-empty common prefix relative to the previous +// one, we emit: +// +// - the relative location of the common prefix, encoded as a negative number. +// +// - a portion of the given call stack that's beyond the common prefix +// +// - the length of the given call stack, including the length of the common +// prefix. +// +// The resulting RadixArray requires a somewhat unintuitive backward traversal +// to reconstruct a call stack -- read the call stack length and scan backward +// while collecting frames in the leaf to root order. build, the caller of this +// function, reverses RadixArray in place so that we can reconstruct a call +// stack as if we were deserializing an array in a typical way -- the call stack +// length followed by the frames in the leaf-to-root order except that we need +// to handle pointers to parents along the way. +// +// To quickly determine the location of the common prefix within RadixArray, +// Indexes caches the indexes of the previous call stack's frames within +// RadixArray. +LinearCallStackId CallStackRadixTreeBuilder::encodeCallStack( + const llvm::SmallVector *CallStack, + const llvm::SmallVector *Prev, + const llvm::DenseMap &MemProfFrameIndexes) { + // Compute the length of the common root prefix between Prev and CallStack. + uint32_t CommonLen = 0; + if (Prev) { + auto Pos = std::mismatch(Prev->rbegin(), Prev->rend(), CallStack->rbegin(), + CallStack->rend()); + CommonLen = std::distance(CallStack->rbegin(), Pos.second); + } + + // Drop the portion beyond CommonLen. + assert(CommonLen <= Indexes.size()); + Indexes.resize(CommonLen); + + // Append a pointer to the parent. + if (CommonLen) { + uint32_t CurrentIndex = RadixArray.size(); + uint32_t ParentIndex = Indexes.back(); + // The offset to the parent must be negative because we are pointing to an + // element we've already added to RadixArray. + assert(ParentIndex < CurrentIndex); + RadixArray.push_back(ParentIndex - CurrentIndex); + } + + // Copy the part of the call stack beyond the common prefix to RadixArray. + assert(CommonLen <= CallStack->size()); + for (FrameId F : llvm::drop_begin(llvm::reverse(*CallStack), CommonLen)) { + // Remember the index of F in RadixArray. + Indexes.push_back(RadixArray.size()); + RadixArray.push_back(MemProfFrameIndexes.find(F)->second); + } + assert(CallStack->size() == Indexes.size()); + + // End with the call stack length. + RadixArray.push_back(CallStack->size()); + + // Return the index within RadixArray where we can start reconstructing a + // given call stack from. + return RadixArray.size() - 1; +} + +void CallStackRadixTreeBuilder::build( + llvm::MapVector> + &&MemProfCallStackData, + const llvm::DenseMap &MemProfFrameIndexes) { + // Take the vector portion of MemProfCallStackData. The vector is exactly + // what we need to sort. Also, we no longer need its lookup capability. + llvm::SmallVector CallStacks = MemProfCallStackData.takeVector(); + + // Return early if we have no work to do. + if (CallStacks.empty()) { + RadixArray.clear(); + CallStackPos.clear(); + return; + } + + // Sort the list of call stacks in the dictionary order to maximize the length + // of the common prefix between two adjacent call stacks. + llvm::sort(CallStacks, [&](const CSIdPair &L, const CSIdPair &R) { + // Call stacks are stored from leaf to root. Perform comparisons from the + // root. + return std::lexicographical_compare( + L.second.rbegin(), L.second.rend(), R.second.rbegin(), R.second.rend(), + [&](FrameId F1, FrameId F2) { return F1 < F2; }); + }); + + // Reserve some reasonable amount of storage. + RadixArray.clear(); + RadixArray.reserve(CallStacks.size() * 8); + + // Indexes will grow as long as the longest call stack. + Indexes.clear(); + Indexes.reserve(512); + + // CallStackPos will grow to exactly CallStacks.size() entries. + CallStackPos.clear(); + CallStackPos.reserve(CallStacks.size()); + + // Compute the radix array. We encode one call stack at a time, computing the + // longest prefix that's shared with the previous call stack we encode. For + // each call stack we encode, we remember a mapping from CallStackId to its + // position within RadixArray. + // + // As an optimization, we encode from the last call stack in CallStacks to + // reduce the number of times we follow pointers to the parents. Consider the + // list of call stacks that has been sorted in the dictionary order: + // + // Call Stack 1: F1 + // Call Stack 2: F1 -> F2 + // Call Stack 3: F1 -> F2 -> F3 + // + // If we traversed CallStacks in the forward order, we would end up with a + // radix tree like: + // + // Call Stack 1: F1 + // | + // Call Stack 2: +---> F2 + // | + // Call Stack 3: +---> F3 + // + // Notice that each call stack jumps to the previous one. However, if we + // traverse CallStacks in the reverse order, then Call Stack 3 has the + // complete call stack encoded without any pointers. Call Stack 1 and 2 point + // to appropriate prefixes of Call Stack 3. + const llvm::SmallVector *Prev = nullptr; + for (const auto &[CSId, CallStack] : llvm::reverse(CallStacks)) { + LinearCallStackId Pos = + encodeCallStack(&CallStack, Prev, MemProfFrameIndexes); + CallStackPos.insert({CSId, Pos}); + Prev = &CallStack; + } + + // "RadixArray.size() - 1" below is problematic if RadixArray is empty. + assert(!RadixArray.empty()); + + // Reverse the radix array in place. We do so mostly for intuitive + // deserialization where we would read the length field and then the call + // stack frames proper just like any other array deserialization, except + // that we have occasional jumps to take advantage of prefixes. + for (size_t I = 0, J = RadixArray.size() - 1; I < J; ++I, --J) + std::swap(RadixArray[I], RadixArray[J]); + + // "Reverse" the indexes stored in CallStackPos. + for (auto &[K, V] : CallStackPos) + V = RadixArray.size() - 1 - V; +} + void verifyIndexedMemProfRecord(const IndexedMemProfRecord &Record) { for (const auto &AS : Record.AllocSites) { assert(AS.CSId == hashCallStack(AS.CallStack)); diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp index a913718d0fe0..e120bc3e3594 100644 --- a/llvm/unittests/ProfileData/MemProfTest.cpp +++ b/llvm/unittests/ProfileData/MemProfTest.cpp @@ -662,4 +662,111 @@ TEST(MemProf, MissingFrameId) { ASSERT_TRUE(FrameIdConv.LastUnmappedId.has_value()); EXPECT_EQ(*FrameIdConv.LastUnmappedId, 3U); } + +// Verify CallStackRadixTreeBuilder can handle empty inputs. +TEST(MemProf, RadixTreeBuilderEmpty) { + llvm::DenseMap MemProfFrameIndexes; + llvm::MapVector> MemProfCallStackData; + llvm::memprof::CallStackRadixTreeBuilder Builder; + Builder.build(std::move(MemProfCallStackData), MemProfFrameIndexes); + ASSERT_THAT(Builder.getRadixArray(), testing::IsEmpty()); + const auto &Mappings = Builder.getCallStackPos(); + ASSERT_THAT(Mappings, testing::IsEmpty()); +} + +// Verify CallStackRadixTreeBuilder can handle one trivial call stack. +TEST(MemProf, RadixTreeBuilderOne) { + llvm::DenseMap MemProfFrameIndexes = { + {11, 1}, {12, 2}, {13, 3}}; + llvm::SmallVector CS1 = {13, 12, 11}; + llvm::MapVector> MemProfCallStackData; + MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS1), CS1}); + llvm::memprof::CallStackRadixTreeBuilder Builder; + Builder.build(std::move(MemProfCallStackData), MemProfFrameIndexes); + EXPECT_THAT(Builder.getRadixArray(), testing::ElementsAreArray({ + 3U, // Size of CS1, + 3U, // MemProfFrameIndexes[13] + 2U, // MemProfFrameIndexes[12] + 1U // MemProfFrameIndexes[11] + })); + const auto &Mappings = Builder.getCallStackPos(); + ASSERT_THAT(Mappings, SizeIs(1)); + EXPECT_THAT(Mappings, testing::Contains(testing::Pair( + llvm::memprof::hashCallStack(CS1), 0U))); +} + +// Verify CallStackRadixTreeBuilder can form a link between two call stacks. +TEST(MemProf, RadixTreeBuilderTwo) { + llvm::DenseMap MemProfFrameIndexes = { + {11, 1}, {12, 2}, {13, 3}}; + llvm::SmallVector CS1 = {12, 11}; + llvm::SmallVector CS2 = {13, 12, 11}; + llvm::MapVector> MemProfCallStackData; + MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS1), CS1}); + MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS2), CS2}); + llvm::memprof::CallStackRadixTreeBuilder Builder; + Builder.build(std::move(MemProfCallStackData), MemProfFrameIndexes); + EXPECT_THAT(Builder.getRadixArray(), + testing::ElementsAreArray({ + 2U, // Size of CS1 + static_cast(-3), // Jump 3 steps + 3U, // Size of CS2 + 3U, // MemProfFrameIndexes[13] + 2U, // MemProfFrameIndexes[12] + 1U // MemProfFrameIndexes[11] + })); + const auto &Mappings = Builder.getCallStackPos(); + ASSERT_THAT(Mappings, SizeIs(2)); + EXPECT_THAT(Mappings, testing::Contains(testing::Pair( + llvm::memprof::hashCallStack(CS1), 0U))); + EXPECT_THAT(Mappings, testing::Contains(testing::Pair( + llvm::memprof::hashCallStack(CS2), 2U))); +} + +// Verify CallStackRadixTreeBuilder can form a jump to a prefix that itself has +// another jump to another prefix. +TEST(MemProf, RadixTreeBuilderSuccessiveJumps) { + llvm::DenseMap MemProfFrameIndexes = { + {11, 1}, {12, 2}, {13, 3}, {14, 4}, {15, 5}, {16, 6}, {17, 7}, {18, 8}, + }; + llvm::SmallVector CS1 = {14, 13, 12, 11}; + llvm::SmallVector CS2 = {15, 13, 12, 11}; + llvm::SmallVector CS3 = {17, 16, 12, 11}; + llvm::SmallVector CS4 = {18, 16, 12, 11}; + llvm::MapVector> MemProfCallStackData; + MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS1), CS1}); + MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS2), CS2}); + MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS3), CS3}); + MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS4), CS4}); + llvm::memprof::CallStackRadixTreeBuilder Builder; + Builder.build(std::move(MemProfCallStackData), MemProfFrameIndexes); + EXPECT_THAT(Builder.getRadixArray(), + testing::ElementsAreArray({ + 4U, // Size of CS1 + 4U, // MemProfFrameIndexes[14] + static_cast(-3), // Jump 3 steps + 4U, // Size of CS2 + 5U, // MemProfFrameIndexes[15] + 3U, // MemProfFrameIndexes[13] + static_cast(-7), // Jump 7 steps + 4U, // Size of CS3 + 7U, // MemProfFrameIndexes[17] + static_cast(-3), // Jump 3 steps + 4U, // Size of CS4 + 8U, // MemProfFrameIndexes[18] + 6U, // MemProfFrameIndexes[16] + 2U, // MemProfFrameIndexes[12] + 1U // MemProfFrameIndexes[11] + })); + const auto &Mappings = Builder.getCallStackPos(); + ASSERT_THAT(Mappings, SizeIs(4)); + EXPECT_THAT(Mappings, testing::Contains(testing::Pair( + llvm::memprof::hashCallStack(CS1), 0U))); + EXPECT_THAT(Mappings, testing::Contains(testing::Pair( + llvm::memprof::hashCallStack(CS2), 3U))); + EXPECT_THAT(Mappings, testing::Contains(testing::Pair( + llvm::memprof::hashCallStack(CS3), 7U))); + EXPECT_THAT(Mappings, testing::Contains(testing::Pair( + llvm::memprof::hashCallStack(CS4), 10U))); +} } // namespace -- GitLab From 21f5ee014db5df4fcf3dfe4350fe9c6627f799ad Mon Sep 17 00:00:00 2001 From: Congcong Cai Date: Fri, 7 Jun 2024 07:02:45 +0800 Subject: [PATCH 169/462] [clang-tidy]fix crashing when self include cycles for misc-header-include-cycle (#94636) Fixes: #94634 --- .../clang-tidy/misc/HeaderIncludeCycleCheck.cpp | 6 ++++-- clang-tools-extra/docs/ReleaseNotes.rst | 4 ++++ .../clang-tidy/checkers/misc/header-include-cycle.self.cpp | 3 +++ 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 clang-tools-extra/test/clang-tidy/checkers/misc/header-include-cycle.self.cpp diff --git a/clang-tools-extra/clang-tidy/misc/HeaderIncludeCycleCheck.cpp b/clang-tools-extra/clang-tidy/misc/HeaderIncludeCycleCheck.cpp index fadfdc869d37..37bc577c646a 100644 --- a/clang-tools-extra/clang-tidy/misc/HeaderIncludeCycleCheck.cpp +++ b/clang-tools-extra/clang-tidy/misc/HeaderIncludeCycleCheck.cpp @@ -139,8 +139,10 @@ public: auto CurrentIt = Files.rbegin(); do { - Check.diag(CurrentIt->Loc, "'%0' included from here", DiagnosticIDs::Note) - << CurrentIt->Name; + if (CurrentIt->Loc.isValid()) + Check.diag(CurrentIt->Loc, "'%0' included from here", + DiagnosticIDs::Note) + << CurrentIt->Name; } while (CurrentIt++ != It); } diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 661b2b1620d0..da30aceb8d49 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -317,6 +317,10 @@ Changes in existing checks Additionally, the option `UseHeaderFileExtensions` is removed, so that the check uses the `HeaderFileExtensions` option unconditionally. +- Improved :doc:`misc-header-include-cycle + ` check by avoiding crash for self + include cycles. + - Improved :doc:`misc-unused-using-decls ` check by replacing the local option `HeaderFileExtensions` by the global option of the same name. diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/header-include-cycle.self.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/header-include-cycle.self.cpp new file mode 100644 index 000000000000..245dd0a65a8b --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/misc/header-include-cycle.self.cpp @@ -0,0 +1,3 @@ +// RUN: not clang-tidy %s -checks='-*,misc-header-include-cycle' + +#include "header-include-cycle.self.cpp" -- GitLab From 86070a84c454614275d0904f285f4bd20b6b7849 Mon Sep 17 00:00:00 2001 From: Konstantin Varlamov Date: Thu, 6 Jun 2024 16:35:33 -0700 Subject: [PATCH 170/462] [libc++][hardening] Finish documenting hardening. (#92021) --- libcxx/docs/Hardening.rst | 373 ++++++++++++++++++++++++++- libcxx/docs/ReleaseNotes/18.rst | 2 +- libcxx/docs/TestingLibcxx.rst | 45 ++++ libcxx/include/__configuration/abi.h | 3 +- 4 files changed, 410 insertions(+), 13 deletions(-) diff --git a/libcxx/docs/Hardening.rst b/libcxx/docs/Hardening.rst index 0761f42368e9..996e7aed881a 100644 --- a/libcxx/docs/Hardening.rst +++ b/libcxx/docs/Hardening.rst @@ -1,4 +1,4 @@ -.. _hardening-modes: +.. _hardening: =============== Hardening Modes @@ -29,8 +29,11 @@ modes are: rigour impacts performance more than fast mode: we recommend benchmarking to determine if that is acceptable for your program. - **Debug mode**, which enables all the available checks in the library, - including internal assertions, some of which might be very expensive. This - mode is intended to be used for testing, not in production. + including heuristic checks that might have significant performance overhead as + well as internal library assertions. This mode should be used in + non-production environments (such as test suites, CI, or local development). + We don’t commit to a particular level of performance in this mode and it’s + *not* intended to be used in production. .. note:: @@ -72,17 +75,367 @@ to control the level by passing **one** of the following options to the compiler Notes for vendors ----------------- -Vendors can set the default hardening mode by providing ``LIBCXX_HARDENING_MODE`` -as a configuration option, with the possible values of ``none``, ``fast``, -``extensive`` and ``debug``. The default value is ``none`` which doesn't enable -any hardening checks (this mode is sometimes called the ``unchecked`` mode). +Vendors can set the default hardening mode by providing +``LIBCXX_HARDENING_MODE`` as a configuration option, with the possible values of +``none``, ``fast``, ``extensive`` and ``debug``. The default value is ``none`` +which doesn't enable any hardening checks (this mode is sometimes called the +``unchecked`` mode). This option controls both the hardening mode that the precompiled library is built with and the default hardening mode that users will build with. If set to ``none``, the precompiled library will not contain any assertions, and user code will default to building without assertions. -Iterator bounds checking ------------------------- +Vendors can also override the way the program is terminated when an assertion +fails by :ref:`providing a custom header `. -TODO(hardening) +Assertion categories +==================== + +Inside the library, individual assertions are grouped into different +*categories*. Each hardening mode enables a different set of assertion +categories; categories provide an additional layer of abstraction that makes it +easier to reason about the high-level semantics of a hardening mode. + +.. note:: + + Users are not intended to interact with these categories directly -- the + categories are considered internal to the library and subject to change. + +- ``valid-element-access`` -- checks that any attempts to access a container + element, whether through the container object or through an iterator, are + valid and do not attempt to go out of bounds or otherwise access + a non-existent element. This also includes operations that set up an imminent + invalid access (e.g. incrementing an end iterator). For iterator checks to + work, bounded iterators must be enabled in the ABI. Types like + ``std::optional`` and ``std::function`` are considered containers (with at + most one element) for the purposes of this check. + +- ``valid-input-range`` -- checks that ranges (whether expressed as an iterator + pair, an iterator and a sentinel, an iterator and a count, or + a ``std::range``) given as input to library functions are valid: + - the sentinel is reachable from the begin iterator; + - TODO(hardening): both iterators refer to the same container. + + ("input" here refers to "an input given to an algorithm", not to an iterator + category) + + Violating assertions in this category leads to an out-of-bounds access. + +- ``non-null`` -- checks that the pointer being dereferenced is not null. On + most modern platforms, the zero address does not refer to an actual location + in memory, so a null pointer dereference would not compromise the memory + security of a program (however, it is still undefined behavior that can result + in strange errors due to compiler optimizations). + +- ``non-overlapping-ranges`` -- for functions that take several ranges as + arguments, checks that those ranges do not overlap. + +- ``valid-deallocation`` -- checks that an attempt to deallocate memory is valid + (e.g. the given object was allocated by the given allocator). Violating this + category typically results in a memory leak. + +- ``valid-external-api-call`` -- checks that a call to an external API doesn't + fail in an unexpected manner. This includes triggering documented cases of + undefined behavior in an external library (like attempting to unlock an + unlocked mutex in pthreads). Any API external to the library falls under this + category (from system calls to compiler intrinsics). We generally don't expect + these failures to compromise memory safety or otherwise create an immediate + security issue. + +- ``compatible-allocator`` -- checks any operations that exchange nodes between + containers to make sure the containers have compatible allocators. + +- ``argument-within-domain`` -- checks that the given argument is within the + domain of valid arguments for the function. Violating this typically produces + an incorrect result (e.g. ``std::clamp`` returns the original value without + clamping it due to incorrect functors) or puts an object into an invalid state + (e.g. a string view where only a subset of elements is accessible). This + category is for assertions violating which doesn't cause any immediate issues + in the library -- whatever the consequences are, they will happen in the user + code. + +- ``pedantic`` -- checks preconditions that are imposed by the Standard, but + violating which happens to be benign in libc++. + +- ``semantic-requirement`` -- checks that the given argument satisfies the + semantic requirements imposed by the Standard. Typically, there is no simple + way to completely prove that a semantic requirement is satisfied; thus, this + would often be a heuristic check and it might be quite expensive. + +- ``internal`` -- checks that internal invariants of the library hold. These + assertions don't depend on user input. + +- ``uncategorized`` -- for assertions that haven't been properly classified yet. + This category is an escape hatch used for some existing assertions in the + library; all new code should have its assertions properly classified. + +Mapping between the hardening modes and the assertion categories +================================================================ + +.. list-table:: + :header-rows: 1 + :widths: auto + + * - Category name + - ``fast`` + - ``extensive`` + - ``debug`` + * - ``valid-element-access`` + - ✅ + - ✅ + - ✅ + * - ``valid-input-range`` + - ✅ + - ✅ + - ✅ + * - ``non-null`` + - ❌ + - ✅ + - ✅ + * - ``non-overlapping-ranges`` + - ❌ + - ✅ + - ✅ + * - ``valid-deallocation`` + - ❌ + - ✅ + - ✅ + * - ``valid-external-api-call`` + - ❌ + - ✅ + - ✅ + * - ``compatible-allocator`` + - ❌ + - ✅ + - ✅ + * - ``argument-within-domain`` + - ❌ + - ✅ + - ✅ + * - ``pedantic`` + - ❌ + - ✅ + - ✅ + * - ``semantic-requirement`` + - ❌ + - ❌ + - ✅ + * - ``internal`` + - ❌ + - ❌ + - ✅ + * - ``uncategorized`` + - ❌ + - ✅ + - ✅ + +.. note:: + + At the moment, each subsequent hardening mode is a strict superset of the + previous one (in other words, each subsequent mode only enables additional + assertion categories without disabling any), but this won't necessarily be + true for any hardening modes that might be added in the future. + +.. note:: + + The categories enabled by each mode are subject to change and users should not + rely on the precise assertions enabled by a mode at a given point in time. + However, the library does guarantee to keep the hardening modes stable and + to fulfill the semantics documented here. + +Hardening assertion failure +=========================== + +In production modes (``fast`` and ``extensive``), a hardening assertion failure +immediately ``_traps `` +the program. This is the safest approach that also minimizes the code size +penalty as the failure handler maps to a single instruction. The downside is +that the failure provides no additional details other than the stack trace +(which might also be affected by optimizations). + +TODO(hardening): describe ``__builtin_verbose_trap`` once we can use it. + +In the ``debug`` mode, an assertion failure terminates the program in an +unspecified manner and also outputs the associated error message to the error +output. This is less secure and increases the size of the binary (among other +things, it has to store the error message strings) but makes the failure easier +to debug. It also allows testing the error messages in our test suite. + +.. _override-assertion-handler: + +Overriding the assertion failure handler +---------------------------------------- + +Vendors can override the default assertion handler mechanism by following these +steps: + +- create a header file that provides a definition of a macro called + ``_LIBCPP_ASSERTION_HANDLER``. The macro will be invoked when a hardening + assertion fails, with a single parameter containing a null-terminated string + with the error message. +- when configuring the library, provide the path to custom header (relative to + the root of the repository) via the CMake variable + ``LIBCXX_ASSERTION_HANDLER_FILE``. + +Note that almost all libc++ headers include the assertion handler header which +means it should not include anything non-trivial from the standard library to +avoid creating circular dependencies. + +There is no existing mechanism for users to override the assertion handler +because the ability to do the override other than at configure-time carries an +unavoidable code size penalty that would otherwise be imposed on all users, +whether they require such customization or not. Instead, we let vendors decide +what's right on their platform for their users -- a vendor who wishes to provide +this capability is free to do so, e.g. by declaring the assertion handler as an +overridable function. + +ABI +=== + +Setting a hardening mode does **not** affect the ABI. Each mode uses the subset +of checks available in the current ABI configuration which is determined by the +platform. + +It is important to stress that whether a particular check is enabled depends on +the combination of the selected hardening mode and the hardening-related ABI +options. Some checks require changing the ABI from the "default" to store +additional information in the library classes -- e.g. checking whether an +iterator is valid upon dereference generally requires storing data about bounds +inside the iterator object. Using ``std::span`` as an example, setting the +hardening mode to ``fast`` will always enable the ``valid-element-access`` +checks when accessing elements via a ``std::span`` object, but whether +dereferencing a ``std::span`` iterator does the equivalent check depends on the +ABI configuration. + +ABI options +----------- + +Vendors can use the following ABI options to enable additional hardening checks: + +- ``_LIBCPP_ABI_BOUNDED_ITERATORS`` -- changes the iterator type of select + containers (see below) to a bounded iterator that keeps track of whether it's + within the bounds of the original container and asserts valid bounds on every + dereference. + + ABI impact: changes the iterator type of the relevant containers. + + Supported containers: + + - ``span``; + - ``string_view``. + +ABI tags +-------- + +We use ABI tags to allow translation units built with different hardening modes +to interact with each other without causing ODR violations. Knowing how +hardening modes are encoded into the ABI tags might be useful to examine +a binary and determine whether it was built with hardening enabled. + +.. warning:: + We don't commit to the encoding scheme used by the ABI tags being stable + between different releases of libc++. The tags themselves are never stable, by + design -- new releases increase the version number. The following describes + the state of the latest release and is for informational purposes only. + +The first character of an ABI tag encodes the hardening mode: + +- ``f`` -- [f]ast mode; +- ``s`` -- extensive ("[s]afe") mode; +- ``d`` -- [d]ebug mode; +- ``n`` -- [n]one mode. + +Hardened containers status +========================== + +.. list-table:: + :header-rows: 1 + :widths: auto + + * - Name + - Member functions + - Iterators (ABI-dependent) + * - ``span`` + - ✅ + - ✅ + * - ``string_view`` + - ✅ + - ✅ + * - ``array`` + - ✅ + - ❌ + * - ``vector`` + - ✅ + - ❌ + * - ``string`` + - ✅ + - ❌ + * - ``list`` + - ✅ + - ❌ + * - ``forward_list`` + - ❌ + - ❌ + * - ``deque`` + - ✅ + - ❌ + * - ``map`` + - ❌ + - ❌ + * - ``set`` + - ❌ + - ❌ + * - ``multimap`` + - ❌ + - ❌ + * - ``multiset`` + - ❌ + - ❌ + * - ``unordered_map`` + - Partial + - Partial + * - ``unordered_set`` + - Partial + - Partial + * - ``unordered_multimap`` + - Partial + - Partial + * - ``unordered_multiset`` + - Partial + - Partial + * - ``mdspan`` + - ✅ + - ❌ + * - ``optional`` + - ✅ + - N/A + * - ``function`` + - ❌ + - N/A + * - ``variant`` + - N/A + - N/A + * - ``any`` + - N/A + - N/A + * - ``expected`` + - ✅ + - N/A + * - ``valarray`` + - Partial + - N/A + * - ``bitset`` + - ❌ + - N/A + +Testing +======= + +Please see :ref:`Testing documentation `. + +Further reading +=============== + +- ``_Hardening RFC ``: + contains some of the design rationale. diff --git a/libcxx/docs/ReleaseNotes/18.rst b/libcxx/docs/ReleaseNotes/18.rst index fcd630e09b44..4f7b9b362e5e 100644 --- a/libcxx/docs/ReleaseNotes/18.rst +++ b/libcxx/docs/ReleaseNotes/18.rst @@ -40,7 +40,7 @@ and C++26 features. New hardened modes for the library have been added, replacing the legacy debug mode that was removed in the LLVM 17 release. Unlike the legacy debug mode, some of these hardening modes are -also intended to be used in production. See :ref:`hardening-modes` for more details. +also intended to be used in production. See :ref:`hardening` for more details. Work on the ranges support has progressed. See :ref:`ranges-status` for the current status. diff --git a/libcxx/docs/TestingLibcxx.rst b/libcxx/docs/TestingLibcxx.rst index 50ee9d4ee400..d9f4fe467fe3 100644 --- a/libcxx/docs/TestingLibcxx.rst +++ b/libcxx/docs/TestingLibcxx.rst @@ -480,3 +480,48 @@ For example: $ ./algorithms.libcxx.out --benchmark_filter=BM_Sort.* # Only runs the sort benchmarks For more information about running benchmarks see `Google Benchmark`_. + + +.. _testing-hardening-assertions: + +Testing hardening assertions +============================ + +Each hardening assertion should be tested using death tests (via the +``TEST_LIBCPP_ASSERT_FAILURE`` macro). Use the ``libcpp-hardening-mode`` Lit +feature to make sure the assertion is enabled in (and only in) the intended +modes. The convention is to use `assert.` in the name of the test file to make +it easier to identify as a hardening test, e.g. ``assert.my_func.pass.cpp``. +A toy example: + +.. code-block:: cpp + + // Note: the following three annotations are currently needed to use the + // `TEST_LIBCPP_ASSERT_FAILURE`. + // REQUIRES: has-unix-headers + // UNSUPPORTED: c++03 + // XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing + + // Example: only run this test in `fast`/`extensive`/`debug` modes. + // UNSUPPORTED: libcpp-hardening-mode=none + // Example: only run this test in the `debug` mode. + // REQUIRES: libcpp-hardening-mode=debug + // Example: only run this test in `extensive`/`debug` modes. + // REQUIRES: libcpp-hardening-mode={{extensive|debug}} + + #include + + #include "check_assertion.h" // Contains the `TEST_LIBCPP_ASSERT_FAILURE` macro + + int main(int, char**) { + std::type_being_tested foo; + int bad_input = -1; + TEST_LIBCPP_ASSERT_FAILURE(foo.some_function_that_asserts(bad_input), + "The expected assertion message"); + + return 0; + } + +Note that error messages are only tested (matched) if the ``debug`` +hardening mode is used. + diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h index 17aceb042f52..73375fa47d07 100644 --- a/libcxx/include/__configuration/abi.h +++ b/libcxx/include/__configuration/abi.h @@ -127,8 +127,7 @@ // // Supported containers: // - `span`; -// - `string_view`; -// - `array`. +// - `string_view`. // #define _LIBCPP_ABI_BOUNDED_ITERATORS #if defined(_LIBCPP_COMPILER_CLANG_BASED) -- GitLab From aae025a01b423ce43e554bc93da15016fa647beb Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 6 Jun 2024 17:10:52 -0700 Subject: [PATCH 171/462] [ELF] Improve -r section group tests --- lld/test/ELF/relocatable-comdat.s | 77 ++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/lld/test/ELF/relocatable-comdat.s b/lld/test/ELF/relocatable-comdat.s index 160c48d9fb4d..45ca9fb7a248 100644 --- a/lld/test/ELF/relocatable-comdat.s +++ b/lld/test/ELF/relocatable-comdat.s @@ -3,46 +3,71 @@ ## may be rewritten because group members may change their indices. Additionally, ## group member may be combined or discarded (e.g. /DISCARD/ or --gc-sections). -# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o -# RUN: ld.lld -r %t.o %t.o -o %t.ro -# RUN: llvm-readelf -g -S %t.ro | FileCheck %s +# RUN: rm -rf %t && split-file %s %t && cd %t +# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o +# RUN: ld.lld -r a.o a.o -o a.ro +# RUN: llvm-readelf -g -S a.ro | FileCheck %s -# CHECK: Name Type Address Off Size ES Flg Lk Inf Al -# CHECK: .group GROUP 0000000000000000 {{.*}} 000014 04 {{[1-9]}} [[#]] 4 +# CHECK: Name Type Address Off Size ES Flg Lk Inf Al +# CHECK: .group GROUP 0000000000000000 {{.*}} 00001c 04 [[#]] [[#]] 4 +# CHECK-NEXT: .rodata.bar PROGBITS 0000000000000000 {{.*}} 000001 00 AG 0 0 1 +# CHECK-NEXT: .rodata.foo PROGBITS 0000000000000000 {{.*}} 000001 00 AG 0 0 1 +# CHECK-NEXT: .text.bar PROGBITS 0000000000000000 {{.*}} 000008 00 AXG 0 0 1 +# CHECK-NEXT: .rela.text.bar RELA 0000000000000000 {{.*}} 000018 18 IG [[#]] [[#]] 8 +# CHECK-NEXT: .text.foo PROGBITS 0000000000000000 {{.*}} 000008 00 AXG [[#]] [[#]] 1 +# CHECK-NEXT: .rela.text.foo RELA 0000000000000000 {{.*}} 000018 18 IG [[#]] [[#]] 8 +# CHECK-NEXT: .note.GNU-stack -# CHECK: COMDAT group section [{{.*}}] `.group' [abc] contains 4 sections: +# CHECK: COMDAT group section [{{.*}}] `.group' [abc] contains 6 sections: # CHECK-NEXT: Name # CHECK-NEXT: .rodata.bar # CHECK-NEXT: .rodata.foo # CHECK-NEXT: .text.bar +# CHECK-NEXT: .rela.text.bar # CHECK-NEXT: .text.foo +# CHECK-NEXT: .rela.text.foo ## Rewrite SHT_GROUP content if some members are combined. -# RUN: echo 'SECTIONS { .rodata : {*(.rodata.*)} .text : {*(.text.*)} }' > %t1.lds -# RUN: ld.lld -r -T %t1.lds %t.o %t.o -o %t1.ro -# RUN: llvm-readelf -g -S %t1.ro | FileCheck %s --check-prefix=SCRIPT1 +# RUN: echo 'SECTIONS { .rodata : {*(.rodata.*)} .text : {*(.text.*)} }' > combine.lds +# RUN: ld.lld -r -T combine.lds a.o a.o -o combine.ro +# RUN: llvm-readelf -g -S combine.ro | FileCheck %s --check-prefix=COMBINE -# SCRIPT1: Name Type Address Off Size ES Flg Lk Inf Al -# SCRIPT1: .group GROUP 0000000000000000 {{.*}} 00000c 04 {{[1-9]}} [[#]] 4 +# COMBINE: Name Type Address Off Size ES Flg Lk Inf Al +# COMBINE: .rodata PROGBITS 0000000000000000 {{.*}} 000002 00 AG 0 0 1 +# COMBINE-NEXT: .text PROGBITS 0000000000000000 {{.*}} 000010 00 AXG 0 0 4 +# COMBINE-NEXT: .group GROUP 0000000000000000 {{.*}} 000014 04 [[#]] [[#]] 4 +# COMBINE-NEXT: .rela.text RELA 0000000000000000 {{.*}} 000018 18 IG [[#]] [[#]] 8 +# COMBINE-NEXT: .rela.text RELA 0000000000000000 {{.*}} 000018 18 IG [[#]] [[#]] 8 +# COMBINE-NEXT: .note.GNU-stack -# SCRIPT1: COMDAT group section [{{.*}}] `.group' [abc] contains 2 sections: -# SCRIPT1-NEXT: Name -# SCRIPT1-NEXT: .rodata -# SCRIPT1-NEXT: .text +# COMBINE: COMDAT group section [{{.*}}] `.group' [abc] contains 4 sections: +# COMBINE-NEXT: Name +# COMBINE-NEXT: .rodata +# COMBINE-NEXT: .text +# COMBINE-NEXT: .rela.text +# COMBINE-NEXT: .rela.text -# RUN: echo 'SECTIONS { /DISCARD/ : {*(.rodata.*)} }' > %t2.lds -# RUN: ld.lld -r -T %t2.lds %t.o %t.o -o %t2.ro -# RUN: llvm-readelf -g -S %t2.ro | FileCheck %s --check-prefix=SCRIPT2 +# RUN: echo 'SECTIONS { /DISCARD/ : {*(.rodata.*)} }' > discard-rodata.lds +# RUN: ld.lld -r -T discard-rodata.lds a.o a.o -o discard-rodata.ro +# RUN: llvm-readelf -g -S discard-rodata.ro | FileCheck %s --check-prefix=NO-RODATA ## Handle discarded group members. -# SCRIPT2: [Nr] Name Type Address Off Size ES Flg Lk Inf Al -# SCRIPT2: [ 2] .group GROUP 0000000000000000 {{.*}} 00000c 04 {{[1-9]}} [[#]] 4 +# NO-RODATA: Name Type Address Off Size ES Flg Lk Inf Al +# NO-RODATA: .group GROUP 0000000000000000 {{.*}} 000014 04 [[#]] [[#]] 4 +# NO-RODATA-NEXT: .text.bar PROGBITS 0000000000000000 {{.*}} 000008 00 AXG 0 0 1 +# NO-RODATA-NEXT: .rela.text.bar RELA 0000000000000000 {{.*}} 000018 18 IG [[#]] [[#]] 8 +# NO-RODATA-NEXT: .text.foo PROGBITS 0000000000000000 {{.*}} 000008 00 AXG [[#]] [[#]] 1 +# NO-RODATA-NEXT: .rela.text.foo RELA 0000000000000000 {{.*}} 000018 18 IG [[#]] [[#]] 8 +# NO-RODATA-NEXT: .note.GNU-stack -# SCRIPT2: COMDAT group section [{{.*}}] `.group' [abc] contains 2 sections: -# SCRIPT2-NEXT: Name -# SCRIPT2-NEXT: .text.bar -# SCRIPT2-NEXT: .text.foo +# NO-RODATA: COMDAT group section [{{.*}}] `.group' [abc] contains 4 sections: +# NO-RODATA-NEXT: Name +# NO-RODATA-NEXT: .text.bar +# NO-RODATA-NEXT: .rela.text.bar +# NO-RODATA-NEXT: .text.foo +# NO-RODATA-NEXT: .rela.text.foo +#--- a.s .weak abc abc: @@ -52,6 +77,6 @@ abc: .byte 42 .section .text.bar,"axG",@progbits,abc,comdat -.quad 42 +.quad abc .section .text.foo,"axG",@progbits,abc,comdat -.long 42 +.quad abc -- GitLab From 5e0fc93d01c216544c12b60c30fe8ac6c9931eb9 Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Thu, 6 Jun 2024 17:25:46 -0700 Subject: [PATCH 172/462] [clang-format] Fix a bug in `AlignAfterOpenBracket: DontAlign` (#94561) Fixes #94555. --- clang/lib/Format/ContinuationIndenter.cpp | 2 +- clang/unittests/Format/FormatTest.cpp | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp index 6b9fbfe0ebf5..be684ac71cd6 100644 --- a/clang/lib/Format/ContinuationIndenter.cpp +++ b/clang/lib/Format/ContinuationIndenter.cpp @@ -1712,7 +1712,7 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State, (!Previous || Previous->isNot(tok::kw_return) || (Style.Language != FormatStyle::LK_Java && PrecedenceLevel > 0)) && (Style.AlignAfterOpenBracket != FormatStyle::BAS_DontAlign || - PrecedenceLevel != prec::Comma || Current.NestingLevel == 0) && + PrecedenceLevel > prec::Comma || Current.NestingLevel == 0) && (!Style.isTableGen() || (Previous && Previous->isOneOf(TT_TableGenDAGArgListComma, TT_TableGenDAGArgListCommaToBreak)))) { diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index 4e427268fb82..dbc1916825f3 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -9241,6 +9241,14 @@ TEST_F(FormatTest, AlignsAfterOpenBracket) { " b));", Style); + Style.ColumnLimit = 30; + verifyFormat("for (int foo = 0; foo < FOO;\n" + " ++foo) {\n" + " bar(foo);\n" + "}", + Style); + Style.ColumnLimit = 80; + Style.AlignAfterOpenBracket = FormatStyle::BAS_AlwaysBreak; Style.BinPackArguments = false; Style.BinPackParameters = false; -- GitLab From 79cd6c3d01c6c69ca870418a3c602dbd1bef29e4 Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Thu, 6 Jun 2024 17:48:37 -0700 Subject: [PATCH 173/462] [dfsan] Add test case for sscanf (#94700) This test case shows a limitation of DFSan's sscanf implementation (introduced in https://reviews.llvm.org/D153775): it simply ignores ordinary characters in the format string, instead of actually comparing them against the input. This may change the semantics of instrumented programs. Importantly, this also means that DFSan's release_shadow_space.c test, which relies on sscanf to scrape the RSS from /proc/maps output, will incorrectly match lines that don't contain RSS information. As a result, it adding together numbers from irrelevant output (e.g., base addresses), resulting in test flakiness (https://github.com/llvm/llvm-project/issues/91287). --- compiler-rt/test/dfsan/sscanf.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 compiler-rt/test/dfsan/sscanf.c diff --git a/compiler-rt/test/dfsan/sscanf.c b/compiler-rt/test/dfsan/sscanf.c new file mode 100644 index 000000000000..dbc2de4ba96c --- /dev/null +++ b/compiler-rt/test/dfsan/sscanf.c @@ -0,0 +1,19 @@ +// RUN: %clang_dfsan %s -o %t && %run %t +// XFAIL: * + +#include +#include + +int main(int argc, char *argv[]) { + char buf[256] = "10000000000-100000000000 rw-p 00000000 00:00 0"; + long rss = 0; + // This test exposes a bug in DFSan's sscanf, that leads to flakiness + // in release_shadow_space.c (see + // https://github.com/llvm/llvm-project/issues/91287) + if (sscanf(buf, "Garbage text before, %ld, Garbage text after", &rss) == 1) { + printf("Error: matched %ld\n", rss); + return 1; + } + + return 0; +} -- GitLab From a1fa43d030168d1ecd04031b4790b101e651770a Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 6 Jun 2024 17:49:52 -0700 Subject: [PATCH 174/462] [ELF] Simplify code. NFC Make it easier to add CREL support. --- lld/ELF/Writer.cpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index 544db201d1ff..0aceb941a1dc 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -783,13 +783,11 @@ template void Writer::addRelIpltSymbols() { // __rela_iplt_{start,end} are initially defined relative to dummy section 0. // We'll override Out::elfHeader with relaDyn later when we are sure that // .rela.dyn will be present in the output. - ElfSym::relaIpltStart = addOptionalRegular( - config->isRela ? "__rela_iplt_start" : "__rel_iplt_start", - Out::elfHeader, 0, STV_HIDDEN); - - ElfSym::relaIpltEnd = addOptionalRegular( - config->isRela ? "__rela_iplt_end" : "__rel_iplt_end", - Out::elfHeader, 0, STV_HIDDEN); + std::string name = config->isRela ? "__rela_iplt_start" : "__rel_iplt_start"; + ElfSym::relaIpltStart = + addOptionalRegular(name, Out::elfHeader, 0, STV_HIDDEN); + name.replace(name.size() - 5, 5, "end"); + ElfSym::relaIpltEnd = addOptionalRegular(name, Out::elfHeader, 0, STV_HIDDEN); } // This function generates assignments for predefined symbols (e.g. _end or @@ -2487,11 +2485,12 @@ template void Writer::assignFileOffsets() { lastRX->lastSec == sec) off = alignToPowerOf2(off, config->maxPageSize); } - for (OutputSection *osec : outputSections) - if (!(osec->flags & SHF_ALLOC)) { - osec->offset = alignToPowerOf2(off, osec->addralign); - off = osec->offset + osec->size; - } + for (OutputSection *osec : outputSections) { + if (osec->flags & SHF_ALLOC) + continue; + osec->offset = alignToPowerOf2(off, osec->addralign); + off = osec->offset + osec->size; + } sectionHeaderOff = alignToPowerOf2(off, config->wordsize); fileSize = sectionHeaderOff + (outputSections.size() + 1) * sizeof(Elf_Shdr); -- GitLab From f42025c2178b30894df657af6aa92e45c7cc0dbc Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 6 Jun 2024 17:51:32 -0700 Subject: [PATCH 175/462] [SPIRV] Fix -Wunused-but-set-variable. NFC --- llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp b/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp index 4b0e7c421df3..2744c25d1bc7 100644 --- a/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp @@ -106,7 +106,7 @@ public: const std::unordered_set ToReplace, BasicBlock *NewTarget) { auto *T = BB->getTerminator(); - if (auto *RI = dyn_cast(T)) + if (isa(T)) return; if (auto *BI = dyn_cast(T)) { -- GitLab From 59003d714a9447bf03e0ce25ad4492290e80b110 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 10 Apr 2024 12:07:37 -0500 Subject: [PATCH 176/462] [InstCombine] Add tests for expanding `foldSelectValueEquivalence`; NFC --- llvm/test/Transforms/InstCombine/select.ll | 46 ++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll index a0ee4383495c..5cc4ee360db9 100644 --- a/llvm/test/Transforms/InstCombine/select.ll +++ b/llvm/test/Transforms/InstCombine/select.ll @@ -667,7 +667,7 @@ define i1 @test39(i1 %cond, double %x) { ; CHECK-LABEL: @test39( ; CHECK-NEXT: ret i1 true ; - %s = select i1 %cond, double %x, double 0x7FF0000000000000 ; RHS = +infty + %s = select i1 %cond, double %x, double 0x7FF0000000000000 ; RHS = +infty %cmp = fcmp ule double %x, %s ret i1 %cmp } @@ -1364,7 +1364,7 @@ define i32 @PR23757_ne(i32 %x, ptr %p) { ; CHECK-NEXT: ret i32 -2147483648 ; %cmp = icmp ne i32 %x, 2147483647 - store i1 %cmp, ptr %p ; thwart predicate canonicalization + store i1 %cmp, ptr %p ; thwart predicate canonicalization %add = add nsw i32 %x, 1 %sel = select i1 %cmp, i32 -2147483648, i32 %add ret i32 %sel @@ -1378,7 +1378,7 @@ define i32 @PR23757_ne_swapped(i32 %x, ptr %p) { ; CHECK-NEXT: ret i32 [[ADD]] ; %cmp = icmp ne i32 %x, 2147483647 - store i1 %cmp, ptr %p ; thwart predicate canonicalization + store i1 %cmp, ptr %p ; thwart predicate canonicalization %add = add nsw i32 %x, 1 %sel = select i1 %cmp, i32 %add, i32 -2147483648 ret i32 %sel @@ -2809,6 +2809,46 @@ define <2 x i8> @select_replacement_add_eq_vec_undef(<2 x i8> %x, <2 x i8> %y) { ret <2 x i8> %sel } +define <2 x i8> @select_replacement_add_eq_vec_undef_okay(<2 x i8> %x, <2 x i8> %y) { +; CHECK-LABEL: @select_replacement_add_eq_vec_undef_okay( +; CHECK-NEXT: [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> , <2 x i8> [[Y:%.*]] +; CHECK-NEXT: ret <2 x i8> [[SEL]] +; + %cmp = icmp eq <2 x i8> %x, + %add = add <2 x i8> %x, + %sel = select <2 x i1> %cmp, <2 x i8> %add, <2 x i8> %y + ret <2 x i8> %sel +} + + +define <2 x i8> @select_replacement_add_eq_vec_undef_okay_todo(<2 x i8> %x, <2 x i8> %y) { +; CHECK-LABEL: @select_replacement_add_eq_vec_undef_okay_todo( +; CHECK-NEXT: [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[ADD:%.*]] = add <2 x i8> [[X]], +; CHECK-NEXT: [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[ADD]], <2 x i8> [[Y:%.*]] +; CHECK-NEXT: ret <2 x i8> [[SEL]] +; + %cmp = icmp eq <2 x i8> %x, + %add = add <2 x i8> %x, + %sel = select <2 x i1> %cmp, <2 x i8> %add, <2 x i8> %y + ret <2 x i8> %sel +} + +define <2 x i8> @select_replacement_xor_eq_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) { +; CHECK-LABEL: @select_replacement_xor_eq_vec( +; CHECK-NEXT: [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[ADD:%.*]] = xor <2 x i8> [[X]], [[Y]] +; CHECK-NEXT: [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[ADD]], <2 x i8> [[Z:%.*]] +; CHECK-NEXT: ret <2 x i8> [[SEL]] +; + %cmp = icmp eq <2 x i8> %x, %y + %add = xor <2 x i8> %x, %y + %sel = select <2 x i1> %cmp, <2 x i8> %add, <2 x i8> %z + ret <2 x i8> %sel +} + + define i8 @select_replacement_add_ne(i8 %x, i8 %y) { ; CHECK-LABEL: @select_replacement_add_ne( ; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[X:%.*]], 1 -- GitLab From 7e7c29ba087e38056b91f1d783db0883dcc33ef7 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 5 Jun 2024 11:30:53 -0500 Subject: [PATCH 177/462] [InstCombine] Improve coverage of `foldSelectValueEquivalence` for constants We don't need the `noundef` check if the new simplification is a constant. This cleans up regressions from folding multiuse: `(icmp eq/ne (sub/xor x, y), 0)` -> `(icmp eq/ne x, y)`. Closes #88298 --- .../InstCombine/InstCombineSelect.cpp | 59 ++++++++++++------- llvm/test/Transforms/InstCombine/abs-1.ll | 7 +-- llvm/test/Transforms/InstCombine/select.ll | 6 +- 3 files changed, 42 insertions(+), 30 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index d2aaa5e23054..b04e0b300f95 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -1288,21 +1288,36 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel, Swapped = true; } - // In X == Y ? f(X) : Z, try to evaluate f(Y) and replace the operand. - // Make sure Y cannot be undef though, as we might pick different values for - // undef in the icmp and in f(Y). Additionally, take care to avoid replacing - // X == Y ? X : Z with X == Y ? Y : Z, as that would lead to an infinite - // replacement cycle. Value *CmpLHS = Cmp.getOperand(0), *CmpRHS = Cmp.getOperand(1); - if (TrueVal != CmpLHS && isGuaranteedNotToBeUndef(CmpRHS, SQ.AC, &Sel, &DT)) { - if (Value *V = simplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, SQ, - /* AllowRefinement */ true)) - // Require either the replacement or the simplification result to be a - // constant to avoid infinite loops. - // FIXME: Make this check more precise. - if (isa(CmpRHS) || isa(V)) + auto ReplaceOldOpWithNewOp = [&](Value *OldOp, + Value *NewOp) -> Instruction * { + // In X == Y ? f(X) : Z, try to evaluate f(Y) and replace the operand. + // Take care to avoid replacing X == Y ? X : Z with X == Y ? Y : Z, as that + // would lead to an infinite replacement cycle. + // If we will be able to evaluate f(Y) to a constant, we can allow undef, + // otherwise Y cannot be undef as we might pick different values for undef + // in the icmp and in f(Y). + if (TrueVal == OldOp) + return nullptr; + + if (Value *V = simplifyWithOpReplaced(TrueVal, OldOp, NewOp, SQ, + /* AllowRefinement=*/true)) { + // Need some guarantees about the new simplified op to ensure we don't inf + // loop. + // If we simplify to a constant, replace if we aren't creating new undef. + if (match(V, m_ImmConstant()) && + isGuaranteedNotToBeUndef(V, SQ.AC, &Sel, &DT)) return replaceOperand(Sel, Swapped ? 2 : 1, V); + // If NewOp is a constant and OldOp is not replace iff NewOp doesn't + // contain and undef elements. + if (match(NewOp, m_ImmConstant())) { + if (isGuaranteedNotToBeUndef(NewOp, SQ.AC, &Sel, &DT)) + return replaceOperand(Sel, Swapped ? 2 : 1, V); + return nullptr; + } + } + // Even if TrueVal does not simplify, we can directly replace a use of // CmpLHS with CmpRHS, as long as the instruction is not used anywhere // else and is safe to speculatively execute (we may end up executing it @@ -1310,16 +1325,18 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel, // undefined behavior). Only do this if CmpRHS is a constant, as // profitability is not clear for other cases. // FIXME: Support vectors. - if (match(CmpRHS, m_ImmConstant()) && !match(CmpLHS, m_ImmConstant()) && - !Cmp.getType()->isVectorTy()) - if (replaceInInstruction(TrueVal, CmpLHS, CmpRHS)) + if (OldOp == CmpLHS && match(NewOp, m_ImmConstant()) && + !match(OldOp, m_ImmConstant()) && !Cmp.getType()->isVectorTy() && + isGuaranteedNotToBeUndef(NewOp, SQ.AC, &Sel, &DT)) + if (replaceInInstruction(TrueVal, OldOp, NewOp)) return &Sel; - } - if (TrueVal != CmpRHS && isGuaranteedNotToBeUndef(CmpLHS, SQ.AC, &Sel, &DT)) - if (Value *V = simplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, SQ, - /* AllowRefinement */ true)) - if (isa(CmpLHS) || isa(V)) - return replaceOperand(Sel, Swapped ? 2 : 1, V); + return nullptr; + }; + + if (Instruction *R = ReplaceOldOpWithNewOp(CmpLHS, CmpRHS)) + return R; + if (Instruction *R = ReplaceOldOpWithNewOp(CmpRHS, CmpLHS)) + return R; auto *FalseInst = dyn_cast(FalseVal); if (!FalseInst) diff --git a/llvm/test/Transforms/InstCombine/abs-1.ll b/llvm/test/Transforms/InstCombine/abs-1.ll index 32bd7a37053e..0cf7cd97d8ff 100644 --- a/llvm/test/Transforms/InstCombine/abs-1.ll +++ b/llvm/test/Transforms/InstCombine/abs-1.ll @@ -852,11 +852,8 @@ define i8 @abs_diff_signed_sgt_nuw_extra_use3(i8 %a, i8 %b) { define i32 @abs_diff_signed_slt_swap_wrong_pred1(i32 %a, i32 %b) { ; CHECK-LABEL: @abs_diff_signed_slt_swap_wrong_pred1( -; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]] -; CHECK-NEXT: [[SUB_BA:%.*]] = sub nsw i32 [[B]], [[A]] -; CHECK-NEXT: [[SUB_AB:%.*]] = sub nsw i32 [[A]], [[B]] -; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB_BA]], i32 [[SUB_AB]] -; CHECK-NEXT: ret i32 [[COND]] +; CHECK-NEXT: [[SUB_AB:%.*]] = sub nsw i32 [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: ret i32 [[SUB_AB]] ; %cmp = icmp eq i32 %a, %b %sub_ba = sub nsw i32 %b, %a diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll index 5cc4ee360db9..846ede45028e 100644 --- a/llvm/test/Transforms/InstCombine/select.ll +++ b/llvm/test/Transforms/InstCombine/select.ll @@ -2838,8 +2838,7 @@ define <2 x i8> @select_replacement_add_eq_vec_undef_okay_todo(<2 x i8> %x, <2 x define <2 x i8> @select_replacement_xor_eq_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) { ; CHECK-LABEL: @select_replacement_xor_eq_vec( ; CHECK-NEXT: [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[ADD:%.*]] = xor <2 x i8> [[X]], [[Y]] -; CHECK-NEXT: [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[ADD]], <2 x i8> [[Z:%.*]] +; CHECK-NEXT: [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> zeroinitializer, <2 x i8> [[Z:%.*]] ; CHECK-NEXT: ret <2 x i8> [[SEL]] ; %cmp = icmp eq <2 x i8> %x, %y @@ -2905,8 +2904,7 @@ define i8 @select_replacement_sub_noundef_but_may_be_poison(i8 %x, i8 noundef %y define i8 @select_replacement_sub(i8 %x, i8 %y, i8 %z) { ; CHECK-LABEL: @select_replacement_sub( ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[SUB:%.*]] = sub i8 [[X]], [[Y]] -; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], i8 [[SUB]], i8 [[Z:%.*]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], i8 0, i8 [[Z:%.*]] ; CHECK-NEXT: ret i8 [[SEL]] ; %cmp = icmp eq i8 %x, %y -- GitLab From 2d65097b4ff18f99e4baf18e2e0b155ecf478b0a Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 6 Jun 2024 18:12:07 -0700 Subject: [PATCH 178/462] [RISCV] Unify all the code that adds unaligned-scalar/vector-mem to Features vector. (#94660) Instead of having multiple places insert into the Features vector independently, check all the conditions in one place. This avoids a subtle ordering requirement that -mstrict-align processing had to be done after the others. --- clang/lib/Driver/ToolChains/Arch/RISCV.cpp | 31 +++++++++++++--------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp index 2e2bce849467..26789b0ba6e0 100644 --- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp +++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp @@ -67,11 +67,6 @@ static void getRISCFeaturesFromMcpu(const Driver &D, const Arg *A, D.Diag(clang::diag::err_drv_unsupported_option_argument) << A->getSpelling() << Mcpu; } - - if (llvm::RISCV::hasFastUnalignedAccess(Mcpu)) { - Features.push_back("+unaligned-scalar-mem"); - Features.push_back("+unaligned-vector-mem"); - } } void riscv::getRISCVTargetFeatures(const Driver &D, const llvm::Triple &Triple, @@ -82,6 +77,8 @@ void riscv::getRISCVTargetFeatures(const Driver &D, const llvm::Triple &Triple, if (!getArchFeatures(D, MArch, Features, Args)) return; + bool CPUFastUnaligned = false; + // If users give march and mcpu, get std extension feature from MArch // and other features (ex. mirco architecture feature) from mcpu if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ)) { @@ -90,6 +87,9 @@ void riscv::getRISCVTargetFeatures(const Driver &D, const llvm::Triple &Triple, CPU = llvm::sys::getHostCPUName(); getRISCFeaturesFromMcpu(D, A, Triple, CPU, Features); + + if (llvm::RISCV::hasFastUnalignedAccess(CPU)) + CPUFastUnaligned = true; } // Handle features corresponding to "-ffixed-X" options @@ -169,18 +169,23 @@ void riscv::getRISCVTargetFeatures(const Driver &D, const llvm::Triple &Triple, Features.push_back("-relax"); } - // Android requires fast unaligned access on RISCV64. - if (Triple.isAndroid()) { + // If -mstrict-align or -mno-strict-align is passed, use it. Otherwise, the + // unaligned-*-mem is enabled if the CPU supports it or the target is + // Android. + if (const Arg *A = Args.getLastArg(options::OPT_mno_strict_align, + options::OPT_mstrict_align)) { + if (A->getOption().matches(options::OPT_mno_strict_align)) { + Features.push_back("+unaligned-scalar-mem"); + Features.push_back("+unaligned-vector-mem"); + } else { + Features.push_back("-unaligned-scalar-mem"); + Features.push_back("-unaligned-vector-mem"); + } + } else if (CPUFastUnaligned || Triple.isAndroid()) { Features.push_back("+unaligned-scalar-mem"); Features.push_back("+unaligned-vector-mem"); } - // -mstrict-align is default, unless -mno-strict-align is specified. - AddTargetFeature(Args, Features, options::OPT_mno_strict_align, - options::OPT_mstrict_align, "unaligned-scalar-mem"); - AddTargetFeature(Args, Features, options::OPT_mno_strict_align, - options::OPT_mstrict_align, "unaligned-vector-mem"); - // Now add any that the user explicitly requested on the command line, // which may override the defaults. handleTargetFeaturesGroup(D, Triple, Args, Features, -- GitLab From c967c7e0223845b54a5220e2d4211fee50c5fb30 Mon Sep 17 00:00:00 2001 From: Nour Date: Fri, 7 Jun 2024 04:36:33 +0300 Subject: [PATCH 179/462] [clang-format][NFC] Remove an else after a return statement (#94548) --- clang/tools/clang-format/ClangFormat.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp index 50dd073c4125..6cba1267f3b0 100644 --- a/clang/tools/clang-format/ClangFormat.cpp +++ b/clang/tools/clang-format/ClangFormat.cpp @@ -509,8 +509,7 @@ static bool format(StringRef FileName, bool ErrorOnIncompleteFormat = false) { if (OutputXML || DryRun) { if (DryRun) return emitReplacementWarnings(Replaces, AssumedFileName, Code); - else - outputXML(Replaces, FormatChanges, Status, Cursor, CursorPosition); + outputXML(Replaces, FormatChanges, Status, Cursor, CursorPosition); } else { IntrusiveRefCntPtr InMemoryFileSystem( new llvm::vfs::InMemoryFileSystem); -- GitLab From a10135f4922624f92c3a108d23ab64adc09dd285 Mon Sep 17 00:00:00 2001 From: Gedare Bloom Date: Thu, 6 Jun 2024 19:38:21 -0600 Subject: [PATCH 180/462] [clang-format]: Annotate colons found in inline assembly (#92617) Short-circuit the parsing of tok::colon to label colons found within lines starting with asm as InlineASMColon. Fixes #92616. --------- Co-authored-by: Owen Pan --- clang/lib/Format/TokenAnnotator.cpp | 9 +-- clang/unittests/Format/TokenAnnotatorTest.cpp | 78 ++++++++++++++++++- 2 files changed, 76 insertions(+), 11 deletions(-) diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 26c0aa36bdcb..1fe3b61a5a81 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -1358,6 +1358,8 @@ private: Line.First->startsSequence(tok::kw_export, Keywords.kw_module) || Line.First->startsSequence(tok::kw_export, Keywords.kw_import)) { Tok->setType(TT_ModulePartitionColon); + } else if (Line.First->is(tok::kw_asm)) { + Tok->setType(TT_InlineASMColon); } else if (Contexts.back().ColonIsDictLiteral || Style.isProto()) { Tok->setType(TT_DictLiteral); if (Style.Language == FormatStyle::LK_TextProto) { @@ -1425,13 +1427,6 @@ private: // This handles a special macro in ObjC code where selectors including // the colon are passed as macro arguments. Tok->setType(TT_ObjCMethodExpr); - } else if (Contexts.back().ContextKind == tok::l_paren && - !Line.InPragmaDirective) { - if (Style.isTableGen() && Contexts.back().IsTableGenDAGArg) { - Tok->setType(TT_TableGenDAGArgListColon); - break; - } - Tok->setType(TT_InlineASMColon); } break; case tok::pipe: diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp index 3d609a1f041b..8cc5c239d30a 100644 --- a/clang/unittests/Format/TokenAnnotatorTest.cpp +++ b/clang/unittests/Format/TokenAnnotatorTest.cpp @@ -1496,12 +1496,82 @@ TEST_F(TokenAnnotatorTest, RequiresDoesNotChangeParsingOfTheRest) { TEST_F(TokenAnnotatorTest, UnderstandsAsm) { auto Tokens = annotate("__asm{\n" - "a:\n" - "};"); - ASSERT_EQ(Tokens.size(), 7u) << Tokens; + "\"a\":\n" + ": x\n" + ":};"); + ASSERT_EQ(Tokens.size(), 10u) << Tokens; EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown); EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_InlineASMBrace); - EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_InlineASMBrace); + EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[7], tok::r_brace, TT_InlineASMBrace); + + Tokens = annotate("__asm(\n" + "\"a\":\n" + ": x\n" + ":);"); + ASSERT_EQ(Tokens.size(), 10u) << Tokens; + EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown); + EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon); + + Tokens = annotate("asm volatile (\n" + "\"a_label:\"\n" + ":\n" + ": x\n" + ":);"); + ASSERT_EQ(Tokens.size(), 11u) << Tokens; + EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown); + EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[7], tok::colon, TT_InlineASMColon); + + Tokens = annotate("__asm__(\n" + "\"a_label:\"\n" + ": x\n" + ":\n" + ": y);"); + ASSERT_EQ(Tokens.size(), 11u) << Tokens; + EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown); + EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon); + + Tokens = annotate("__asm volatile (\n" + "\"a_label:\"\n" + "\"a b c(%%x)\"\n" + ":\n" + ": x\n" + ":);"); + ASSERT_EQ(Tokens.size(), 12u) << Tokens; + EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown); + EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[8], tok::colon, TT_InlineASMColon); + + Tokens = annotate("asm(\n" + "\"insn\"\n" + ": \"=r\" (var1), \"=&r\" (value)\n" + ":\n" + ": \"memory\");"); + ASSERT_EQ(Tokens.size(), 19u) << Tokens; + EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown); + EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[13], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[14], tok::colon, TT_InlineASMColon); + + Tokens = annotate("__asm__ volatile (\n" + "\"ldr r1, [r0, %%[sym]]\"\n" + ":\n" + ": [sym] \"J\" (aaaaa(aaaa, aaaa))\n" + ");"); + ASSERT_EQ(Tokens.size(), 21u) << Tokens; + EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown); + EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon); + EXPECT_TOKEN(Tokens[6], tok::l_square, TT_InlineASMSymbolicNameLSquare); } TEST_F(TokenAnnotatorTest, UnderstandsObjCBlock) { -- GitLab From f2441b0297501610b59527f93e3174c814d63941 Mon Sep 17 00:00:00 2001 From: Weining Lu Date: Fri, 7 Jun 2024 09:52:32 +0800 Subject: [PATCH 181/462] [Driver][test] Rename loongarch-{default-,}features.c --- .../Driver/{loongarch-default-features.c => loongarch-features.c} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename clang/test/Driver/{loongarch-default-features.c => loongarch-features.c} (100%) diff --git a/clang/test/Driver/loongarch-default-features.c b/clang/test/Driver/loongarch-features.c similarity index 100% rename from clang/test/Driver/loongarch-default-features.c rename to clang/test/Driver/loongarch-features.c -- GitLab From 5c104879c1a98eeb845c03e7c45206bd48e88f0c Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Thu, 6 Jun 2024 11:51:05 +0800 Subject: [PATCH 182/462] [serialization] no transitive decl change (#92083) Following of https://github.com/llvm/llvm-project/pull/86912 The motivation of the patch series is that, for a module interface unit `X`, when the dependent modules of `X` changes, if the changes is not relevant with `X`, we hope the BMI of `X` won't change. For the specific patch, we hope if the changes was about irrelevant declaration changes, we hope the BMI of `X` won't change. **However**, I found the patch itself is not very useful in practice, since the adding or removing declarations, will change the state of identifiers and types in most cases. That said, for the most simple example, ``` // partA.cppm export module m:partA; // partA.v1.cppm export module m:partA; export void a() {} // partB.cppm export module m:partB; export void b() {} // m.cppm export module m; export import :partA; export import :partB; // onlyUseB; export module onlyUseB; import m; export inline void onluUseB() { b(); } ``` the BMI of `onlyUseB` will change after we change the implementation of `partA.cppm` to `partA.v1.cppm`. Since `partA.v1.cppm` introduces new identifiers and types (the function prototype). So in this patch, we have to write the tests as: ``` // partA.cppm export module m:partA; export int getA() { ... } export int getA2(int) { ... } // partA.v1.cppm export module m:partA; export int getA() { ... } export int getA(int) { ... } export int getA2(int) { ... } // partB.cppm export module m:partB; export void b() {} // m.cppm export module m; export import :partA; export import :partB; // onlyUseB; export module onlyUseB; import m; export inline void onluUseB() { b(); } ``` so that the new introduced declaration `int getA(int)` doesn't introduce new identifiers and types, then the BMI of `onlyUseB` can keep unchanged. While it looks not so great, the patch should be the base of the patch to erase the transitive change for identifiers and types since I don't know how can we introduce new types and identifiers without introducing new declarations. Given how tightly the relationship between declarations, types and identifiers, I think we can only reach the ideal state after we made the series for all of the three entties. The design of the patch is similar to https://github.com/llvm/llvm-project/pull/86912, which extends the 32-bit DeclID to 64-bit and use the higher bits to store the module file index and the lower bits to store the Local Decl ID. A slight difference is that we only use 48 bits to store the new DeclID since we try to use the higher 16 bits to store the module ID in the prefix of Decl class. Previously, we use 32 bits to store the module ID and 32 bits to store the DeclID. I don't want to allocate additional space so I tried to make the additional space the same as 64 bits. An potential interesting thing here is about the relationship between the module ID and the module file index. I feel we can get the module file index by the module ID. But I didn't prove it or implement it. Since I want to make the patch itself as small as possible. We can make it in the future if we want. Another change in the patch is the new concept Decl Index, which means the index of the very big array `DeclsLoaded` in ASTReader. Previously, the index of a loaded declaration is simply the Decl ID minus PREDEFINED_DECL_NUMs. So there are some places they got used ambiguously. But this patch tried to split these two concepts. As https://github.com/llvm/llvm-project/pull/86912 did, the change will increase the on-disk PCM file sizes. As the declaration ID may be the most IDs in the PCM file, this can have the biggest impact on the size. In my experiments, this change will bring 6.6% increase of the on-disk PCM size. No compile-time performance regression observed. Given the benefits in the motivation example, I think the cost is worthwhile. --- clang/include/clang/AST/ASTUnresolvedSet.h | 9 +- clang/include/clang/AST/DeclAccessPair.h | 30 +++- clang/include/clang/AST/DeclBase.h | 17 +- clang/include/clang/AST/DeclID.h | 18 +- clang/include/clang/AST/UnresolvedSet.h | 1 + .../include/clang/Serialization/ASTBitCodes.h | 29 +++- clang/include/clang/Serialization/ASTReader.h | 36 ++-- .../include/clang/Serialization/ModuleFile.h | 18 +- .../clang/Serialization/ModuleManager.h | 2 +- clang/lib/AST/DeclBase.cpp | 40 ++++- clang/lib/AST/DeclCXX.cpp | 4 +- clang/lib/Serialization/ASTReader.cpp | 163 ++++++++++-------- clang/lib/Serialization/ASTReaderDecl.cpp | 16 +- clang/lib/Serialization/ASTWriter.cpp | 7 +- clang/lib/Serialization/ModuleFile.cpp | 3 +- .../Modules/no-transitive-decls-change.cppm | 112 ++++++++++++ 16 files changed, 338 insertions(+), 167 deletions(-) create mode 100644 clang/test/Modules/no-transitive-decls-change.cppm diff --git a/clang/include/clang/AST/ASTUnresolvedSet.h b/clang/include/clang/AST/ASTUnresolvedSet.h index 398ffb188c95..dcce3bc63df2 100644 --- a/clang/include/clang/AST/ASTUnresolvedSet.h +++ b/clang/include/clang/AST/ASTUnresolvedSet.h @@ -16,6 +16,7 @@ #include "clang/AST/ASTVector.h" #include "clang/AST/DeclAccessPair.h" +#include "clang/AST/DeclID.h" #include "clang/AST/UnresolvedSet.h" #include "clang/Basic/Specifiers.h" #include @@ -56,6 +57,10 @@ public: Decls.push_back(DeclAccessPair::make(D, AS), C); } + void addLazyDecl(ASTContext &C, GlobalDeclID ID, AccessSpecifier AS) { + Decls.push_back(DeclAccessPair::makeLazy(ID.get(), AS), C); + } + /// Replaces the given declaration with the new one, once. /// /// \return true if the set changed @@ -109,10 +114,10 @@ public: void reserve(ASTContext &C, unsigned N) { Impl.reserve(C, N); } - void addLazyDecl(ASTContext &C, uintptr_t ID, AccessSpecifier AS) { + void addLazyDecl(ASTContext &C, GlobalDeclID ID, AccessSpecifier AS) { assert(Impl.empty() || Impl.Decls.isLazy()); Impl.Decls.setLazy(true); - Impl.addDecl(C, reinterpret_cast(ID << 2), AS); + Impl.addLazyDecl(C, ID, AS); } }; diff --git a/clang/include/clang/AST/DeclAccessPair.h b/clang/include/clang/AST/DeclAccessPair.h index 805342c2910a..17f29c3e97ce 100644 --- a/clang/include/clang/AST/DeclAccessPair.h +++ b/clang/include/clang/AST/DeclAccessPair.h @@ -27,9 +27,17 @@ class NamedDecl; /// A POD class for pairing a NamedDecl* with an access specifier. /// Can be put into unions. class DeclAccessPair { - uintptr_t Ptr; // we'd use llvm::PointerUnion, but it isn't trivial + /// Use the lower 2 bit to store AccessSpecifier. Use the higher + /// 61 bit to store the pointer to a NamedDecl or the DeclID to + /// a NamedDecl. If the 3rd bit is set, storing the DeclID, otherwise + /// storing the pointer. + // + // we'd use llvm::PointerUnion, but it isn't trivial + uint64_t Ptr; - enum { Mask = 0x3 }; + enum { ASMask = 0x3, Mask = 0x7 }; + + bool isDeclID() const { return (Ptr >> 2) & 0x1; } public: static DeclAccessPair make(NamedDecl *D, AccessSpecifier AS) { @@ -38,12 +46,22 @@ public: return p; } + static DeclAccessPair makeLazy(uint64_t ID, AccessSpecifier AS) { + DeclAccessPair p; + p.Ptr = (ID << 3) | (0x1 << 2) | uint64_t(AS); + return p; + } + + uint64_t getDeclID() const { + assert(isDeclID()); + return (~Mask & Ptr) >> 3; + } + NamedDecl *getDecl() const { + assert(!isDeclID()); return reinterpret_cast(~Mask & Ptr); } - AccessSpecifier getAccess() const { - return AccessSpecifier(Mask & Ptr); - } + AccessSpecifier getAccess() const { return AccessSpecifier(ASMask & Ptr); } void setDecl(NamedDecl *D) { set(D, getAccess()); @@ -52,7 +70,7 @@ public: set(getDecl(), AS); } void set(NamedDecl *D, AccessSpecifier AS) { - Ptr = uintptr_t(AS) | reinterpret_cast(D); + Ptr = uint64_t(AS) | reinterpret_cast(D); } operator NamedDecl*() const { return getDecl(); } diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h index 600ce73c7f01..5f19af1891b7 100644 --- a/clang/include/clang/AST/DeclBase.h +++ b/clang/include/clang/AST/DeclBase.h @@ -708,10 +708,7 @@ public: /// Set the owning module ID. This may only be called for /// deserialized Decls. - void setOwningModuleID(unsigned ID) { - assert(isFromASTFile() && "Only works on a deserialized declaration"); - *((unsigned*)this - 2) = ID; - } + void setOwningModuleID(unsigned ID); public: /// Determine the availability of the given declaration. @@ -784,19 +781,11 @@ public: /// Retrieve the global declaration ID associated with this /// declaration, which specifies where this Decl was loaded from. - GlobalDeclID getGlobalID() const { - if (isFromASTFile()) - return (*((const GlobalDeclID *)this - 1)); - return GlobalDeclID(); - } + GlobalDeclID getGlobalID() const; /// Retrieve the global ID of the module that owns this particular /// declaration. - unsigned getOwningModuleID() const { - if (isFromASTFile()) - return *((const unsigned*)this - 2); - return 0; - } + unsigned getOwningModuleID() const; private: Module *getOwningModuleSlow() const; diff --git a/clang/include/clang/AST/DeclID.h b/clang/include/clang/AST/DeclID.h index 614ba06b6386..32d2ed41a374 100644 --- a/clang/include/clang/AST/DeclID.h +++ b/clang/include/clang/AST/DeclID.h @@ -19,6 +19,8 @@ #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/iterator.h" +#include + namespace clang { /// Predefined declaration IDs. @@ -107,12 +109,16 @@ public: /// /// DeclID should only be used directly in serialization. All other users /// should use LocalDeclID or GlobalDeclID. - using DeclID = uint32_t; + using DeclID = uint64_t; protected: DeclIDBase() : ID(PREDEF_DECL_NULL_ID) {} explicit DeclIDBase(DeclID ID) : ID(ID) {} + explicit DeclIDBase(unsigned LocalID, unsigned ModuleFileIndex) { + ID = (DeclID)LocalID | ((DeclID)ModuleFileIndex << 32); + } + public: DeclID get() const { return ID; } @@ -124,6 +130,10 @@ public: bool isInvalid() const { return ID == PREDEF_DECL_NULL_ID; } + unsigned getModuleFileIndex() const { return ID >> 32; } + + unsigned getLocalDeclIndex() const; + friend bool operator==(const DeclIDBase &LHS, const DeclIDBase &RHS) { return LHS.ID == RHS.ID; } @@ -156,6 +166,9 @@ public: LocalDeclID(PredefinedDeclIDs ID) : Base(ID) {} explicit LocalDeclID(DeclID ID) : Base(ID) {} + explicit LocalDeclID(unsigned LocalID, unsigned ModuleFileIndex) + : Base(LocalID, ModuleFileIndex) {} + LocalDeclID &operator++() { ++ID; return *this; @@ -175,6 +188,9 @@ public: GlobalDeclID() : Base() {} explicit GlobalDeclID(DeclID ID) : Base(ID) {} + explicit GlobalDeclID(unsigned LocalID, unsigned ModuleFileIndex) + : Base(LocalID, ModuleFileIndex) {} + // For DeclIDIterator to be able to convert a GlobalDeclID // to a LocalDeclID. explicit operator LocalDeclID() const { return LocalDeclID(this->ID); } diff --git a/clang/include/clang/AST/UnresolvedSet.h b/clang/include/clang/AST/UnresolvedSet.h index ee31be969b6e..1369725ab4e9 100644 --- a/clang/include/clang/AST/UnresolvedSet.h +++ b/clang/include/clang/AST/UnresolvedSet.h @@ -47,6 +47,7 @@ public: // temporaries with defaulted ctors are not zero initialized. UnresolvedSetIterator() : iterator_adaptor_base(nullptr) {} + uint64_t getDeclID() const { return I->getDeclID(); } NamedDecl *getDecl() const { return I->getDecl(); } void setDecl(NamedDecl *ND) const { return I->setDecl(ND); } AccessSpecifier getAccess() const { return I->getAccess(); } diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h index f59ff6af4c76..52a6c5e10f80 100644 --- a/clang/include/clang/Serialization/ASTBitCodes.h +++ b/clang/include/clang/Serialization/ASTBitCodes.h @@ -255,6 +255,12 @@ public: } }; +// The unaligned decl ID used in the Blobs of bistreams. +using unaligned_decl_id_t = + llvm::support::detail::packed_endian_specific_integral< + serialization::DeclID, llvm::endianness::native, + llvm::support::unaligned>; + /// The number of predefined preprocessed entity IDs. const unsigned int NUM_PREDEF_PP_ENTITY_IDS = 1; @@ -1980,33 +1986,44 @@ enum CleanupObjectKind { COK_Block, COK_CompoundLiteral }; /// Describes the categories of an Objective-C class. struct ObjCCategoriesInfo { - // The ID of the definition - LocalDeclID DefinitionID; + // The ID of the definition. Use unaligned_decl_id_t to keep + // ObjCCategoriesInfo 32-bit aligned. + unaligned_decl_id_t DefinitionID; // Offset into the array of category lists. unsigned Offset; + ObjCCategoriesInfo() = default; + ObjCCategoriesInfo(LocalDeclID ID, unsigned Offset) + : DefinitionID(ID.get()), Offset(Offset) {} + + LocalDeclID getDefinitionID() const { return LocalDeclID(DefinitionID); } + friend bool operator<(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.DefinitionID < Y.DefinitionID; + return X.getDefinitionID() < Y.getDefinitionID(); } friend bool operator>(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.DefinitionID > Y.DefinitionID; + return X.getDefinitionID() > Y.getDefinitionID(); } friend bool operator<=(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.DefinitionID <= Y.DefinitionID; + return X.getDefinitionID() <= Y.getDefinitionID(); } friend bool operator>=(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.DefinitionID >= Y.DefinitionID; + return X.getDefinitionID() >= Y.getDefinitionID(); } }; +static_assert(alignof(ObjCCategoriesInfo) <= 4); +static_assert(std::is_standard_layout_v && + std::is_trivial_v); + /// A key used when looking up entities by \ref DeclarationName. /// /// Different \ref DeclarationNames are mapped to different keys, but the diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index bea58d60d36c..0a9006223dcb 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -504,12 +504,6 @@ private: /// = I + 1 has already been loaded. llvm::PagedVector DeclsLoaded; - using GlobalDeclMapType = ContinuousRangeMap; - - /// Mapping from global declaration IDs to the module in which the - /// declaration resides. - GlobalDeclMapType GlobalDeclMap; - using FileOffset = std::pair; using FileOffsetsTy = SmallVector; using DeclUpdateOffsetsMap = llvm::DenseMap; @@ -592,10 +586,11 @@ private: struct FileDeclsInfo { ModuleFile *Mod = nullptr; - ArrayRef Decls; + ArrayRef Decls; FileDeclsInfo() = default; - FileDeclsInfo(ModuleFile *Mod, ArrayRef Decls) + FileDeclsInfo(ModuleFile *Mod, + ArrayRef Decls) : Mod(Mod), Decls(Decls) {} }; @@ -604,11 +599,7 @@ private: /// An array of lexical contents of a declaration context, as a sequence of /// Decl::Kind, DeclID pairs. - using unaligned_decl_id_t = - llvm::support::detail::packed_endian_specific_integral< - serialization::DeclID, llvm::endianness::native, - llvm::support::unaligned>; - using LexicalContents = ArrayRef; + using LexicalContents = ArrayRef; /// Map from a DeclContext to its lexical contents. llvm::DenseMap> @@ -1489,10 +1480,11 @@ private: unsigned ClientLoadCapabilities); public: - class ModuleDeclIterator : public llvm::iterator_adaptor_base< - ModuleDeclIterator, const LocalDeclID *, - std::random_access_iterator_tag, const Decl *, - ptrdiff_t, const Decl *, const Decl *> { + class ModuleDeclIterator + : public llvm::iterator_adaptor_base< + ModuleDeclIterator, const serialization::unaligned_decl_id_t *, + std::random_access_iterator_tag, const Decl *, ptrdiff_t, + const Decl *, const Decl *> { ASTReader *Reader = nullptr; ModuleFile *Mod = nullptr; @@ -1500,11 +1492,11 @@ public: ModuleDeclIterator() : iterator_adaptor_base(nullptr) {} ModuleDeclIterator(ASTReader *Reader, ModuleFile *Mod, - const LocalDeclID *Pos) + const serialization::unaligned_decl_id_t *Pos) : iterator_adaptor_base(Pos), Reader(Reader), Mod(Mod) {} value_type operator*() const { - return Reader->GetDecl(Reader->getGlobalDeclID(*Mod, *I)); + return Reader->GetDecl(Reader->getGlobalDeclID(*Mod, (LocalDeclID)*I)); } value_type operator->() const { return **this; } @@ -1544,6 +1536,9 @@ private: StringRef Arg2 = StringRef(), StringRef Arg3 = StringRef()) const; void Error(llvm::Error &&Err) const; + /// Translate a \param GlobalDeclID to the index of DeclsLoaded array. + unsigned translateGlobalDeclIDToIndex(GlobalDeclID ID) const; + public: /// Load the AST file and validate its contents against the given /// Preprocessor. @@ -1915,7 +1910,8 @@ public: /// Retrieve the module file that owns the given declaration, or NULL /// if the declaration is not from a module file. - ModuleFile *getOwningModuleFile(const Decl *D); + ModuleFile *getOwningModuleFile(const Decl *D) const; + ModuleFile *getOwningModuleFile(GlobalDeclID ID) const; /// Returns the source location for the decl \p ID. SourceLocation getSourceLocationForDeclID(GlobalDeclID ID); diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 992d26a8b88c..56193d44dd6f 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -454,23 +454,11 @@ public: /// by the declaration ID (-1). const DeclOffset *DeclOffsets = nullptr; - /// Base declaration ID for declarations local to this module. - serialization::DeclID BaseDeclID = 0; - - /// Remapping table for declaration IDs in this module. - ContinuousRangeMap DeclRemap; - - /// Mapping from the module files that this module file depends on - /// to the base declaration ID for that module as it is understood within this - /// module. - /// - /// This is effectively a reverse global-to-local mapping for declaration - /// IDs, so that we can interpret a true global ID (for this translation unit) - /// as a local ID (for this module file). - llvm::DenseMap GlobalToLocalDeclIDs; + /// Base declaration index in ASTReader for declarations local to this module. + unsigned BaseDeclIndex = 0; /// Array of file-level DeclIDs sorted by file. - const LocalDeclID *FileSortedDecls = nullptr; + const serialization::unaligned_decl_id_t *FileSortedDecls = nullptr; unsigned NumFileSortedDecls = 0; /// Array of category list location information within this diff --git a/clang/include/clang/Serialization/ModuleManager.h b/clang/include/clang/Serialization/ModuleManager.h index d770bc52eaf4..f898dab39f06 100644 --- a/clang/include/clang/Serialization/ModuleManager.h +++ b/clang/include/clang/Serialization/ModuleManager.h @@ -45,7 +45,7 @@ namespace serialization { /// Manages the set of modules loaded by an AST reader. class ModuleManager { /// The chain of AST files, in the order in which we started to load - /// them (this order isn't really useful for anything). + /// them. SmallVector, 2> Chain; /// The chain of non-module PCH files. The first entry is the one named diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp index 1e9c879e371b..7f1ed9c691e9 100644 --- a/clang/lib/AST/DeclBase.cpp +++ b/clang/lib/AST/DeclBase.cpp @@ -74,18 +74,17 @@ void *Decl::operator new(std::size_t Size, const ASTContext &Context, GlobalDeclID ID, std::size_t Extra) { // Allocate an extra 8 bytes worth of storage, which ensures that the // resulting pointer will still be 8-byte aligned. - static_assert(sizeof(unsigned) * 2 >= alignof(Decl), - "Decl won't be misaligned"); + static_assert(sizeof(uint64_t) >= alignof(Decl), "Decl won't be misaligned"); void *Start = Context.Allocate(Size + Extra + 8); void *Result = (char*)Start + 8; - unsigned *PrefixPtr = (unsigned *)Result - 2; + uint64_t *PrefixPtr = (uint64_t *)Result - 1; - // Zero out the first 4 bytes; this is used to store the owning module ID. - PrefixPtr[0] = 0; + *PrefixPtr = ID.get(); - // Store the global declaration ID in the second 4 bytes. - PrefixPtr[1] = ID.get(); + // We leave the upper 16 bits to store the module IDs. 48 bits should be + // sufficient to store a declaration ID. + assert(*PrefixPtr < llvm::maskTrailingOnes(48)); return Result; } @@ -111,6 +110,29 @@ void *Decl::operator new(std::size_t Size, const ASTContext &Ctx, return ::operator new(Size + Extra, Ctx); } +GlobalDeclID Decl::getGlobalID() const { + if (!isFromASTFile()) + return GlobalDeclID(); + // See the comments in `Decl::operator new` for details. + uint64_t ID = *((const uint64_t *)this - 1); + return GlobalDeclID(ID & llvm::maskTrailingOnes(48)); +} + +unsigned Decl::getOwningModuleID() const { + if (!isFromASTFile()) + return 0; + + uint64_t ID = *((const uint64_t *)this - 1); + return ID >> 48; +} + +void Decl::setOwningModuleID(unsigned ID) { + assert(isFromASTFile() && "Only works on a deserialized declaration"); + uint64_t *IDAddress = (uint64_t *)this - 1; + *IDAddress &= llvm::maskTrailingOnes(48); + *IDAddress |= (uint64_t)ID << 48; +} + Module *Decl::getOwningModuleSlow() const { assert(isFromASTFile() && "Not from AST file?"); return getASTContext().getExternalSource()->getModule(getOwningModuleID()); @@ -2163,3 +2185,7 @@ DependentDiagnostic *DependentDiagnostic::Create(ASTContext &C, return DD; } + +unsigned DeclIDBase::getLocalDeclIndex() const { + return ID & llvm::maskTrailingOnes(32); +} diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp index 75c441293d62..7f2c786547b9 100644 --- a/clang/lib/AST/DeclCXX.cpp +++ b/clang/lib/AST/DeclCXX.cpp @@ -68,8 +68,8 @@ void LazyASTUnresolvedSet::getFromExternalSource(ASTContext &C) const { assert(Source && "getFromExternalSource with no external source"); for (ASTUnresolvedSet::iterator I = Impl.begin(); I != Impl.end(); ++I) - I.setDecl(cast(Source->GetExternalDecl( - GlobalDeclID(reinterpret_cast(I.getDecl()) >> 2)))); + I.setDecl( + cast(Source->GetExternalDecl(GlobalDeclID(I.getDeclID())))); Impl.Decls.setLazy(false); } diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 33cea32c3be6..59338b44db32 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -1658,7 +1658,7 @@ bool ASTReader::ReadSLocEntry(int ID) { unsigned NumFileDecls = Record[7]; if (NumFileDecls && ContextObj) { - const LocalDeclID *FirstDecl = F->FileSortedDecls + Record[6]; + const unaligned_decl_id_t *FirstDecl = F->FileSortedDecls + Record[6]; assert(F->FileSortedDecls && "FILE_SORTED_DECLS not encountered yet ?"); FileDeclIDs[FID] = FileDeclsInfo(F, llvm::ArrayRef(FirstDecl, NumFileDecls)); @@ -3377,26 +3377,11 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F, "duplicate DECL_OFFSET record in AST file"); F.DeclOffsets = (const DeclOffset *)Blob.data(); F.LocalNumDecls = Record[0]; - unsigned LocalBaseDeclID = Record[1]; - F.BaseDeclID = getTotalNumDecls(); - - if (F.LocalNumDecls > 0) { - // Introduce the global -> local mapping for declarations within this - // module. - GlobalDeclMap.insert(std::make_pair( - GlobalDeclID(getTotalNumDecls() + NUM_PREDEF_DECL_IDS), &F)); - - // Introduce the local -> global mapping for declarations within this - // module. - F.DeclRemap.insertOrReplace( - std::make_pair(LocalBaseDeclID, F.BaseDeclID - LocalBaseDeclID)); - - // Introduce the global -> local mapping for declarations within this - // module. - F.GlobalToLocalDeclIDs[&F] = LocalBaseDeclID; + F.BaseDeclIndex = getTotalNumDecls(); + if (F.LocalNumDecls > 0) DeclsLoaded.resize(DeclsLoaded.size() + F.LocalNumDecls); - } + break; } @@ -3631,7 +3616,7 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F, break; case FILE_SORTED_DECLS: - F.FileSortedDecls = (const LocalDeclID *)Blob.data(); + F.FileSortedDecls = (const unaligned_decl_id_t *)Blob.data(); F.NumFileSortedDecls = Record[0]; break; @@ -4058,7 +4043,6 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { RemapBuilder PreprocessedEntityRemap(F.PreprocessedEntityRemap); RemapBuilder SubmoduleRemap(F.SubmoduleRemap); RemapBuilder SelectorRemap(F.SelectorRemap); - RemapBuilder DeclRemap(F.DeclRemap); RemapBuilder TypeRemap(F.TypeRemap); auto &ImportedModuleVector = F.TransitiveImports; @@ -4097,8 +4081,6 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { endian::readNext(Data); uint32_t SelectorIDOffset = endian::readNext(Data); - uint32_t DeclIDOffset = - endian::readNext(Data); uint32_t TypeIndexOffset = endian::readNext(Data); @@ -4116,11 +4098,7 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { PreprocessedEntityRemap); mapOffset(SubmoduleIDOffset, OM->BaseSubmoduleID, SubmoduleRemap); mapOffset(SelectorIDOffset, OM->BaseSelectorID, SelectorRemap); - mapOffset(DeclIDOffset, OM->BaseDeclID, DeclRemap); mapOffset(TypeIndexOffset, OM->BaseTypeIndex, TypeRemap); - - // Global -> local mappings. - F.GlobalToLocalDeclIDs[OM] = DeclIDOffset; } } @@ -4645,7 +4623,7 @@ ASTReader::ASTReadResult ASTReader::ReadAST(StringRef FileName, ModuleKind Type, // that we load any additional categories. if (ContextObj) { for (unsigned I = 0, N = ObjCClassesLoaded.size(); I != N; ++I) { - loadObjCCategories(GlobalDeclID(ObjCClassesLoaded[I]->getGlobalID()), + loadObjCCategories(ObjCClassesLoaded[I]->getGlobalID(), ObjCClassesLoaded[I], PreviousGeneration); } } @@ -7644,18 +7622,25 @@ CXXBaseSpecifier *ASTReader::GetExternalCXXBaseSpecifiers(uint64_t Offset) { GlobalDeclID ASTReader::getGlobalDeclID(ModuleFile &F, LocalDeclID LocalID) const { - DeclID ID = LocalID.get(); - if (ID < NUM_PREDEF_DECL_IDS) - return GlobalDeclID(ID); + if (LocalID.get() < NUM_PREDEF_DECL_IDS) + return GlobalDeclID(LocalID.get()); + + unsigned OwningModuleFileIndex = LocalID.getModuleFileIndex(); + DeclID ID = LocalID.getLocalDeclIndex(); if (!F.ModuleOffsetMap.empty()) ReadModuleOffsetMap(F); - ContinuousRangeMap::iterator I = - F.DeclRemap.find(ID - NUM_PREDEF_DECL_IDS); - assert(I != F.DeclRemap.end() && "Invalid index into decl index remap"); + ModuleFile *OwningModuleFile = + OwningModuleFileIndex == 0 + ? &F + : F.TransitiveImports[OwningModuleFileIndex - 1]; + + if (OwningModuleFileIndex == 0) + ID -= NUM_PREDEF_DECL_IDS; - return GlobalDeclID(ID + I->second); + uint64_t NewModuleFileIndex = OwningModuleFile->Index + 1; + return GlobalDeclID(ID, NewModuleFileIndex); } bool ASTReader::isDeclIDFromModule(GlobalDeclID ID, ModuleFile &M) const { @@ -7663,31 +7648,33 @@ bool ASTReader::isDeclIDFromModule(GlobalDeclID ID, ModuleFile &M) const { if (ID.get() < NUM_PREDEF_DECL_IDS) return false; - return ID.get() - NUM_PREDEF_DECL_IDS >= M.BaseDeclID && - ID.get() - NUM_PREDEF_DECL_IDS < M.BaseDeclID + M.LocalNumDecls; + unsigned ModuleFileIndex = ID.getModuleFileIndex(); + return M.Index == ModuleFileIndex - 1; +} + +ModuleFile *ASTReader::getOwningModuleFile(GlobalDeclID ID) const { + // Predefined decls aren't from any module. + if (ID.get() < NUM_PREDEF_DECL_IDS) + return nullptr; + + uint64_t ModuleFileIndex = ID.getModuleFileIndex(); + assert(ModuleFileIndex && "Untranslated Local Decl?"); + + return &getModuleManager()[ModuleFileIndex - 1]; } -ModuleFile *ASTReader::getOwningModuleFile(const Decl *D) { +ModuleFile *ASTReader::getOwningModuleFile(const Decl *D) const { if (!D->isFromASTFile()) return nullptr; - GlobalDeclMapType::const_iterator I = - GlobalDeclMap.find(GlobalDeclID(D->getGlobalID())); - assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); - return I->second; + + return getOwningModuleFile(D->getGlobalID()); } SourceLocation ASTReader::getSourceLocationForDeclID(GlobalDeclID ID) { if (ID.get() < NUM_PREDEF_DECL_IDS) return SourceLocation(); - unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; - - if (Index > DeclsLoaded.size()) { - Error("declaration ID out-of-range for AST file"); - return SourceLocation(); - } - - if (Decl *D = DeclsLoaded[Index]) + if (Decl *D = GetExistingDecl(ID)) return D->getLocation(); SourceLocation Loc; @@ -7754,8 +7741,19 @@ static Decl *getPredefinedDecl(ASTContext &Context, PredefinedDeclIDs ID) { llvm_unreachable("PredefinedDeclIDs unknown enum value"); } +unsigned ASTReader::translateGlobalDeclIDToIndex(GlobalDeclID GlobalID) const { + ModuleFile *OwningModuleFile = getOwningModuleFile(GlobalID); + if (!OwningModuleFile) { + assert(GlobalID.get() < NUM_PREDEF_DECL_IDS && "Untransalted Global ID?"); + return GlobalID.get(); + } + + return OwningModuleFile->BaseDeclIndex + GlobalID.getLocalDeclIndex(); +} + Decl *ASTReader::GetExistingDecl(GlobalDeclID ID) { assert(ContextObj && "reading decl with no AST context"); + if (ID.get() < NUM_PREDEF_DECL_IDS) { Decl *D = getPredefinedDecl(*ContextObj, (PredefinedDeclIDs)ID); if (D) { @@ -7768,7 +7766,7 @@ Decl *ASTReader::GetExistingDecl(GlobalDeclID ID) { return D; } - unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; + unsigned Index = translateGlobalDeclIDToIndex(ID); if (Index >= DeclsLoaded.size()) { assert(0 && "declaration ID out-of-range for AST file"); @@ -7783,7 +7781,7 @@ Decl *ASTReader::GetDecl(GlobalDeclID ID) { if (ID.get() < NUM_PREDEF_DECL_IDS) return GetExistingDecl(ID); - unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; + unsigned Index = translateGlobalDeclIDToIndex(ID); if (Index >= DeclsLoaded.size()) { assert(0 && "declaration ID out-of-range for AST file"); @@ -7802,20 +7800,31 @@ Decl *ASTReader::GetDecl(GlobalDeclID ID) { LocalDeclID ASTReader::mapGlobalIDToModuleFileGlobalID(ModuleFile &M, GlobalDeclID GlobalID) { - DeclID ID = GlobalID.get(); - if (ID < NUM_PREDEF_DECL_IDS) + if (GlobalID.get() < NUM_PREDEF_DECL_IDS) + return LocalDeclID(GlobalID.get()); + + if (!M.ModuleOffsetMap.empty()) + ReadModuleOffsetMap(M); + + ModuleFile *Owner = getOwningModuleFile(GlobalID); + DeclID ID = GlobalID.getLocalDeclIndex(); + + if (Owner == &M) { + ID += NUM_PREDEF_DECL_IDS; return LocalDeclID(ID); + } - GlobalDeclMapType::const_iterator I = GlobalDeclMap.find(GlobalID); - assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); - ModuleFile *Owner = I->second; + uint64_t OrignalModuleFileIndex = 0; + for (unsigned I = 0; I < M.TransitiveImports.size(); I++) + if (M.TransitiveImports[I] == Owner) { + OrignalModuleFileIndex = I + 1; + break; + } - llvm::DenseMap::iterator Pos = - M.GlobalToLocalDeclIDs.find(Owner); - if (Pos == M.GlobalToLocalDeclIDs.end()) + if (!OrignalModuleFileIndex) return LocalDeclID(); - return LocalDeclID(ID - Owner->BaseDeclID + Pos->second); + return LocalDeclID(ID, OrignalModuleFileIndex); } GlobalDeclID ASTReader::ReadDeclID(ModuleFile &F, const RecordData &Record, @@ -7894,32 +7903,34 @@ void ASTReader::FindExternalLexicalDecls( namespace { -class DeclIDComp { +class UnalignedDeclIDComp { ASTReader &Reader; ModuleFile &Mod; public: - DeclIDComp(ASTReader &Reader, ModuleFile &M) : Reader(Reader), Mod(M) {} + UnalignedDeclIDComp(ASTReader &Reader, ModuleFile &M) + : Reader(Reader), Mod(M) {} - bool operator()(LocalDeclID L, LocalDeclID R) const { + bool operator()(unaligned_decl_id_t L, unaligned_decl_id_t R) const { SourceLocation LHS = getLocation(L); SourceLocation RHS = getLocation(R); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - bool operator()(SourceLocation LHS, LocalDeclID R) const { + bool operator()(SourceLocation LHS, unaligned_decl_id_t R) const { SourceLocation RHS = getLocation(R); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - bool operator()(LocalDeclID L, SourceLocation RHS) const { + bool operator()(unaligned_decl_id_t L, SourceLocation RHS) const { SourceLocation LHS = getLocation(L); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - SourceLocation getLocation(LocalDeclID ID) const { + SourceLocation getLocation(unaligned_decl_id_t ID) const { return Reader.getSourceManager().getFileLoc( - Reader.getSourceLocationForDeclID(Reader.getGlobalDeclID(Mod, ID))); + Reader.getSourceLocationForDeclID( + Reader.getGlobalDeclID(Mod, (LocalDeclID)ID))); } }; @@ -7942,8 +7953,8 @@ void ASTReader::FindFileRegionDecls(FileID File, BeginLoc = SM.getLocForStartOfFile(File).getLocWithOffset(Offset); SourceLocation EndLoc = BeginLoc.getLocWithOffset(Length); - DeclIDComp DIDComp(*this, *DInfo.Mod); - ArrayRef::iterator BeginIt = + UnalignedDeclIDComp DIDComp(*this, *DInfo.Mod); + ArrayRef::iterator BeginIt = llvm::lower_bound(DInfo.Decls, BeginLoc, DIDComp); if (BeginIt != DInfo.Decls.begin()) --BeginIt; @@ -7952,17 +7963,18 @@ void ASTReader::FindFileRegionDecls(FileID File, // to backtrack until we find it otherwise we will fail to report that the // region overlaps with an objc container. while (BeginIt != DInfo.Decls.begin() && - GetDecl(getGlobalDeclID(*DInfo.Mod, *BeginIt)) + GetDecl(getGlobalDeclID(*DInfo.Mod, (LocalDeclID)(*BeginIt))) ->isTopLevelDeclInObjCContainer()) --BeginIt; - ArrayRef::iterator EndIt = + ArrayRef::iterator EndIt = llvm::upper_bound(DInfo.Decls, EndLoc, DIDComp); if (EndIt != DInfo.Decls.end()) ++EndIt; - for (ArrayRef::iterator DIt = BeginIt; DIt != EndIt; ++DIt) - Decls.push_back(GetDecl(getGlobalDeclID(*DInfo.Mod, *DIt))); + for (ArrayRef::iterator DIt = BeginIt; DIt != EndIt; + ++DIt) + Decls.push_back(GetDecl(getGlobalDeclID(*DInfo.Mod, (LocalDeclID)(*DIt)))); } bool @@ -8169,7 +8181,6 @@ LLVM_DUMP_METHOD void ASTReader::dump() { dumpModuleIDMap("Global bit offset map", GlobalBitOffsetsMap); dumpModuleIDMap("Global source location entry map", GlobalSLocEntryMap); dumpModuleIDMap("Global type map", GlobalTypeMap); - dumpModuleIDMap("Global declaration map", GlobalDeclMap); dumpModuleIDMap("Global identifier map", GlobalIdentifierMap); dumpModuleIDMap("Global macro map", GlobalMacroMap); dumpModuleIDMap("Global submodule map", GlobalSubmoduleMap); @@ -9185,7 +9196,7 @@ void ASTRecordReader::readUnresolvedSet(LazyASTUnresolvedSet &Set) { while (NumDecls--) { GlobalDeclID ID = readDeclID(); AccessSpecifier AS = (AccessSpecifier) readInt(); - Set.addLazyDecl(getContext(), ID.get(), AS); + Set.addLazyDecl(getContext(), ID, AS); } } diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp index 765c083ce8df..cf2dc32e30b9 100644 --- a/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/clang/lib/Serialization/ASTReaderDecl.cpp @@ -2924,7 +2924,7 @@ void ASTDeclReader::mergeTemplatePattern(RedeclarableTemplateDecl *D, auto *ExistingPattern = Existing->getTemplatedDecl(); RedeclarableResult Result( /*MergeWith*/ ExistingPattern, - GlobalDeclID(DPattern->getCanonicalDecl()->getGlobalID()), IsKeyDecl); + DPattern->getCanonicalDecl()->getGlobalID(), IsKeyDecl); if (auto *DClass = dyn_cast(DPattern)) { // Merge with any existing definition. @@ -3245,11 +3245,10 @@ bool ASTReader::isConsumerInterestedIn(Decl *D) { /// Get the correct cursor and offset for loading a declaration. ASTReader::RecordLocation ASTReader::DeclCursorForID(GlobalDeclID ID, SourceLocation &Loc) { - GlobalDeclMapType::iterator I = GlobalDeclMap.find(ID); - assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); - ModuleFile *M = I->second; - const DeclOffset &DOffs = - M->DeclOffsets[ID.get() - M->BaseDeclID - NUM_PREDEF_DECL_IDS]; + ModuleFile *M = getOwningModuleFile(ID); + assert(M); + unsigned LocalDeclIndex = ID.getLocalDeclIndex(); + const DeclOffset &DOffs = M->DeclOffsets[LocalDeclIndex]; Loc = ReadSourceLocation(*M, DOffs.getRawLoc()); return RecordLocation(M, DOffs.getBitOffset(M->DeclsBlockStartOffset)); } @@ -3792,7 +3791,6 @@ void ASTReader::markIncompleteDeclChain(Decl *D) { /// Read the declaration at the given offset from the AST file. Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) { - unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; SourceLocation DeclLoc; RecordLocation Loc = DeclCursorForID(ID, DeclLoc); llvm::BitstreamCursor &DeclsCursor = Loc.F->DeclsCursor; @@ -4123,7 +4121,7 @@ Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) { } assert(D && "Unknown declaration reading AST file"); - LoadedDecl(Index, D); + LoadedDecl(translateGlobalDeclIDToIndex(ID), D); // Set the DeclContext before doing any deserialization, to make sure internal // calls to Decl::getASTContext() by Decl's methods will find the // TranslationUnitDecl without crashing. @@ -4449,7 +4447,7 @@ namespace { M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap, Compare); if (Result == M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap || - Result->DefinitionID != LocalID) { + Result->getDefinitionID() != LocalID) { // We didn't find anything. If the class definition is in this module // file, then the module files it depends on cannot have any categories, // so suppress further lookup. diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 953990a4aca6..ee3e687636e6 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -3357,12 +3357,10 @@ void ASTWriter::WriteTypeDeclOffsets() { Abbrev = std::make_shared(); Abbrev->Add(BitCodeAbbrevOp(DECL_OFFSET)); Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of declarations - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // base decl ID Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // declarations block unsigned DeclOffsetAbbrev = Stream.EmitAbbrev(std::move(Abbrev)); { - RecordData::value_type Record[] = {DECL_OFFSET, DeclOffsets.size(), - FirstDeclID.get() - NUM_PREDEF_DECL_IDS}; + RecordData::value_type Record[] = {DECL_OFFSET, DeclOffsets.size()}; Stream.EmitRecordWithBlob(DeclOffsetAbbrev, Record, bytes(DeclOffsets)); } } @@ -5423,7 +5421,6 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, M.NumPreprocessedEntities); writeBaseIDOrNone(M.BaseSubmoduleID, M.LocalNumSubmodules); writeBaseIDOrNone(M.BaseSelectorID, M.LocalNumSelectors); - writeBaseIDOrNone(M.BaseDeclID, M.LocalNumDecls); writeBaseIDOrNone(M.BaseTypeIndex, M.LocalNumTypes); } } @@ -6618,13 +6615,11 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) { // Note, this will get called multiple times, once one the reader starts up // and again each time it's done reading a PCH or module. - FirstDeclID = LocalDeclID(NUM_PREDEF_DECL_IDS + Chain->getTotalNumDecls()); FirstTypeID = NUM_PREDEF_TYPE_IDS + Chain->getTotalNumTypes(); FirstIdentID = NUM_PREDEF_IDENT_IDS + Chain->getTotalNumIdentifiers(); FirstMacroID = NUM_PREDEF_MACRO_IDS + Chain->getTotalNumMacros(); FirstSubmoduleID = NUM_PREDEF_SUBMODULE_IDS + Chain->getTotalNumSubmodules(); FirstSelectorID = NUM_PREDEF_SELECTOR_IDS + Chain->getTotalNumSelectors(); - NextDeclID = FirstDeclID; NextTypeID = FirstTypeID; NextIdentID = FirstIdentID; NextMacroID = FirstMacroID; diff --git a/clang/lib/Serialization/ModuleFile.cpp b/clang/lib/Serialization/ModuleFile.cpp index 2c42d33a8f5d..f64a59bd9489 100644 --- a/clang/lib/Serialization/ModuleFile.cpp +++ b/clang/lib/Serialization/ModuleFile.cpp @@ -87,7 +87,6 @@ LLVM_DUMP_METHOD void ModuleFile::dump() { << " Number of types: " << LocalNumTypes << '\n'; dumpLocalRemap("Type index local -> global map", TypeRemap); - llvm::errs() << " Base decl ID: " << BaseDeclID << '\n' + llvm::errs() << " Base decl index: " << BaseDeclIndex << '\n' << " Number of decls: " << LocalNumDecls << '\n'; - dumpLocalRemap("Decl ID local -> global map", DeclRemap); } diff --git a/clang/test/Modules/no-transitive-decls-change.cppm b/clang/test/Modules/no-transitive-decls-change.cppm new file mode 100644 index 000000000000..42ac061bc90b --- /dev/null +++ b/clang/test/Modules/no-transitive-decls-change.cppm @@ -0,0 +1,112 @@ +// Testing that changing a declaration in an unused module file won't change +// the BMI of the current module file. +// +// RUN: rm -rf %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/m-partA.cppm -emit-reduced-module-interface -o %t/m-partA.pcm +// RUN: %clang_cc1 -std=c++20 %t/m-partA.v1.cppm -emit-reduced-module-interface -o \ +// RUN: %t/m-partA.v1.pcm +// RUN: %clang_cc1 -std=c++20 %t/m-partB.cppm -emit-reduced-module-interface -o %t/m-partB.pcm +// RUN: %clang_cc1 -std=c++20 %t/m.cppm -emit-reduced-module-interface -o %t/m.pcm \ +// RUN: -fmodule-file=m:partA=%t/m-partA.pcm -fmodule-file=m:partB=%t/m-partB.pcm +// RUN: %clang_cc1 -std=c++20 %t/m.cppm -emit-reduced-module-interface -o %t/m.v1.pcm \ +// RUN: -fmodule-file=m:partA=%t/m-partA.v1.pcm -fmodule-file=m:partB=%t/m-partB.pcm +// +// RUN: %clang_cc1 -std=c++20 %t/useBOnly.cppm -emit-reduced-module-interface -o %t/useBOnly.pcm \ +// RUN: -fmodule-file=m=%t/m.pcm -fmodule-file=m:partA=%t/m-partA.pcm \ +// RUN: -fmodule-file=m:partB=%t/m-partB.pcm +// RUN: %clang_cc1 -std=c++20 %t/useBOnly.cppm -emit-reduced-module-interface -o %t/useBOnly.v1.pcm \ +// RUN: -fmodule-file=m=%t/m.v1.pcm -fmodule-file=m:partA=%t/m-partA.v1.pcm \ +// RUN: -fmodule-file=m:partB=%t/m-partB.pcm +// Since useBOnly only uses partB from module M, the change in partA shouldn't affect +// useBOnly. +// RUN: diff %t/useBOnly.pcm %t/useBOnly.v1.pcm &> /dev/null + +//--- m-partA.cppm +export module m:partA; + +namespace A_Impl { + inline int getAImpl() { + return 43; + } + + inline int getA2Impl() { + return 43; + } +} + +namespace A { + using A_Impl::getAImpl; +} + +export inline int getA() { + return 43; +} + +export inline int getA2(int) { + return 88; +} + +//--- m-partA.v1.cppm +export module m:partA; + +namespace A_Impl { + inline int getAImpl() { + return 43; + } + + inline int getA2Impl() { + return 43; + } +} + +namespace A { + using A_Impl::getAImpl; + // Adding a new declaration without introducing a new declaration name. + using A_Impl::getA2Impl; +} + +inline int getA() { + return 43; +} + +inline int getA2(int) { + return 88; +} + +// Now we add a new declaration without introducing new identifier and new types. +// The consuming module which didn't use m:partA completely is expected to be +// not changed. +inline int getA(int) { + return 88; +} + +//--- m-partB.cppm +export module m:partB; + +export inline int getB() { + return 430; +} + +//--- m.cppm +export module m; +export import :partA; +export import :partB; + +//--- useBOnly.cppm +export module useBOnly; +import m; + +export inline int get() { + return getB(); +} + +//--- useAOnly.cppm +export module useAOnly; +import m; + +export inline int get() { + A a; + return a.getValue(); +} -- GitLab From 222e0a042c1c7c3b1aec7557bcb60d1543c1737a Mon Sep 17 00:00:00 2001 From: Jon Roelofs Date: Thu, 6 Jun 2024 20:21:54 -0700 Subject: [PATCH 183/462] [llvm][ScheduleDAG] Re-arrange SUnit's members to make it smaller (#94547) before: ``` *** Dumping AST Record Layout 0 | class llvm::SUnit 0 | SDNode * Node 8 | MachineInstr * Instr 16 | SUnit * OrigNode 24 | const MCSchedClassDesc * SchedClass 32 | class llvm::SmallVector Preds 32 | class llvm::SmallVectorImpl (base) 32 | class llvm::SmallVectorTemplateBase (base) 32 | class llvm::SmallVectorTemplateCommon (base) 32 | class llvm::SmallVectorBase (base) 32 | void * BeginX 40 | unsigned int Size 44 | unsigned int Capacity 48 | struct llvm::SmallVectorStorage (base) 48 | char[64] InlineElts 112 | class llvm::SmallVector Succs 112 | class llvm::SmallVectorImpl (base) 112 | class llvm::SmallVectorTemplateBase (base) 112 | class llvm::SmallVectorTemplateCommon (base) 112 | class llvm::SmallVectorBase (base) 112 | void * BeginX 120 | unsigned int Size 124 | unsigned int Capacity 128 | struct llvm::SmallVectorStorage (base) 128 | char[64] InlineElts 192 | unsigned int NodeNum 196 | unsigned int NodeQueueId 200 | unsigned int NumPreds 204 | unsigned int NumSuccs 208 | unsigned int NumPredsLeft 212 | unsigned int NumSuccsLeft 216 | unsigned int WeakPredsLeft 220 | unsigned int WeakSuccsLeft 224 | unsigned short NumRegDefsLeft 226 | unsigned short Latency 228:0-0 | _Bool isVRegCycle 228:1-1 | _Bool isCall 228:2-2 | _Bool isCallOp 228:3-3 | _Bool isTwoAddress 228:4-4 | _Bool isCommutable 228:5-5 | _Bool hasPhysRegUses 228:6-6 | _Bool hasPhysRegDefs 228:7-7 | _Bool hasPhysRegClobbers 229:0-0 | _Bool isPending 229:1-1 | _Bool isAvailable 229:2-2 | _Bool isScheduled 229:3-3 | _Bool isScheduleHigh 229:4-4 | _Bool isScheduleLow 229:5-5 | _Bool isCloned 229:6-6 | _Bool isUnbuffered 229:7-7 | _Bool hasReservedResource 232 | Sched::Preference SchedulingPref 236:0-0 | _Bool isDepthCurrent 236:1-1 | _Bool isHeightCurrent 240 | unsigned int Depth 244 | unsigned int Height 248 | unsigned int TopReadyCycle 252 | unsigned int BotReadyCycle 256 | const TargetRegisterClass * CopyDstRC 264 | const TargetRegisterClass * CopySrcRC | [sizeof=272, dsize=272, align=8, | nvsize=272, nvalign=8] ``` after: ``` *** Dumping AST Record Layout 0 | class llvm::SUnit 0 | union llvm::SUnit::(anonymous at /Users/jonathan_roelofs/llvm-upstream/llvm/include/llvm/CodeGen/ScheduleDAG.h:246:5) 0 | SDNode * Node 0 | MachineInstr * Instr 8 | SUnit * OrigNode 16 | const MCSchedClassDesc * SchedClass 24 | const TargetRegisterClass * CopyDstRC 32 | const TargetRegisterClass * CopySrcRC 40 | class llvm::SmallVector Preds 40 | class llvm::SmallVectorImpl (base) 40 | class llvm::SmallVectorTemplateBase (base) 40 | class llvm::SmallVectorTemplateCommon (base) 40 | class llvm::SmallVectorBase (base) 40 | void * BeginX 48 | unsigned int Size 52 | unsigned int Capacity 56 | struct llvm::SmallVectorStorage (base) 56 | char[64] InlineElts 120 | class llvm::SmallVector Succs 120 | class llvm::SmallVectorImpl (base) 120 | class llvm::SmallVectorTemplateBase (base) 120 | class llvm::SmallVectorTemplateCommon (base) 120 | class llvm::SmallVectorBase (base) 120 | void * BeginX 128 | unsigned int Size 132 | unsigned int Capacity 136 | struct llvm::SmallVectorStorage (base) 136 | char[64] InlineElts 200 | unsigned int NodeNum 204 | unsigned int NodeQueueId 208 | unsigned int NumPreds 212 | unsigned int NumSuccs 216 | unsigned int NumPredsLeft 220 | unsigned int NumSuccsLeft 224 | unsigned int WeakPredsLeft 228 | unsigned int WeakSuccsLeft 232 | unsigned int TopReadyCycle 236 | unsigned int BotReadyCycle 240 | unsigned int Depth 244 | unsigned int Height 248:0-0 | _Bool isVRegCycle 248:1-1 | _Bool isCall 248:2-2 | _Bool isCallOp 248:3-3 | _Bool isTwoAddress 248:4-4 | _Bool isCommutable 248:5-5 | _Bool hasPhysRegUses 248:6-6 | _Bool hasPhysRegDefs 248:7-7 | _Bool hasPhysRegClobbers 249:0-0 | _Bool isPending 249:1-1 | _Bool isAvailable 249:2-2 | _Bool isScheduled 249:3-3 | _Bool isScheduleHigh 249:4-4 | _Bool isScheduleLow 249:5-5 | _Bool isCloned 249:6-6 | _Bool isUnbuffered 249:7-7 | _Bool hasReservedResource 250 | unsigned short NumRegDefsLeft 252 | unsigned short Latency 254:0-0 | _Bool isDepthCurrent 254:1-1 | _Bool isHeightCurrent 254:2-2 | _Bool isNode 254:3-3 | _Bool isInst 254:4-7 | Sched::Preference SchedulingPref | [sizeof=256, dsize=255, align=8, | nvsize=255, nvalign=8] ``` --- llvm/include/llvm/CodeGen/ScheduleDAG.h | 94 +++++++++++++--------- llvm/include/llvm/CodeGen/TargetLowering.h | 17 ++-- 2 files changed, 65 insertions(+), 46 deletions(-) diff --git a/llvm/include/llvm/CodeGen/ScheduleDAG.h b/llvm/include/llvm/CodeGen/ScheduleDAG.h index c5172e8c542b..53265b286c7c 100644 --- a/llvm/include/llvm/CodeGen/ScheduleDAG.h +++ b/llvm/include/llvm/CodeGen/ScheduleDAG.h @@ -243,8 +243,10 @@ class TargetRegisterInfo; private: enum : unsigned { BoundaryID = ~0u }; - SDNode *Node = nullptr; ///< Representative node. - MachineInstr *Instr = nullptr; ///< Alternatively, a MachineInstr. + union { + SDNode *Node; ///< Representative node. + MachineInstr *Instr; ///< Alternatively, a MachineInstr. + }; public: SUnit *OrigNode = nullptr; ///< If not this, the node from which this node @@ -253,6 +255,10 @@ class TargetRegisterInfo; const MCSchedClassDesc *SchedClass = nullptr; ///< nullptr or resolved SchedClass. + const TargetRegisterClass *CopyDstRC = + nullptr; ///< Is a special copy node if != nullptr. + const TargetRegisterClass *CopySrcRC = nullptr; + SmallVector Preds; ///< All sunit predecessors. SmallVector Succs; ///< All sunit successors. @@ -269,8 +275,14 @@ class TargetRegisterInfo; unsigned NumSuccsLeft = 0; ///< # of succs not scheduled. unsigned WeakPredsLeft = 0; ///< # of weak preds not scheduled. unsigned WeakSuccsLeft = 0; ///< # of weak succs not scheduled. - unsigned short NumRegDefsLeft = 0; ///< # of reg defs with no scheduled use. - unsigned short Latency = 0; ///< Node latency. + unsigned TopReadyCycle = 0; ///< Cycle relative to start when node is ready. + unsigned BotReadyCycle = 0; ///< Cycle relative to end when node is ready. + + private: + unsigned Depth = 0; ///< Node depth. + unsigned Height = 0; ///< Node height. + + public: bool isVRegCycle : 1; ///< May use and def the same vreg. bool isCall : 1; ///< Is a function call. bool isCallOp : 1; ///< Is a function call operand. @@ -287,52 +299,54 @@ class TargetRegisterInfo; bool isCloned : 1; ///< True if this node has been cloned. bool isUnbuffered : 1; ///< Uses an unbuffered resource. bool hasReservedResource : 1; ///< Uses a reserved resource. - Sched::Preference SchedulingPref = Sched::None; ///< Scheduling preference. + unsigned short NumRegDefsLeft = 0; ///< # of reg defs with no scheduled use. + unsigned short Latency = 0; ///< Node latency. private: bool isDepthCurrent : 1; ///< True if Depth is current. bool isHeightCurrent : 1; ///< True if Height is current. - unsigned Depth = 0; ///< Node depth. - unsigned Height = 0; ///< Node height. + bool isNode : 1; ///< True if the representative is an SDNode + bool isInst : 1; ///< True if the representative is a MachineInstr public: - unsigned TopReadyCycle = 0; ///< Cycle relative to start when node is ready. - unsigned BotReadyCycle = 0; ///< Cycle relative to end when node is ready. - - const TargetRegisterClass *CopyDstRC = - nullptr; ///< Is a special copy node if != nullptr. - const TargetRegisterClass *CopySrcRC = nullptr; + Sched::Preference SchedulingPref : 4; ///< Scheduling preference. + static_assert(Sched::Preference::Last <= (1 << 4), + "not enough bits in bitfield"); /// Constructs an SUnit for pre-regalloc scheduling to represent an /// SDNode and any nodes flagged to it. SUnit(SDNode *node, unsigned nodenum) - : Node(node), NodeNum(nodenum), isVRegCycle(false), isCall(false), - isCallOp(false), isTwoAddress(false), isCommutable(false), - hasPhysRegUses(false), hasPhysRegDefs(false), hasPhysRegClobbers(false), - isPending(false), isAvailable(false), isScheduled(false), - isScheduleHigh(false), isScheduleLow(false), isCloned(false), - isUnbuffered(false), hasReservedResource(false), isDepthCurrent(false), - isHeightCurrent(false) {} + : Node(node), NodeNum(nodenum), isVRegCycle(false), isCall(false), + isCallOp(false), isTwoAddress(false), isCommutable(false), + hasPhysRegUses(false), hasPhysRegDefs(false), + hasPhysRegClobbers(false), isPending(false), isAvailable(false), + isScheduled(false), isScheduleHigh(false), isScheduleLow(false), + isCloned(false), isUnbuffered(false), hasReservedResource(false), + isDepthCurrent(false), isHeightCurrent(false), isNode(true), + isInst(false), SchedulingPref(Sched::None) {} /// Constructs an SUnit for post-regalloc scheduling to represent a /// MachineInstr. SUnit(MachineInstr *instr, unsigned nodenum) - : Instr(instr), NodeNum(nodenum), isVRegCycle(false), isCall(false), - isCallOp(false), isTwoAddress(false), isCommutable(false), - hasPhysRegUses(false), hasPhysRegDefs(false), hasPhysRegClobbers(false), - isPending(false), isAvailable(false), isScheduled(false), - isScheduleHigh(false), isScheduleLow(false), isCloned(false), - isUnbuffered(false), hasReservedResource(false), isDepthCurrent(false), - isHeightCurrent(false) {} + : Instr(instr), NodeNum(nodenum), isVRegCycle(false), isCall(false), + isCallOp(false), isTwoAddress(false), isCommutable(false), + hasPhysRegUses(false), hasPhysRegDefs(false), + hasPhysRegClobbers(false), isPending(false), isAvailable(false), + isScheduled(false), isScheduleHigh(false), isScheduleLow(false), + isCloned(false), isUnbuffered(false), hasReservedResource(false), + isDepthCurrent(false), isHeightCurrent(false), isNode(false), + isInst(true), SchedulingPref(Sched::None) {} /// Constructs a placeholder SUnit. SUnit() - : isVRegCycle(false), isCall(false), isCallOp(false), isTwoAddress(false), - isCommutable(false), hasPhysRegUses(false), hasPhysRegDefs(false), - hasPhysRegClobbers(false), isPending(false), isAvailable(false), - isScheduled(false), isScheduleHigh(false), isScheduleLow(false), - isCloned(false), isUnbuffered(false), hasReservedResource(false), - isDepthCurrent(false), isHeightCurrent(false) {} + : Node(nullptr), isVRegCycle(false), isCall(false), isCallOp(false), + isTwoAddress(false), isCommutable(false), hasPhysRegUses(false), + hasPhysRegDefs(false), hasPhysRegClobbers(false), isPending(false), + isAvailable(false), isScheduled(false), isScheduleHigh(false), + isScheduleLow(false), isCloned(false), isUnbuffered(false), + hasReservedResource(false), isDepthCurrent(false), + isHeightCurrent(false), isNode(false), isInst(false), + SchedulingPref(Sched::None) {} /// Boundary nodes are placeholders for the boundary of the /// scheduling region. @@ -346,32 +360,36 @@ class TargetRegisterInfo; /// Assigns the representative SDNode for this SUnit. This may be used /// during pre-regalloc scheduling. void setNode(SDNode *N) { - assert(!Instr && "Setting SDNode of SUnit with MachineInstr!"); + assert(!isInst && "Setting SDNode of SUnit with MachineInstr!"); Node = N; + isNode = true; } /// Returns the representative SDNode for this SUnit. This may be used /// during pre-regalloc scheduling. SDNode *getNode() const { - assert(!Instr && "Reading SDNode of SUnit with MachineInstr!"); + assert(!isInst && (isNode || !Instr) && + "Reading SDNode of SUnit without SDNode!"); return Node; } /// Returns true if this SUnit refers to a machine instruction as /// opposed to an SDNode. - bool isInstr() const { return Instr; } + bool isInstr() const { return isInst && Instr; } /// Assigns the instruction for the SUnit. This may be used during /// post-regalloc scheduling. void setInstr(MachineInstr *MI) { - assert(!Node && "Setting MachineInstr of SUnit with SDNode!"); + assert(!isNode && "Setting MachineInstr of SUnit with SDNode!"); Instr = MI; + isInst = true; } /// Returns the representative MachineInstr for this SUnit. This may be used /// during post-regalloc scheduling. MachineInstr *getInstr() const { - assert(!Node && "Reading MachineInstr of SUnit with SDNode!"); + assert(!isNode && (isInst || !Node) && + "Reading MachineInstr of SUnit without MachineInstr!"); return Instr; } diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index aa7a32e86ad8..974a5301f6e2 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -97,14 +97,15 @@ class Value; namespace Sched { enum Preference : uint8_t { - None, // No preference - Source, // Follow source order. - RegPressure, // Scheduling for lowest register pressure. - Hybrid, // Scheduling for both latency and register pressure. - ILP, // Scheduling for ILP in low register pressure mode. - VLIW, // Scheduling for VLIW targets. - Fast, // Fast suboptimal list scheduling - Linearize // Linearize DAG, no scheduling + None, // No preference + Source, // Follow source order. + RegPressure, // Scheduling for lowest register pressure. + Hybrid, // Scheduling for both latency and register pressure. + ILP, // Scheduling for ILP in low register pressure mode. + VLIW, // Scheduling for VLIW targets. + Fast, // Fast suboptimal list scheduling + Linearize, // Linearize DAG, no scheduling + Last = Linearize // Marker for the last Sched::Preference }; } // end namespace Sched -- GitLab From 4f70c5ec4a57e84642fa0772536f120cd9c75edb Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Fri, 7 Jun 2024 11:19:55 +0800 Subject: [PATCH 184/462] Revert "[serialization] no transitive decl change (#92083)" This reverts commit 5c104879c1a98eeb845c03e7c45206bd48e88f0c. The ArmV7 bot is complaining the change breaks the alignment. --- clang/include/clang/AST/ASTUnresolvedSet.h | 9 +- clang/include/clang/AST/DeclAccessPair.h | 30 +--- clang/include/clang/AST/DeclBase.h | 17 +- clang/include/clang/AST/DeclID.h | 18 +- clang/include/clang/AST/UnresolvedSet.h | 1 - .../include/clang/Serialization/ASTBitCodes.h | 29 +--- clang/include/clang/Serialization/ASTReader.h | 36 ++-- .../include/clang/Serialization/ModuleFile.h | 18 +- .../clang/Serialization/ModuleManager.h | 2 +- clang/lib/AST/DeclBase.cpp | 40 +---- clang/lib/AST/DeclCXX.cpp | 4 +- clang/lib/Serialization/ASTReader.cpp | 163 ++++++++---------- clang/lib/Serialization/ASTReaderDecl.cpp | 16 +- clang/lib/Serialization/ASTWriter.cpp | 7 +- clang/lib/Serialization/ModuleFile.cpp | 3 +- .../Modules/no-transitive-decls-change.cppm | 112 ------------ 16 files changed, 167 insertions(+), 338 deletions(-) delete mode 100644 clang/test/Modules/no-transitive-decls-change.cppm diff --git a/clang/include/clang/AST/ASTUnresolvedSet.h b/clang/include/clang/AST/ASTUnresolvedSet.h index dcce3bc63df2..398ffb188c95 100644 --- a/clang/include/clang/AST/ASTUnresolvedSet.h +++ b/clang/include/clang/AST/ASTUnresolvedSet.h @@ -16,7 +16,6 @@ #include "clang/AST/ASTVector.h" #include "clang/AST/DeclAccessPair.h" -#include "clang/AST/DeclID.h" #include "clang/AST/UnresolvedSet.h" #include "clang/Basic/Specifiers.h" #include @@ -57,10 +56,6 @@ public: Decls.push_back(DeclAccessPair::make(D, AS), C); } - void addLazyDecl(ASTContext &C, GlobalDeclID ID, AccessSpecifier AS) { - Decls.push_back(DeclAccessPair::makeLazy(ID.get(), AS), C); - } - /// Replaces the given declaration with the new one, once. /// /// \return true if the set changed @@ -114,10 +109,10 @@ public: void reserve(ASTContext &C, unsigned N) { Impl.reserve(C, N); } - void addLazyDecl(ASTContext &C, GlobalDeclID ID, AccessSpecifier AS) { + void addLazyDecl(ASTContext &C, uintptr_t ID, AccessSpecifier AS) { assert(Impl.empty() || Impl.Decls.isLazy()); Impl.Decls.setLazy(true); - Impl.addLazyDecl(C, ID, AS); + Impl.addDecl(C, reinterpret_cast(ID << 2), AS); } }; diff --git a/clang/include/clang/AST/DeclAccessPair.h b/clang/include/clang/AST/DeclAccessPair.h index 17f29c3e97ce..805342c2910a 100644 --- a/clang/include/clang/AST/DeclAccessPair.h +++ b/clang/include/clang/AST/DeclAccessPair.h @@ -27,17 +27,9 @@ class NamedDecl; /// A POD class for pairing a NamedDecl* with an access specifier. /// Can be put into unions. class DeclAccessPair { - /// Use the lower 2 bit to store AccessSpecifier. Use the higher - /// 61 bit to store the pointer to a NamedDecl or the DeclID to - /// a NamedDecl. If the 3rd bit is set, storing the DeclID, otherwise - /// storing the pointer. - // - // we'd use llvm::PointerUnion, but it isn't trivial - uint64_t Ptr; + uintptr_t Ptr; // we'd use llvm::PointerUnion, but it isn't trivial - enum { ASMask = 0x3, Mask = 0x7 }; - - bool isDeclID() const { return (Ptr >> 2) & 0x1; } + enum { Mask = 0x3 }; public: static DeclAccessPair make(NamedDecl *D, AccessSpecifier AS) { @@ -46,22 +38,12 @@ public: return p; } - static DeclAccessPair makeLazy(uint64_t ID, AccessSpecifier AS) { - DeclAccessPair p; - p.Ptr = (ID << 3) | (0x1 << 2) | uint64_t(AS); - return p; - } - - uint64_t getDeclID() const { - assert(isDeclID()); - return (~Mask & Ptr) >> 3; - } - NamedDecl *getDecl() const { - assert(!isDeclID()); return reinterpret_cast(~Mask & Ptr); } - AccessSpecifier getAccess() const { return AccessSpecifier(ASMask & Ptr); } + AccessSpecifier getAccess() const { + return AccessSpecifier(Mask & Ptr); + } void setDecl(NamedDecl *D) { set(D, getAccess()); @@ -70,7 +52,7 @@ public: set(getDecl(), AS); } void set(NamedDecl *D, AccessSpecifier AS) { - Ptr = uint64_t(AS) | reinterpret_cast(D); + Ptr = uintptr_t(AS) | reinterpret_cast(D); } operator NamedDecl*() const { return getDecl(); } diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h index 5f19af1891b7..600ce73c7f01 100644 --- a/clang/include/clang/AST/DeclBase.h +++ b/clang/include/clang/AST/DeclBase.h @@ -708,7 +708,10 @@ public: /// Set the owning module ID. This may only be called for /// deserialized Decls. - void setOwningModuleID(unsigned ID); + void setOwningModuleID(unsigned ID) { + assert(isFromASTFile() && "Only works on a deserialized declaration"); + *((unsigned*)this - 2) = ID; + } public: /// Determine the availability of the given declaration. @@ -781,11 +784,19 @@ public: /// Retrieve the global declaration ID associated with this /// declaration, which specifies where this Decl was loaded from. - GlobalDeclID getGlobalID() const; + GlobalDeclID getGlobalID() const { + if (isFromASTFile()) + return (*((const GlobalDeclID *)this - 1)); + return GlobalDeclID(); + } /// Retrieve the global ID of the module that owns this particular /// declaration. - unsigned getOwningModuleID() const; + unsigned getOwningModuleID() const { + if (isFromASTFile()) + return *((const unsigned*)this - 2); + return 0; + } private: Module *getOwningModuleSlow() const; diff --git a/clang/include/clang/AST/DeclID.h b/clang/include/clang/AST/DeclID.h index 32d2ed41a374..614ba06b6386 100644 --- a/clang/include/clang/AST/DeclID.h +++ b/clang/include/clang/AST/DeclID.h @@ -19,8 +19,6 @@ #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/iterator.h" -#include - namespace clang { /// Predefined declaration IDs. @@ -109,16 +107,12 @@ public: /// /// DeclID should only be used directly in serialization. All other users /// should use LocalDeclID or GlobalDeclID. - using DeclID = uint64_t; + using DeclID = uint32_t; protected: DeclIDBase() : ID(PREDEF_DECL_NULL_ID) {} explicit DeclIDBase(DeclID ID) : ID(ID) {} - explicit DeclIDBase(unsigned LocalID, unsigned ModuleFileIndex) { - ID = (DeclID)LocalID | ((DeclID)ModuleFileIndex << 32); - } - public: DeclID get() const { return ID; } @@ -130,10 +124,6 @@ public: bool isInvalid() const { return ID == PREDEF_DECL_NULL_ID; } - unsigned getModuleFileIndex() const { return ID >> 32; } - - unsigned getLocalDeclIndex() const; - friend bool operator==(const DeclIDBase &LHS, const DeclIDBase &RHS) { return LHS.ID == RHS.ID; } @@ -166,9 +156,6 @@ public: LocalDeclID(PredefinedDeclIDs ID) : Base(ID) {} explicit LocalDeclID(DeclID ID) : Base(ID) {} - explicit LocalDeclID(unsigned LocalID, unsigned ModuleFileIndex) - : Base(LocalID, ModuleFileIndex) {} - LocalDeclID &operator++() { ++ID; return *this; @@ -188,9 +175,6 @@ public: GlobalDeclID() : Base() {} explicit GlobalDeclID(DeclID ID) : Base(ID) {} - explicit GlobalDeclID(unsigned LocalID, unsigned ModuleFileIndex) - : Base(LocalID, ModuleFileIndex) {} - // For DeclIDIterator to be able to convert a GlobalDeclID // to a LocalDeclID. explicit operator LocalDeclID() const { return LocalDeclID(this->ID); } diff --git a/clang/include/clang/AST/UnresolvedSet.h b/clang/include/clang/AST/UnresolvedSet.h index 1369725ab4e9..ee31be969b6e 100644 --- a/clang/include/clang/AST/UnresolvedSet.h +++ b/clang/include/clang/AST/UnresolvedSet.h @@ -47,7 +47,6 @@ public: // temporaries with defaulted ctors are not zero initialized. UnresolvedSetIterator() : iterator_adaptor_base(nullptr) {} - uint64_t getDeclID() const { return I->getDeclID(); } NamedDecl *getDecl() const { return I->getDecl(); } void setDecl(NamedDecl *ND) const { return I->setDecl(ND); } AccessSpecifier getAccess() const { return I->getAccess(); } diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h index 52a6c5e10f80..f59ff6af4c76 100644 --- a/clang/include/clang/Serialization/ASTBitCodes.h +++ b/clang/include/clang/Serialization/ASTBitCodes.h @@ -255,12 +255,6 @@ public: } }; -// The unaligned decl ID used in the Blobs of bistreams. -using unaligned_decl_id_t = - llvm::support::detail::packed_endian_specific_integral< - serialization::DeclID, llvm::endianness::native, - llvm::support::unaligned>; - /// The number of predefined preprocessed entity IDs. const unsigned int NUM_PREDEF_PP_ENTITY_IDS = 1; @@ -1986,44 +1980,33 @@ enum CleanupObjectKind { COK_Block, COK_CompoundLiteral }; /// Describes the categories of an Objective-C class. struct ObjCCategoriesInfo { - // The ID of the definition. Use unaligned_decl_id_t to keep - // ObjCCategoriesInfo 32-bit aligned. - unaligned_decl_id_t DefinitionID; + // The ID of the definition + LocalDeclID DefinitionID; // Offset into the array of category lists. unsigned Offset; - ObjCCategoriesInfo() = default; - ObjCCategoriesInfo(LocalDeclID ID, unsigned Offset) - : DefinitionID(ID.get()), Offset(Offset) {} - - LocalDeclID getDefinitionID() const { return LocalDeclID(DefinitionID); } - friend bool operator<(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.getDefinitionID() < Y.getDefinitionID(); + return X.DefinitionID < Y.DefinitionID; } friend bool operator>(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.getDefinitionID() > Y.getDefinitionID(); + return X.DefinitionID > Y.DefinitionID; } friend bool operator<=(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.getDefinitionID() <= Y.getDefinitionID(); + return X.DefinitionID <= Y.DefinitionID; } friend bool operator>=(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.getDefinitionID() >= Y.getDefinitionID(); + return X.DefinitionID >= Y.DefinitionID; } }; -static_assert(alignof(ObjCCategoriesInfo) <= 4); -static_assert(std::is_standard_layout_v && - std::is_trivial_v); - /// A key used when looking up entities by \ref DeclarationName. /// /// Different \ref DeclarationNames are mapped to different keys, but the diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index 0a9006223dcb..bea58d60d36c 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -504,6 +504,12 @@ private: /// = I + 1 has already been loaded. llvm::PagedVector DeclsLoaded; + using GlobalDeclMapType = ContinuousRangeMap; + + /// Mapping from global declaration IDs to the module in which the + /// declaration resides. + GlobalDeclMapType GlobalDeclMap; + using FileOffset = std::pair; using FileOffsetsTy = SmallVector; using DeclUpdateOffsetsMap = llvm::DenseMap; @@ -586,11 +592,10 @@ private: struct FileDeclsInfo { ModuleFile *Mod = nullptr; - ArrayRef Decls; + ArrayRef Decls; FileDeclsInfo() = default; - FileDeclsInfo(ModuleFile *Mod, - ArrayRef Decls) + FileDeclsInfo(ModuleFile *Mod, ArrayRef Decls) : Mod(Mod), Decls(Decls) {} }; @@ -599,7 +604,11 @@ private: /// An array of lexical contents of a declaration context, as a sequence of /// Decl::Kind, DeclID pairs. - using LexicalContents = ArrayRef; + using unaligned_decl_id_t = + llvm::support::detail::packed_endian_specific_integral< + serialization::DeclID, llvm::endianness::native, + llvm::support::unaligned>; + using LexicalContents = ArrayRef; /// Map from a DeclContext to its lexical contents. llvm::DenseMap> @@ -1480,11 +1489,10 @@ private: unsigned ClientLoadCapabilities); public: - class ModuleDeclIterator - : public llvm::iterator_adaptor_base< - ModuleDeclIterator, const serialization::unaligned_decl_id_t *, - std::random_access_iterator_tag, const Decl *, ptrdiff_t, - const Decl *, const Decl *> { + class ModuleDeclIterator : public llvm::iterator_adaptor_base< + ModuleDeclIterator, const LocalDeclID *, + std::random_access_iterator_tag, const Decl *, + ptrdiff_t, const Decl *, const Decl *> { ASTReader *Reader = nullptr; ModuleFile *Mod = nullptr; @@ -1492,11 +1500,11 @@ public: ModuleDeclIterator() : iterator_adaptor_base(nullptr) {} ModuleDeclIterator(ASTReader *Reader, ModuleFile *Mod, - const serialization::unaligned_decl_id_t *Pos) + const LocalDeclID *Pos) : iterator_adaptor_base(Pos), Reader(Reader), Mod(Mod) {} value_type operator*() const { - return Reader->GetDecl(Reader->getGlobalDeclID(*Mod, (LocalDeclID)*I)); + return Reader->GetDecl(Reader->getGlobalDeclID(*Mod, *I)); } value_type operator->() const { return **this; } @@ -1536,9 +1544,6 @@ private: StringRef Arg2 = StringRef(), StringRef Arg3 = StringRef()) const; void Error(llvm::Error &&Err) const; - /// Translate a \param GlobalDeclID to the index of DeclsLoaded array. - unsigned translateGlobalDeclIDToIndex(GlobalDeclID ID) const; - public: /// Load the AST file and validate its contents against the given /// Preprocessor. @@ -1910,8 +1915,7 @@ public: /// Retrieve the module file that owns the given declaration, or NULL /// if the declaration is not from a module file. - ModuleFile *getOwningModuleFile(const Decl *D) const; - ModuleFile *getOwningModuleFile(GlobalDeclID ID) const; + ModuleFile *getOwningModuleFile(const Decl *D); /// Returns the source location for the decl \p ID. SourceLocation getSourceLocationForDeclID(GlobalDeclID ID); diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 56193d44dd6f..992d26a8b88c 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -454,11 +454,23 @@ public: /// by the declaration ID (-1). const DeclOffset *DeclOffsets = nullptr; - /// Base declaration index in ASTReader for declarations local to this module. - unsigned BaseDeclIndex = 0; + /// Base declaration ID for declarations local to this module. + serialization::DeclID BaseDeclID = 0; + + /// Remapping table for declaration IDs in this module. + ContinuousRangeMap DeclRemap; + + /// Mapping from the module files that this module file depends on + /// to the base declaration ID for that module as it is understood within this + /// module. + /// + /// This is effectively a reverse global-to-local mapping for declaration + /// IDs, so that we can interpret a true global ID (for this translation unit) + /// as a local ID (for this module file). + llvm::DenseMap GlobalToLocalDeclIDs; /// Array of file-level DeclIDs sorted by file. - const serialization::unaligned_decl_id_t *FileSortedDecls = nullptr; + const LocalDeclID *FileSortedDecls = nullptr; unsigned NumFileSortedDecls = 0; /// Array of category list location information within this diff --git a/clang/include/clang/Serialization/ModuleManager.h b/clang/include/clang/Serialization/ModuleManager.h index f898dab39f06..d770bc52eaf4 100644 --- a/clang/include/clang/Serialization/ModuleManager.h +++ b/clang/include/clang/Serialization/ModuleManager.h @@ -45,7 +45,7 @@ namespace serialization { /// Manages the set of modules loaded by an AST reader. class ModuleManager { /// The chain of AST files, in the order in which we started to load - /// them. + /// them (this order isn't really useful for anything). SmallVector, 2> Chain; /// The chain of non-module PCH files. The first entry is the one named diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp index 7f1ed9c691e9..1e9c879e371b 100644 --- a/clang/lib/AST/DeclBase.cpp +++ b/clang/lib/AST/DeclBase.cpp @@ -74,17 +74,18 @@ void *Decl::operator new(std::size_t Size, const ASTContext &Context, GlobalDeclID ID, std::size_t Extra) { // Allocate an extra 8 bytes worth of storage, which ensures that the // resulting pointer will still be 8-byte aligned. - static_assert(sizeof(uint64_t) >= alignof(Decl), "Decl won't be misaligned"); + static_assert(sizeof(unsigned) * 2 >= alignof(Decl), + "Decl won't be misaligned"); void *Start = Context.Allocate(Size + Extra + 8); void *Result = (char*)Start + 8; - uint64_t *PrefixPtr = (uint64_t *)Result - 1; + unsigned *PrefixPtr = (unsigned *)Result - 2; - *PrefixPtr = ID.get(); + // Zero out the first 4 bytes; this is used to store the owning module ID. + PrefixPtr[0] = 0; - // We leave the upper 16 bits to store the module IDs. 48 bits should be - // sufficient to store a declaration ID. - assert(*PrefixPtr < llvm::maskTrailingOnes(48)); + // Store the global declaration ID in the second 4 bytes. + PrefixPtr[1] = ID.get(); return Result; } @@ -110,29 +111,6 @@ void *Decl::operator new(std::size_t Size, const ASTContext &Ctx, return ::operator new(Size + Extra, Ctx); } -GlobalDeclID Decl::getGlobalID() const { - if (!isFromASTFile()) - return GlobalDeclID(); - // See the comments in `Decl::operator new` for details. - uint64_t ID = *((const uint64_t *)this - 1); - return GlobalDeclID(ID & llvm::maskTrailingOnes(48)); -} - -unsigned Decl::getOwningModuleID() const { - if (!isFromASTFile()) - return 0; - - uint64_t ID = *((const uint64_t *)this - 1); - return ID >> 48; -} - -void Decl::setOwningModuleID(unsigned ID) { - assert(isFromASTFile() && "Only works on a deserialized declaration"); - uint64_t *IDAddress = (uint64_t *)this - 1; - *IDAddress &= llvm::maskTrailingOnes(48); - *IDAddress |= (uint64_t)ID << 48; -} - Module *Decl::getOwningModuleSlow() const { assert(isFromASTFile() && "Not from AST file?"); return getASTContext().getExternalSource()->getModule(getOwningModuleID()); @@ -2185,7 +2163,3 @@ DependentDiagnostic *DependentDiagnostic::Create(ASTContext &C, return DD; } - -unsigned DeclIDBase::getLocalDeclIndex() const { - return ID & llvm::maskTrailingOnes(32); -} diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp index 7f2c786547b9..75c441293d62 100644 --- a/clang/lib/AST/DeclCXX.cpp +++ b/clang/lib/AST/DeclCXX.cpp @@ -68,8 +68,8 @@ void LazyASTUnresolvedSet::getFromExternalSource(ASTContext &C) const { assert(Source && "getFromExternalSource with no external source"); for (ASTUnresolvedSet::iterator I = Impl.begin(); I != Impl.end(); ++I) - I.setDecl( - cast(Source->GetExternalDecl(GlobalDeclID(I.getDeclID())))); + I.setDecl(cast(Source->GetExternalDecl( + GlobalDeclID(reinterpret_cast(I.getDecl()) >> 2)))); Impl.Decls.setLazy(false); } diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 59338b44db32..33cea32c3be6 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -1658,7 +1658,7 @@ bool ASTReader::ReadSLocEntry(int ID) { unsigned NumFileDecls = Record[7]; if (NumFileDecls && ContextObj) { - const unaligned_decl_id_t *FirstDecl = F->FileSortedDecls + Record[6]; + const LocalDeclID *FirstDecl = F->FileSortedDecls + Record[6]; assert(F->FileSortedDecls && "FILE_SORTED_DECLS not encountered yet ?"); FileDeclIDs[FID] = FileDeclsInfo(F, llvm::ArrayRef(FirstDecl, NumFileDecls)); @@ -3377,11 +3377,26 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F, "duplicate DECL_OFFSET record in AST file"); F.DeclOffsets = (const DeclOffset *)Blob.data(); F.LocalNumDecls = Record[0]; - F.BaseDeclIndex = getTotalNumDecls(); + unsigned LocalBaseDeclID = Record[1]; + F.BaseDeclID = getTotalNumDecls(); - if (F.LocalNumDecls > 0) - DeclsLoaded.resize(DeclsLoaded.size() + F.LocalNumDecls); + if (F.LocalNumDecls > 0) { + // Introduce the global -> local mapping for declarations within this + // module. + GlobalDeclMap.insert(std::make_pair( + GlobalDeclID(getTotalNumDecls() + NUM_PREDEF_DECL_IDS), &F)); + + // Introduce the local -> global mapping for declarations within this + // module. + F.DeclRemap.insertOrReplace( + std::make_pair(LocalBaseDeclID, F.BaseDeclID - LocalBaseDeclID)); + + // Introduce the global -> local mapping for declarations within this + // module. + F.GlobalToLocalDeclIDs[&F] = LocalBaseDeclID; + DeclsLoaded.resize(DeclsLoaded.size() + F.LocalNumDecls); + } break; } @@ -3616,7 +3631,7 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F, break; case FILE_SORTED_DECLS: - F.FileSortedDecls = (const unaligned_decl_id_t *)Blob.data(); + F.FileSortedDecls = (const LocalDeclID *)Blob.data(); F.NumFileSortedDecls = Record[0]; break; @@ -4043,6 +4058,7 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { RemapBuilder PreprocessedEntityRemap(F.PreprocessedEntityRemap); RemapBuilder SubmoduleRemap(F.SubmoduleRemap); RemapBuilder SelectorRemap(F.SelectorRemap); + RemapBuilder DeclRemap(F.DeclRemap); RemapBuilder TypeRemap(F.TypeRemap); auto &ImportedModuleVector = F.TransitiveImports; @@ -4081,6 +4097,8 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { endian::readNext(Data); uint32_t SelectorIDOffset = endian::readNext(Data); + uint32_t DeclIDOffset = + endian::readNext(Data); uint32_t TypeIndexOffset = endian::readNext(Data); @@ -4098,7 +4116,11 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { PreprocessedEntityRemap); mapOffset(SubmoduleIDOffset, OM->BaseSubmoduleID, SubmoduleRemap); mapOffset(SelectorIDOffset, OM->BaseSelectorID, SelectorRemap); + mapOffset(DeclIDOffset, OM->BaseDeclID, DeclRemap); mapOffset(TypeIndexOffset, OM->BaseTypeIndex, TypeRemap); + + // Global -> local mappings. + F.GlobalToLocalDeclIDs[OM] = DeclIDOffset; } } @@ -4623,7 +4645,7 @@ ASTReader::ASTReadResult ASTReader::ReadAST(StringRef FileName, ModuleKind Type, // that we load any additional categories. if (ContextObj) { for (unsigned I = 0, N = ObjCClassesLoaded.size(); I != N; ++I) { - loadObjCCategories(ObjCClassesLoaded[I]->getGlobalID(), + loadObjCCategories(GlobalDeclID(ObjCClassesLoaded[I]->getGlobalID()), ObjCClassesLoaded[I], PreviousGeneration); } } @@ -7622,25 +7644,18 @@ CXXBaseSpecifier *ASTReader::GetExternalCXXBaseSpecifiers(uint64_t Offset) { GlobalDeclID ASTReader::getGlobalDeclID(ModuleFile &F, LocalDeclID LocalID) const { - if (LocalID.get() < NUM_PREDEF_DECL_IDS) - return GlobalDeclID(LocalID.get()); - - unsigned OwningModuleFileIndex = LocalID.getModuleFileIndex(); - DeclID ID = LocalID.getLocalDeclIndex(); + DeclID ID = LocalID.get(); + if (ID < NUM_PREDEF_DECL_IDS) + return GlobalDeclID(ID); if (!F.ModuleOffsetMap.empty()) ReadModuleOffsetMap(F); - ModuleFile *OwningModuleFile = - OwningModuleFileIndex == 0 - ? &F - : F.TransitiveImports[OwningModuleFileIndex - 1]; - - if (OwningModuleFileIndex == 0) - ID -= NUM_PREDEF_DECL_IDS; + ContinuousRangeMap::iterator I = + F.DeclRemap.find(ID - NUM_PREDEF_DECL_IDS); + assert(I != F.DeclRemap.end() && "Invalid index into decl index remap"); - uint64_t NewModuleFileIndex = OwningModuleFile->Index + 1; - return GlobalDeclID(ID, NewModuleFileIndex); + return GlobalDeclID(ID + I->second); } bool ASTReader::isDeclIDFromModule(GlobalDeclID ID, ModuleFile &M) const { @@ -7648,33 +7663,31 @@ bool ASTReader::isDeclIDFromModule(GlobalDeclID ID, ModuleFile &M) const { if (ID.get() < NUM_PREDEF_DECL_IDS) return false; - unsigned ModuleFileIndex = ID.getModuleFileIndex(); - return M.Index == ModuleFileIndex - 1; -} - -ModuleFile *ASTReader::getOwningModuleFile(GlobalDeclID ID) const { - // Predefined decls aren't from any module. - if (ID.get() < NUM_PREDEF_DECL_IDS) - return nullptr; - - uint64_t ModuleFileIndex = ID.getModuleFileIndex(); - assert(ModuleFileIndex && "Untranslated Local Decl?"); - - return &getModuleManager()[ModuleFileIndex - 1]; + return ID.get() - NUM_PREDEF_DECL_IDS >= M.BaseDeclID && + ID.get() - NUM_PREDEF_DECL_IDS < M.BaseDeclID + M.LocalNumDecls; } -ModuleFile *ASTReader::getOwningModuleFile(const Decl *D) const { +ModuleFile *ASTReader::getOwningModuleFile(const Decl *D) { if (!D->isFromASTFile()) return nullptr; - - return getOwningModuleFile(D->getGlobalID()); + GlobalDeclMapType::const_iterator I = + GlobalDeclMap.find(GlobalDeclID(D->getGlobalID())); + assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); + return I->second; } SourceLocation ASTReader::getSourceLocationForDeclID(GlobalDeclID ID) { if (ID.get() < NUM_PREDEF_DECL_IDS) return SourceLocation(); - if (Decl *D = GetExistingDecl(ID)) + unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; + + if (Index > DeclsLoaded.size()) { + Error("declaration ID out-of-range for AST file"); + return SourceLocation(); + } + + if (Decl *D = DeclsLoaded[Index]) return D->getLocation(); SourceLocation Loc; @@ -7741,19 +7754,8 @@ static Decl *getPredefinedDecl(ASTContext &Context, PredefinedDeclIDs ID) { llvm_unreachable("PredefinedDeclIDs unknown enum value"); } -unsigned ASTReader::translateGlobalDeclIDToIndex(GlobalDeclID GlobalID) const { - ModuleFile *OwningModuleFile = getOwningModuleFile(GlobalID); - if (!OwningModuleFile) { - assert(GlobalID.get() < NUM_PREDEF_DECL_IDS && "Untransalted Global ID?"); - return GlobalID.get(); - } - - return OwningModuleFile->BaseDeclIndex + GlobalID.getLocalDeclIndex(); -} - Decl *ASTReader::GetExistingDecl(GlobalDeclID ID) { assert(ContextObj && "reading decl with no AST context"); - if (ID.get() < NUM_PREDEF_DECL_IDS) { Decl *D = getPredefinedDecl(*ContextObj, (PredefinedDeclIDs)ID); if (D) { @@ -7766,7 +7768,7 @@ Decl *ASTReader::GetExistingDecl(GlobalDeclID ID) { return D; } - unsigned Index = translateGlobalDeclIDToIndex(ID); + unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; if (Index >= DeclsLoaded.size()) { assert(0 && "declaration ID out-of-range for AST file"); @@ -7781,7 +7783,7 @@ Decl *ASTReader::GetDecl(GlobalDeclID ID) { if (ID.get() < NUM_PREDEF_DECL_IDS) return GetExistingDecl(ID); - unsigned Index = translateGlobalDeclIDToIndex(ID); + unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; if (Index >= DeclsLoaded.size()) { assert(0 && "declaration ID out-of-range for AST file"); @@ -7800,31 +7802,20 @@ Decl *ASTReader::GetDecl(GlobalDeclID ID) { LocalDeclID ASTReader::mapGlobalIDToModuleFileGlobalID(ModuleFile &M, GlobalDeclID GlobalID) { - if (GlobalID.get() < NUM_PREDEF_DECL_IDS) - return LocalDeclID(GlobalID.get()); - - if (!M.ModuleOffsetMap.empty()) - ReadModuleOffsetMap(M); - - ModuleFile *Owner = getOwningModuleFile(GlobalID); - DeclID ID = GlobalID.getLocalDeclIndex(); - - if (Owner == &M) { - ID += NUM_PREDEF_DECL_IDS; + DeclID ID = GlobalID.get(); + if (ID < NUM_PREDEF_DECL_IDS) return LocalDeclID(ID); - } - uint64_t OrignalModuleFileIndex = 0; - for (unsigned I = 0; I < M.TransitiveImports.size(); I++) - if (M.TransitiveImports[I] == Owner) { - OrignalModuleFileIndex = I + 1; - break; - } + GlobalDeclMapType::const_iterator I = GlobalDeclMap.find(GlobalID); + assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); + ModuleFile *Owner = I->second; - if (!OrignalModuleFileIndex) + llvm::DenseMap::iterator Pos = + M.GlobalToLocalDeclIDs.find(Owner); + if (Pos == M.GlobalToLocalDeclIDs.end()) return LocalDeclID(); - return LocalDeclID(ID, OrignalModuleFileIndex); + return LocalDeclID(ID - Owner->BaseDeclID + Pos->second); } GlobalDeclID ASTReader::ReadDeclID(ModuleFile &F, const RecordData &Record, @@ -7903,34 +7894,32 @@ void ASTReader::FindExternalLexicalDecls( namespace { -class UnalignedDeclIDComp { +class DeclIDComp { ASTReader &Reader; ModuleFile &Mod; public: - UnalignedDeclIDComp(ASTReader &Reader, ModuleFile &M) - : Reader(Reader), Mod(M) {} + DeclIDComp(ASTReader &Reader, ModuleFile &M) : Reader(Reader), Mod(M) {} - bool operator()(unaligned_decl_id_t L, unaligned_decl_id_t R) const { + bool operator()(LocalDeclID L, LocalDeclID R) const { SourceLocation LHS = getLocation(L); SourceLocation RHS = getLocation(R); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - bool operator()(SourceLocation LHS, unaligned_decl_id_t R) const { + bool operator()(SourceLocation LHS, LocalDeclID R) const { SourceLocation RHS = getLocation(R); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - bool operator()(unaligned_decl_id_t L, SourceLocation RHS) const { + bool operator()(LocalDeclID L, SourceLocation RHS) const { SourceLocation LHS = getLocation(L); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - SourceLocation getLocation(unaligned_decl_id_t ID) const { + SourceLocation getLocation(LocalDeclID ID) const { return Reader.getSourceManager().getFileLoc( - Reader.getSourceLocationForDeclID( - Reader.getGlobalDeclID(Mod, (LocalDeclID)ID))); + Reader.getSourceLocationForDeclID(Reader.getGlobalDeclID(Mod, ID))); } }; @@ -7953,8 +7942,8 @@ void ASTReader::FindFileRegionDecls(FileID File, BeginLoc = SM.getLocForStartOfFile(File).getLocWithOffset(Offset); SourceLocation EndLoc = BeginLoc.getLocWithOffset(Length); - UnalignedDeclIDComp DIDComp(*this, *DInfo.Mod); - ArrayRef::iterator BeginIt = + DeclIDComp DIDComp(*this, *DInfo.Mod); + ArrayRef::iterator BeginIt = llvm::lower_bound(DInfo.Decls, BeginLoc, DIDComp); if (BeginIt != DInfo.Decls.begin()) --BeginIt; @@ -7963,18 +7952,17 @@ void ASTReader::FindFileRegionDecls(FileID File, // to backtrack until we find it otherwise we will fail to report that the // region overlaps with an objc container. while (BeginIt != DInfo.Decls.begin() && - GetDecl(getGlobalDeclID(*DInfo.Mod, (LocalDeclID)(*BeginIt))) + GetDecl(getGlobalDeclID(*DInfo.Mod, *BeginIt)) ->isTopLevelDeclInObjCContainer()) --BeginIt; - ArrayRef::iterator EndIt = + ArrayRef::iterator EndIt = llvm::upper_bound(DInfo.Decls, EndLoc, DIDComp); if (EndIt != DInfo.Decls.end()) ++EndIt; - for (ArrayRef::iterator DIt = BeginIt; DIt != EndIt; - ++DIt) - Decls.push_back(GetDecl(getGlobalDeclID(*DInfo.Mod, (LocalDeclID)(*DIt)))); + for (ArrayRef::iterator DIt = BeginIt; DIt != EndIt; ++DIt) + Decls.push_back(GetDecl(getGlobalDeclID(*DInfo.Mod, *DIt))); } bool @@ -8181,6 +8169,7 @@ LLVM_DUMP_METHOD void ASTReader::dump() { dumpModuleIDMap("Global bit offset map", GlobalBitOffsetsMap); dumpModuleIDMap("Global source location entry map", GlobalSLocEntryMap); dumpModuleIDMap("Global type map", GlobalTypeMap); + dumpModuleIDMap("Global declaration map", GlobalDeclMap); dumpModuleIDMap("Global identifier map", GlobalIdentifierMap); dumpModuleIDMap("Global macro map", GlobalMacroMap); dumpModuleIDMap("Global submodule map", GlobalSubmoduleMap); @@ -9196,7 +9185,7 @@ void ASTRecordReader::readUnresolvedSet(LazyASTUnresolvedSet &Set) { while (NumDecls--) { GlobalDeclID ID = readDeclID(); AccessSpecifier AS = (AccessSpecifier) readInt(); - Set.addLazyDecl(getContext(), ID, AS); + Set.addLazyDecl(getContext(), ID.get(), AS); } } diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp index cf2dc32e30b9..765c083ce8df 100644 --- a/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/clang/lib/Serialization/ASTReaderDecl.cpp @@ -2924,7 +2924,7 @@ void ASTDeclReader::mergeTemplatePattern(RedeclarableTemplateDecl *D, auto *ExistingPattern = Existing->getTemplatedDecl(); RedeclarableResult Result( /*MergeWith*/ ExistingPattern, - DPattern->getCanonicalDecl()->getGlobalID(), IsKeyDecl); + GlobalDeclID(DPattern->getCanonicalDecl()->getGlobalID()), IsKeyDecl); if (auto *DClass = dyn_cast(DPattern)) { // Merge with any existing definition. @@ -3245,10 +3245,11 @@ bool ASTReader::isConsumerInterestedIn(Decl *D) { /// Get the correct cursor and offset for loading a declaration. ASTReader::RecordLocation ASTReader::DeclCursorForID(GlobalDeclID ID, SourceLocation &Loc) { - ModuleFile *M = getOwningModuleFile(ID); - assert(M); - unsigned LocalDeclIndex = ID.getLocalDeclIndex(); - const DeclOffset &DOffs = M->DeclOffsets[LocalDeclIndex]; + GlobalDeclMapType::iterator I = GlobalDeclMap.find(ID); + assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); + ModuleFile *M = I->second; + const DeclOffset &DOffs = + M->DeclOffsets[ID.get() - M->BaseDeclID - NUM_PREDEF_DECL_IDS]; Loc = ReadSourceLocation(*M, DOffs.getRawLoc()); return RecordLocation(M, DOffs.getBitOffset(M->DeclsBlockStartOffset)); } @@ -3791,6 +3792,7 @@ void ASTReader::markIncompleteDeclChain(Decl *D) { /// Read the declaration at the given offset from the AST file. Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) { + unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; SourceLocation DeclLoc; RecordLocation Loc = DeclCursorForID(ID, DeclLoc); llvm::BitstreamCursor &DeclsCursor = Loc.F->DeclsCursor; @@ -4121,7 +4123,7 @@ Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) { } assert(D && "Unknown declaration reading AST file"); - LoadedDecl(translateGlobalDeclIDToIndex(ID), D); + LoadedDecl(Index, D); // Set the DeclContext before doing any deserialization, to make sure internal // calls to Decl::getASTContext() by Decl's methods will find the // TranslationUnitDecl without crashing. @@ -4447,7 +4449,7 @@ namespace { M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap, Compare); if (Result == M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap || - Result->getDefinitionID() != LocalID) { + Result->DefinitionID != LocalID) { // We didn't find anything. If the class definition is in this module // file, then the module files it depends on cannot have any categories, // so suppress further lookup. diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index ee3e687636e6..953990a4aca6 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -3357,10 +3357,12 @@ void ASTWriter::WriteTypeDeclOffsets() { Abbrev = std::make_shared(); Abbrev->Add(BitCodeAbbrevOp(DECL_OFFSET)); Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of declarations + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // base decl ID Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // declarations block unsigned DeclOffsetAbbrev = Stream.EmitAbbrev(std::move(Abbrev)); { - RecordData::value_type Record[] = {DECL_OFFSET, DeclOffsets.size()}; + RecordData::value_type Record[] = {DECL_OFFSET, DeclOffsets.size(), + FirstDeclID.get() - NUM_PREDEF_DECL_IDS}; Stream.EmitRecordWithBlob(DeclOffsetAbbrev, Record, bytes(DeclOffsets)); } } @@ -5421,6 +5423,7 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, M.NumPreprocessedEntities); writeBaseIDOrNone(M.BaseSubmoduleID, M.LocalNumSubmodules); writeBaseIDOrNone(M.BaseSelectorID, M.LocalNumSelectors); + writeBaseIDOrNone(M.BaseDeclID, M.LocalNumDecls); writeBaseIDOrNone(M.BaseTypeIndex, M.LocalNumTypes); } } @@ -6615,11 +6618,13 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) { // Note, this will get called multiple times, once one the reader starts up // and again each time it's done reading a PCH or module. + FirstDeclID = LocalDeclID(NUM_PREDEF_DECL_IDS + Chain->getTotalNumDecls()); FirstTypeID = NUM_PREDEF_TYPE_IDS + Chain->getTotalNumTypes(); FirstIdentID = NUM_PREDEF_IDENT_IDS + Chain->getTotalNumIdentifiers(); FirstMacroID = NUM_PREDEF_MACRO_IDS + Chain->getTotalNumMacros(); FirstSubmoduleID = NUM_PREDEF_SUBMODULE_IDS + Chain->getTotalNumSubmodules(); FirstSelectorID = NUM_PREDEF_SELECTOR_IDS + Chain->getTotalNumSelectors(); + NextDeclID = FirstDeclID; NextTypeID = FirstTypeID; NextIdentID = FirstIdentID; NextMacroID = FirstMacroID; diff --git a/clang/lib/Serialization/ModuleFile.cpp b/clang/lib/Serialization/ModuleFile.cpp index f64a59bd9489..2c42d33a8f5d 100644 --- a/clang/lib/Serialization/ModuleFile.cpp +++ b/clang/lib/Serialization/ModuleFile.cpp @@ -87,6 +87,7 @@ LLVM_DUMP_METHOD void ModuleFile::dump() { << " Number of types: " << LocalNumTypes << '\n'; dumpLocalRemap("Type index local -> global map", TypeRemap); - llvm::errs() << " Base decl index: " << BaseDeclIndex << '\n' + llvm::errs() << " Base decl ID: " << BaseDeclID << '\n' << " Number of decls: " << LocalNumDecls << '\n'; + dumpLocalRemap("Decl ID local -> global map", DeclRemap); } diff --git a/clang/test/Modules/no-transitive-decls-change.cppm b/clang/test/Modules/no-transitive-decls-change.cppm deleted file mode 100644 index 42ac061bc90b..000000000000 --- a/clang/test/Modules/no-transitive-decls-change.cppm +++ /dev/null @@ -1,112 +0,0 @@ -// Testing that changing a declaration in an unused module file won't change -// the BMI of the current module file. -// -// RUN: rm -rf %t -// RUN: split-file %s %t -// -// RUN: %clang_cc1 -std=c++20 %t/m-partA.cppm -emit-reduced-module-interface -o %t/m-partA.pcm -// RUN: %clang_cc1 -std=c++20 %t/m-partA.v1.cppm -emit-reduced-module-interface -o \ -// RUN: %t/m-partA.v1.pcm -// RUN: %clang_cc1 -std=c++20 %t/m-partB.cppm -emit-reduced-module-interface -o %t/m-partB.pcm -// RUN: %clang_cc1 -std=c++20 %t/m.cppm -emit-reduced-module-interface -o %t/m.pcm \ -// RUN: -fmodule-file=m:partA=%t/m-partA.pcm -fmodule-file=m:partB=%t/m-partB.pcm -// RUN: %clang_cc1 -std=c++20 %t/m.cppm -emit-reduced-module-interface -o %t/m.v1.pcm \ -// RUN: -fmodule-file=m:partA=%t/m-partA.v1.pcm -fmodule-file=m:partB=%t/m-partB.pcm -// -// RUN: %clang_cc1 -std=c++20 %t/useBOnly.cppm -emit-reduced-module-interface -o %t/useBOnly.pcm \ -// RUN: -fmodule-file=m=%t/m.pcm -fmodule-file=m:partA=%t/m-partA.pcm \ -// RUN: -fmodule-file=m:partB=%t/m-partB.pcm -// RUN: %clang_cc1 -std=c++20 %t/useBOnly.cppm -emit-reduced-module-interface -o %t/useBOnly.v1.pcm \ -// RUN: -fmodule-file=m=%t/m.v1.pcm -fmodule-file=m:partA=%t/m-partA.v1.pcm \ -// RUN: -fmodule-file=m:partB=%t/m-partB.pcm -// Since useBOnly only uses partB from module M, the change in partA shouldn't affect -// useBOnly. -// RUN: diff %t/useBOnly.pcm %t/useBOnly.v1.pcm &> /dev/null - -//--- m-partA.cppm -export module m:partA; - -namespace A_Impl { - inline int getAImpl() { - return 43; - } - - inline int getA2Impl() { - return 43; - } -} - -namespace A { - using A_Impl::getAImpl; -} - -export inline int getA() { - return 43; -} - -export inline int getA2(int) { - return 88; -} - -//--- m-partA.v1.cppm -export module m:partA; - -namespace A_Impl { - inline int getAImpl() { - return 43; - } - - inline int getA2Impl() { - return 43; - } -} - -namespace A { - using A_Impl::getAImpl; - // Adding a new declaration without introducing a new declaration name. - using A_Impl::getA2Impl; -} - -inline int getA() { - return 43; -} - -inline int getA2(int) { - return 88; -} - -// Now we add a new declaration without introducing new identifier and new types. -// The consuming module which didn't use m:partA completely is expected to be -// not changed. -inline int getA(int) { - return 88; -} - -//--- m-partB.cppm -export module m:partB; - -export inline int getB() { - return 430; -} - -//--- m.cppm -export module m; -export import :partA; -export import :partB; - -//--- useBOnly.cppm -export module useBOnly; -import m; - -export inline int get() { - return getB(); -} - -//--- useAOnly.cppm -export module useAOnly; -import m; - -export inline int get() { - A a; - return a.getValue(); -} -- GitLab From 0e1d6e2fc48accbfdbb349dc460a4e91edf1d884 Mon Sep 17 00:00:00 2001 From: Christudasan Devadasan Date: Fri, 7 Jun 2024 09:02:18 +0530 Subject: [PATCH 185/462] [AMDGPU] Auto-generating lit test patterns (NFC) (#93837) Test CodeGen/AMDGPU/build_vector.ll has the lit patterns partially hand-written and the rest auto-generated. It doesn't look good when changes are required with future patches. Auto-generating the entire pattern. Moved out the R600 test into build_vector-r600.ll. --- llvm/test/CodeGen/AMDGPU/build_vector-r600.ll | 107 ++++++ llvm/test/CodeGen/AMDGPU/build_vector.ll | 355 ++++++++++++------ 2 files changed, 355 insertions(+), 107 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/build_vector-r600.ll diff --git a/llvm/test/CodeGen/AMDGPU/build_vector-r600.ll b/llvm/test/CodeGen/AMDGPU/build_vector-r600.ll new file mode 100644 index 000000000000..2abcbbcdd1bc --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/build_vector-r600.ll @@ -0,0 +1,107 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc < %s -mtriple=r600-- -mcpu=redwood | FileCheck %s --check-prefixes=R600 + +define amdgpu_kernel void @build_vector2 (ptr addrspace(1) %out) { +; R600-LABEL: build_vector2: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 4, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV * T0.Y, literal.x, +; R600-NEXT: 6(8.407791e-45), 0(0.000000e+00) +; R600-NEXT: MOV T0.X, literal.x, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.y, +; R600-NEXT: 5(7.006492e-45), 2(2.802597e-45) +entry: + store <2 x i32> , ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @build_vector4 (ptr addrspace(1) %out) { +; R600-LABEL: build_vector4: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 8, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV * T0.W, literal.x, +; R600-NEXT: 8(1.121039e-44), 0(0.000000e+00) +; R600-NEXT: MOV * T0.Z, literal.x, +; R600-NEXT: 7(9.809089e-45), 0(0.000000e+00) +; R600-NEXT: MOV * T0.Y, literal.x, +; R600-NEXT: 6(8.407791e-45), 0(0.000000e+00) +; R600-NEXT: MOV T0.X, literal.x, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.y, +; R600-NEXT: 5(7.006492e-45), 2(2.802597e-45) +entry: + store <4 x i32> , ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @build_vector_v2i16 (ptr addrspace(1) %out) { +; R600-LABEL: build_vector_v2i16: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T4.X, T5.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV T4.X, literal.x, +; R600-NEXT: LSHR * T5.X, KC0[2].Y, literal.y, +; R600-NEXT: 393221(5.510200e-40), 2(2.802597e-45) +entry: + store <2 x i16> , ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32 %a) { +; R600-LABEL: build_vector_v2i16_trunc: +; R600: ; %bb.0: +; R600-NEXT: ALU 4, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T4.X, T5.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR * T0.W, KC0[2].Z, literal.x, +; R600-NEXT: 16(2.242078e-44), 0(0.000000e+00) +; R600-NEXT: OR_INT T4.X, PV.W, literal.x, +; R600-NEXT: LSHR * T5.X, KC0[2].Y, literal.y, +; R600-NEXT: 327680(4.591775e-40), 2(2.802597e-45) + %srl = lshr i32 %a, 16 + %trunc = trunc i32 %srl to i16 + %ins.0 = insertelement <2 x i16> undef, i16 %trunc, i32 0 + %ins.1 = insertelement <2 x i16> %ins.0, i16 5, i32 1 + store <2 x i16> %ins.1, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @build_v2i32_from_v4i16_shuffle(ptr addrspace(1) %out, <4 x i16> %in) { +; R600-LABEL: build_v2i32_from_v4i16_shuffle: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 0, @10, KC0[], KC1[] +; R600-NEXT: TEX 1 @6 +; R600-NEXT: ALU 4, @11, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: Fetch clause starting at 6: +; R600-NEXT: VTX_READ_16 T1.X, T0.X, 48, #3 +; R600-NEXT: VTX_READ_16 T0.X, T0.X, 44, #3 +; R600-NEXT: ALU clause starting at 10: +; R600-NEXT: MOV * T0.X, 0.0, +; R600-NEXT: ALU clause starting at 11: +; R600-NEXT: LSHL * T0.Y, T1.X, literal.x, +; R600-NEXT: 16(2.242078e-44), 0(0.000000e+00) +; R600-NEXT: LSHL T0.X, T0.X, literal.x, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.y, +; R600-NEXT: 16(2.242078e-44), 2(2.802597e-45) +entry: + %shuf = shufflevector <4 x i16> %in, <4 x i16> zeroinitializer, <2 x i32> + %zextended = zext <2 x i16> %shuf to <2 x i32> + %shifted = shl <2 x i32> %zextended, + store <2 x i32> %shifted, ptr addrspace(1) %out + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/build_vector.ll b/llvm/test/CodeGen/AMDGPU/build_vector.ll index 99755133f36d..b26d15ed3a1c 100644 --- a/llvm/test/CodeGen/AMDGPU/build_vector.ll +++ b/llvm/test/CodeGen/AMDGPU/build_vector.ll @@ -1,95 +1,256 @@ -; RUN: llc < %s -mtriple=r600 -mcpu=redwood | FileCheck %s --check-prefixes=R600,ALL -; RUN: llc < %s -mtriple=amdgcn -verify-machineinstrs | FileCheck %s --check-prefixes=GFX6,GFX678,ALL -; RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs | FileCheck %s --check-prefixes=GFX8,GFX678,ALL -; RUN: llc < %s -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 -verify-machineinstrs | FileCheck %s --check-prefixes=GFX10,GFX1011,ALL -; RUN: llc < %s -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-vopd=0 -verify-machineinstrs | FileCheck %s --check-prefixes=GFX11,GFX1011,ALL -; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx940 | FileCheck %s --check-prefixes=GFX940,ALL +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc < %s -mtriple=amdgcn -mcpu=tahiti | FileCheck %s --check-prefixes=GFX6,GCN +; RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefixes=GFX8,GCN +; RUN: llc < %s -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 | FileCheck %s --check-prefixes=GFX10,GCN +; RUN: llc < %s -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-vopd=0 | FileCheck %s --check-prefixes=GFX11,GCN +; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx940 | FileCheck %s --check-prefixes=GFX940,GCN -; ALL-LABEL: {{^}}build_vector2: -; R600: MOV -; R600: MOV -; R600-NOT: MOV -; GFX678-DAG: v_mov_b32_e32 v[[X:[0-9]]], 5 -; GFX678-DAG: v_mov_b32_e32 v[[Y:[0-9]]], 6 -; GFX1011-DAG: v_mov_b32_e32 v[[X:[0-9]]], 5 -; GFX1011-DAG: v_mov_b32_e32 v[[Y:[0-9]]], 6 -; GFX678: buffer_store_dwordx2 v[[[X]]:[[Y]]] -; GFX10: global_store_dwordx2 v2, v[0:1], s[0:1] -; GFX11: global_store_b64 v2, v[0:1], s[0:1] define amdgpu_kernel void @build_vector2 (ptr addrspace(1) %out) { +; GFX6-LABEL: build_vector2: +; GFX6: ; %bb.0: ; %entry +; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: s_mov_b32 s2, -1 +; GFX6-NEXT: v_mov_b32_e32 v0, 5 +; GFX6-NEXT: v_mov_b32_e32 v1, 6 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; GFX6-NEXT: s_endpgm +; +; GFX8-LABEL: build_vector2: +; GFX8: ; %bb.0: ; %entry +; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX8-NEXT: v_mov_b32_e32 v0, 5 +; GFX8-NEXT: v_mov_b32_e32 v1, 6 +; GFX8-NEXT: s_waitcnt lgkmcnt(0) +; GFX8-NEXT: v_mov_b32_e32 v3, s1 +; GFX8-NEXT: v_mov_b32_e32 v2, s0 +; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; GFX8-NEXT: s_endpgm +; +; GFX10-LABEL: build_vector2: +; GFX10: ; %bb.0: ; %entry +; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-NEXT: v_mov_b32_e32 v2, 0 +; GFX10-NEXT: v_mov_b32_e32 v0, 5 +; GFX10-NEXT: v_mov_b32_e32 v1, 6 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: build_vector2: +; GFX11: ; %bb.0: ; %entry +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: v_mov_b32_e32 v0, 5 +; GFX11-NEXT: v_mov_b32_e32 v1, 6 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm +; +; GFX940-LABEL: build_vector2: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-NEXT: v_mov_b32_e32 v0, 5 +; GFX940-NEXT: v_mov_b32_e32 v1, 6 +; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1 +; GFX940-NEXT: s_endpgm entry: store <2 x i32> , ptr addrspace(1) %out ret void } -; ALL-LABEL: {{^}}build_vector4: -; R600: MOV -; R600: MOV -; R600: MOV -; R600: MOV -; R600-NOT: MOV -; GFX678-DAG: v_mov_b32_e32 v[[X:[0-9]]], 5 -; GFX678-DAG: v_mov_b32_e32 v[[Y:[0-9]]], 6 -; GFX678-DAG: v_mov_b32_e32 v[[Z:[0-9]]], 7 -; GFX678-DAG: v_mov_b32_e32 v[[W:[0-9]]], 8 -; GFX1011-DAG: v_mov_b32_e32 v[[X:[0-9]]], 5 -; GFX1011-DAG: v_mov_b32_e32 v[[Y:[0-9]]], 6 -; GFX1011-DAG: v_mov_b32_e32 v[[Z:[0-9]]], 7 -; GFX1011-DAG: v_mov_b32_e32 v[[W:[0-9]]], 8 -; GFX678: buffer_store_dwordx4 v[[[X]]:[[W]]] -; GFX10: global_store_dwordx4 v4, v[0:3], s[0:1] -; GFX11: global_store_b128 v4, v[0:3], s[0:1] define amdgpu_kernel void @build_vector4 (ptr addrspace(1) %out) { +; GFX6-LABEL: build_vector4: +; GFX6: ; %bb.0: ; %entry +; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: s_mov_b32 s2, -1 +; GFX6-NEXT: v_mov_b32_e32 v0, 5 +; GFX6-NEXT: v_mov_b32_e32 v1, 6 +; GFX6-NEXT: v_mov_b32_e32 v2, 7 +; GFX6-NEXT: v_mov_b32_e32 v3, 8 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 +; GFX6-NEXT: s_endpgm +; +; GFX8-LABEL: build_vector4: +; GFX8: ; %bb.0: ; %entry +; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX8-NEXT: v_mov_b32_e32 v0, 5 +; GFX8-NEXT: v_mov_b32_e32 v1, 6 +; GFX8-NEXT: v_mov_b32_e32 v2, 7 +; GFX8-NEXT: v_mov_b32_e32 v3, 8 +; GFX8-NEXT: s_waitcnt lgkmcnt(0) +; GFX8-NEXT: v_mov_b32_e32 v5, s1 +; GFX8-NEXT: v_mov_b32_e32 v4, s0 +; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; GFX8-NEXT: s_endpgm +; +; GFX10-LABEL: build_vector4: +; GFX10: ; %bb.0: ; %entry +; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-NEXT: v_mov_b32_e32 v4, 0 +; GFX10-NEXT: v_mov_b32_e32 v0, 5 +; GFX10-NEXT: v_mov_b32_e32 v1, 6 +; GFX10-NEXT: v_mov_b32_e32 v2, 7 +; GFX10-NEXT: v_mov_b32_e32 v3, 8 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: build_vector4: +; GFX11: ; %bb.0: ; %entry +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: v_mov_b32_e32 v4, 0 +; GFX11-NEXT: v_mov_b32_e32 v0, 5 +; GFX11-NEXT: v_mov_b32_e32 v1, 6 +; GFX11-NEXT: v_mov_b32_e32 v2, 7 +; GFX11-NEXT: v_mov_b32_e32 v3, 8 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm +; +; GFX940-LABEL: build_vector4: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: v_mov_b32_e32 v4, 0 +; GFX940-NEXT: v_mov_b32_e32 v0, 5 +; GFX940-NEXT: v_mov_b32_e32 v1, 6 +; GFX940-NEXT: v_mov_b32_e32 v2, 7 +; GFX940-NEXT: v_mov_b32_e32 v3, 8 +; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1 +; GFX940-NEXT: s_endpgm entry: store <4 x i32> , ptr addrspace(1) %out ret void } - -; ALL-LABEL: {{^}}build_vector_v2i16: -; R600: MOV -; R600-NOT: MOV -; GFX678: s_mov_b32 s3, 0xf000 -; GFX678: s_mov_b32 s2, -1 -; GFX678: v_mov_b32_e32 v0, 0x60005 -; GFX678: s_waitcnt lgkmcnt(0) -; GFX678: buffer_store_dword v0, off, s[0:3], 0 -; GFX1011: v_mov_b32_e32 v0, 0 -; GFX1011: v_mov_b32_e32 v1, 0x60005 -; GFX1011: s_waitcnt lgkmcnt(0) -; GFX10: global_store_dword v0, v1, s[0:1] -; GFX11: global_store_b32 v0, v1, s[0:1] define amdgpu_kernel void @build_vector_v2i16 (ptr addrspace(1) %out) { +; GFX6-LABEL: build_vector_v2i16: +; GFX6: ; %bb.0: ; %entry +; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: s_mov_b32 s2, -1 +; GFX6-NEXT: v_mov_b32_e32 v0, 0x60005 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; GFX6-NEXT: s_endpgm +; +; GFX8-LABEL: build_vector_v2i16: +; GFX8: ; %bb.0: ; %entry +; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX8-NEXT: v_mov_b32_e32 v2, 0x60005 +; GFX8-NEXT: s_waitcnt lgkmcnt(0) +; GFX8-NEXT: v_mov_b32_e32 v0, s0 +; GFX8-NEXT: v_mov_b32_e32 v1, s1 +; GFX8-NEXT: flat_store_dword v[0:1], v2 +; GFX8-NEXT: s_endpgm +; +; GFX10-LABEL: build_vector_v2i16: +; GFX10: ; %bb.0: ; %entry +; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-NEXT: v_mov_b32_e32 v0, 0 +; GFX10-NEXT: v_mov_b32_e32 v1, 0x60005 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: global_store_dword v0, v1, s[0:1] +; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: build_vector_v2i16: +; GFX11: ; %bb.0: ; %entry +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: v_mov_b32_e32 v0, 0 +; GFX11-NEXT: v_mov_b32_e32 v1, 0x60005 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm +; +; GFX940-LABEL: build_vector_v2i16: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-NEXT: v_mov_b32_e32 v1, 0x60005 +; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1 +; GFX940-NEXT: s_endpgm entry: store <2 x i16> , ptr addrspace(1) %out ret void } -; ALL-LABEL: {{^}}build_vector_v2i16_trunc: -; R600: LSHR -; R600: OR_INT -; R600: LSHR -; R600-NOT: MOV -; GFX6: s_mov_b32 s3, 0xf000 -; GFX6: s_waitcnt lgkmcnt(0) -; GFX6: v_alignbit_b32 v0, 5, s4, 16 -; GFX6: buffer_store_dword v0, off, s[0:3], 0 -; GFX8: s_mov_b32 s3, 0xf000 -; GFX8: s_mov_b32 s2, -1 -; GFX8: s_waitcnt lgkmcnt(0) -; GFX8: s_lshr_b32 s4, s4, 16 -; GFX8: s_or_b32 s4, s4, 0x50000 -; GFX8: v_mov_b32_e32 v0, s4 -; GFX8: buffer_store_dword v0, off, s[0:3], 0 -; GFX1011: v_mov_b32_e32 v0, 0 -; GFX1011: s_waitcnt lgkmcnt(0) -; GFX10: s_lshr_b32 s2, s2, 16 -; GFX10: s_pack_ll_b32_b16 s2, s2, 5 -; GFX11: s_pack_hl_b32_b16 s2, s2, 5 -; GFX1011: v_mov_b32_e32 v1, s2 -; GFX10: global_store_dword v0, v1, s[0:1] -; GFX11: global_store_b32 v0, v1, s[0:1] define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32 %a) { +; GFX6-LABEL: build_vector_v2i16_trunc: +; GFX6: ; %bb.0: +; GFX6-NEXT: s_load_dword s4, s[0:1], 0xb +; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: s_mov_b32 s2, -1 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: v_alignbit_b32 v0, 5, s4, 16 +; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; GFX6-NEXT: s_endpgm +; +; GFX8-LABEL: build_vector_v2i16_trunc: +; GFX8: ; %bb.0: +; GFX8-NEXT: s_load_dword s2, s[0:1], 0x2c +; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX8-NEXT: s_waitcnt lgkmcnt(0) +; GFX8-NEXT: s_lshr_b32 s2, s2, 16 +; GFX8-NEXT: s_or_b32 s2, s2, 0x50000 +; GFX8-NEXT: v_mov_b32_e32 v0, s0 +; GFX8-NEXT: v_mov_b32_e32 v1, s1 +; GFX8-NEXT: v_mov_b32_e32 v2, s2 +; GFX8-NEXT: flat_store_dword v[0:1], v2 +; GFX8-NEXT: s_endpgm +; +; GFX10-LABEL: build_vector_v2i16_trunc: +; GFX10: ; %bb.0: +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_load_dword s2, s[0:1], 0x8 +; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-NEXT: v_mov_b32_e32 v0, 0 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_lshr_b32 s2, s2, 16 +; GFX10-NEXT: s_pack_ll_b32_b16 s2, s2, 5 +; GFX10-NEXT: v_mov_b32_e32 v1, s2 +; GFX10-NEXT: global_store_dword v0, v1, s[0:1] +; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: build_vector_v2i16_trunc: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x8 +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: v_mov_b32_e32 v0, 0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_pack_hl_b32_b16 s2, s2, 5 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_mov_b32_e32 v1, s2 +; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm +; +; GFX940-LABEL: build_vector_v2i16_trunc: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_load_dword s4, s[0:1], 0x2c +; GFX940-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX940-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: s_lshr_b32 s0, s4, 16 +; GFX940-NEXT: s_pack_ll_b32_b16 s0, s0, 5 +; GFX940-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-NEXT: s_endpgm %srl = lshr i32 %a, 16 %trunc = trunc i32 %srl to i16 %ins.0 = insertelement <2 x i16> undef, i16 %trunc, i32 0 @@ -98,54 +259,33 @@ define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32 ret void } -; R600-LABEL: build_v2i32_from_v4i16_shuffle: -; R600: ; %bb.0: ; %entry -; R600-NEXT: ALU 0, @10, KC0[], KC1[] -; R600-NEXT: TEX 1 @6 -; R600-NEXT: ALU 4, @11, KC0[CB0:0-32], KC1[] -; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 -; R600-NEXT: CF_END -; R600-NEXT: PAD -; R600-NEXT: Fetch clause starting at 6: -; R600-NEXT: VTX_READ_16 T1.X, T0.X, 48, #3 -; R600-NEXT: VTX_READ_16 T0.X, T0.X, 44, #3 -; R600-NEXT: ALU clause starting at 10: -; R600-NEXT: MOV * T0.X, 0.0, -; R600-NEXT: ALU clause starting at 11: -; R600-NEXT: LSHL * T0.Y, T1.X, literal.x, -; R600-NEXT: 16(2.242078e-44), 0(0.000000e+00) -; R600-NEXT: LSHL T0.X, T0.X, literal.x, -; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.y, -; R600-NEXT: 16(2.242078e-44), 2(2.802597e-45) -; +define amdgpu_kernel void @build_v2i32_from_v4i16_shuffle(ptr addrspace(1) %out, <4 x i16> %in) { ; GFX6-LABEL: build_v2i32_from_v4i16_shuffle: ; GFX6: ; %bb.0: ; %entry ; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GFX6-NEXT: s_mov_b32 s7, 0xf000 -; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_lshl_b32 s3, s3, 16 -; GFX6-NEXT: s_lshl_b32 s2, s2, 16 ; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: s_mov_b32 s4, s0 ; GFX6-NEXT: s_mov_b32 s5, s1 -; GFX6-NEXT: v_mov_b32_e32 v0, s2 -; GFX6-NEXT: v_mov_b32_e32 v1, s3 +; GFX6-NEXT: s_lshl_b32 s0, s3, 16 +; GFX6-NEXT: s_lshl_b32 s1, s2, 16 +; GFX6-NEXT: v_mov_b32_e32 v0, s1 +; GFX6-NEXT: v_mov_b32_e32 v1, s0 ; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX6-NEXT: s_endpgm ; ; GFX8-LABEL: build_v2i32_from_v4i16_shuffle: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 -; GFX8-NEXT: s_mov_b32 s7, 0xf000 -; GFX8-NEXT: s_mov_b32 s6, -1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_mov_b32 s4, s0 -; GFX8-NEXT: s_mov_b32 s5, s1 -; GFX8-NEXT: s_lshl_b32 s0, s3, 16 -; GFX8-NEXT: s_lshl_b32 s1, s2, 16 -; GFX8-NEXT: v_mov_b32_e32 v0, s1 -; GFX8-NEXT: v_mov_b32_e32 v1, s0 -; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; GFX8-NEXT: s_lshl_b32 s3, s3, 16 +; GFX8-NEXT: s_lshl_b32 s2, s2, 16 +; GFX8-NEXT: v_mov_b32_e32 v3, s1 +; GFX8-NEXT: v_mov_b32_e32 v0, s2 +; GFX8-NEXT: v_mov_b32_e32 v1, s3 +; GFX8-NEXT: v_mov_b32_e32 v2, s0 +; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GFX8-NEXT: s_endpgm ; ; GFX10-LABEL: build_v2i32_from_v4i16_shuffle: @@ -185,7 +325,6 @@ define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32 ; GFX940-NEXT: v_mov_b32_e32 v1, s3 ; GFX940-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1 ; GFX940-NEXT: s_endpgm -define amdgpu_kernel void @build_v2i32_from_v4i16_shuffle(ptr addrspace(1) %out, <4 x i16> %in) { entry: %shuf = shufflevector <4 x i16> %in, <4 x i16> zeroinitializer, <2 x i32> %zextended = zext <2 x i16> %shuf to <2 x i32> @@ -193,3 +332,5 @@ entry: store <2 x i32> %shifted, ptr addrspace(1) %out ret void } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GCN: {{.*}} -- GitLab From e96e7f1f33637ba5080c82e874622948c0a0c25b Mon Sep 17 00:00:00 2001 From: Christudasan Devadasan Date: Fri, 7 Jun 2024 09:07:11 +0530 Subject: [PATCH 186/462] [AMDGPU] Auto-generated some lit test patterns (NFC). (#94310) Also, converted the R600 RUN lines from some tests into standalone tests. --- llvm/test/CodeGen/AMDGPU/fabs-r600.ll | 159 ++++ llvm/test/CodeGen/AMDGPU/fabs.ll | 252 ++++-- llvm/test/CodeGen/AMDGPU/fneg-fabs-r600.ll | 180 ++++ llvm/test/CodeGen/AMDGPU/fneg-fabs.f64.ll | 267 +++++- llvm/test/CodeGen/AMDGPU/fneg-fabs.ll | 258 +++++- llvm/test/CodeGen/AMDGPU/fneg.ll | 781 ++++++++++++++---- .../AMDGPU/llvm.r600.read.local.size.ll | 428 ++++++++-- llvm/test/CodeGen/AMDGPU/packed-op-sel.ll | 630 ++++++++------ llvm/test/CodeGen/AMDGPU/xor-r600.ll | 478 +++++++++++ llvm/test/CodeGen/AMDGPU/xor.ll | 766 ++++++++++++++--- 10 files changed, 3484 insertions(+), 715 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/fabs-r600.ll create mode 100644 llvm/test/CodeGen/AMDGPU/fneg-fabs-r600.ll create mode 100644 llvm/test/CodeGen/AMDGPU/xor-r600.ll diff --git a/llvm/test/CodeGen/AMDGPU/fabs-r600.ll b/llvm/test/CodeGen/AMDGPU/fabs-r600.ll new file mode 100644 index 000000000000..7e1aa99c3ec4 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/fabs-r600.ll @@ -0,0 +1,159 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 %s + + +; DAGCombiner will transform: +; (fabsf (f32 bitcast (i32 a))) => (f32 bitcast (and (i32 a), 0x7FFFFFFF)) +; unless isFabsFree returns true +define amdgpu_kernel void @s_fabsf_fn_free(ptr addrspace(1) %out, i32 %in) { +; R600-LABEL: s_fabsf_fn_free: +; R600: ; %bb.0: +; R600-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV * T0.W, KC0[2].Z, +; R600-NEXT: MOV T0.X, |PV.W|, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %bc= bitcast i32 %in to float + %fabs = call float @fabsf(float %bc) + store float %fabs, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @s_fabsf_free(ptr addrspace(1) %out, i32 %in) { +; R600-LABEL: s_fabsf_free: +; R600: ; %bb.0: +; R600-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV * T0.W, KC0[2].Z, +; R600-NEXT: MOV T0.X, |PV.W|, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %bc= bitcast i32 %in to float + %fabs = call float @llvm.fabs.f32(float %bc) + store float %fabs, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @s_fabsf_f32(ptr addrspace(1) %out, float %in) { +; R600-LABEL: s_fabsf_f32: +; R600: ; %bb.0: +; R600-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV * T0.W, KC0[2].Z, +; R600-NEXT: MOV T0.X, |PV.W|, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %fabs = call float @llvm.fabs.f32(float %in) + store float %fabs, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @fabs_v2f32(ptr addrspace(1) %out, <2 x float> %in) { +; R600-LABEL: fabs_v2f32: +; R600: ; %bb.0: +; R600-NEXT: ALU 5, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV * T0.W, KC0[3].X, +; R600-NEXT: MOV T0.Y, |PV.W|, +; R600-NEXT: MOV * T0.W, KC0[2].W, +; R600-NEXT: MOV T0.X, |PV.W|, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in) + store <2 x float> %fabs, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @fabsf_v4f32(ptr addrspace(1) %out, <4 x float> %in) { +; R600-LABEL: fabsf_v4f32: +; R600: ; %bb.0: +; R600-NEXT: ALU 9, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV T0.W, KC0[4].X, +; R600-NEXT: MOV * T1.W, KC0[3].W, +; R600-NEXT: MOV * T0.W, |PV.W|, +; R600-NEXT: MOV T0.Z, |T1.W|, +; R600-NEXT: MOV * T1.W, KC0[3].Z, +; R600-NEXT: MOV T0.Y, |PV.W|, +; R600-NEXT: MOV * T1.W, KC0[3].Y, +; R600-NEXT: MOV T0.X, |PV.W|, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %in) + store <4 x float> %fabs, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @fabsf_fn_fold(ptr addrspace(1) %out, float %in0, float %in1) { +; R600-LABEL: fabsf_fn_fold: +; R600: ; %bb.0: +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: MUL_IEEE * T1.X, |KC0[2].Z|, KC0[2].W, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %fabs = call float @fabsf(float %in0) + %fmul = fmul float %fabs, %in1 + store float %fmul, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @fabs_fold(ptr addrspace(1) %out, float %in0, float %in1) { +; R600-LABEL: fabs_fold: +; R600: ; %bb.0: +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: MUL_IEEE * T1.X, |KC0[2].Z|, KC0[2].W, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %fabs = call float @llvm.fabs.f32(float %in0) + %fmul = fmul float %fabs, %in1 + store float %fmul, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @bitpreserve_fabsf_f32(ptr addrspace(1) %out, float %in) { +; R600-LABEL: bitpreserve_fabsf_f32: +; R600: ; %bb.0: +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: ADD * T1.X, |KC0[2].Z|, 1.0, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %in.bc = bitcast float %in to i32 + %int.abs = and i32 %in.bc, 2147483647 + %bc = bitcast i32 %int.abs to float + %fadd = fadd float %bc, 1.0 + store float %fadd, ptr addrspace(1) %out + ret void +} + +declare float @fabsf(float) readnone +declare float @llvm.fabs.f32(float) readnone +declare <2 x float> @llvm.fabs.v2f32(<2 x float>) readnone +declare <4 x float> @llvm.fabs.v4f32(<4 x float>) readnone diff --git a/llvm/test/CodeGen/AMDGPU/fabs.ll b/llvm/test/CodeGen/AMDGPU/fabs.ll index e18c76f89b6c..07581ade57cc 100644 --- a/llvm/test/CodeGen/AMDGPU/fabs.ll +++ b/llvm/test/CodeGen/AMDGPU/fabs.ll @@ -1,104 +1,256 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs -enable-misched=0 < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global,-xnack -enable-misched=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s -; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -march=amdgcn -enable-misched=0 < %s | FileCheck -check-prefixes=GCN,SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -enable-misched=0 < %s | FileCheck -check-prefixes=GCN,VI %s ; DAGCombiner will transform: ; (fabsf (f32 bitcast (i32 a))) => (f32 bitcast (and (i32 a), 0x7FFFFFFF)) ; unless isFabsFree returns true - -; FUNC-LABEL: {{^}}s_fabsf_fn_free: -; R600-NOT: AND -; R600: |PV.{{[XYZW]}}| - -; GCN: s_bitset0_b32 s{{[0-9]+}}, 31 define amdgpu_kernel void @s_fabsf_fn_free(ptr addrspace(1) %out, i32 %in) { +; SI-LABEL: s_fabsf_fn_free: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x9 +; SI-NEXT: s_load_dword s4, s[2:3], 0xb +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_bitset0_b32 s4, 31 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: s_fabsf_fn_free: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 +; VI-NEXT: s_load_dword s2, s[2:3], 0x2c +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: s_bitset0_b32 s2, 31 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %bc= bitcast i32 %in to float %fabs = call float @fabsf(float %bc) store float %fabs, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}s_fabsf_free: -; R600-NOT: AND -; R600: |PV.{{[XYZW]}}| - -; GCN: s_bitset0_b32 s{{[0-9]+}}, 31 define amdgpu_kernel void @s_fabsf_free(ptr addrspace(1) %out, i32 %in) { +; SI-LABEL: s_fabsf_free: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; SI-NEXT: s_load_dword s0, s[0:1], 0xb +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_bitset0_b32 s0, 31 +; SI-NEXT: v_mov_b32_e32 v0, s0 +; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: s_fabsf_free: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; VI-NEXT: s_load_dword s0, s[0:1], 0x2c +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: s_bitset0_b32 s0, 31 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %bc= bitcast i32 %in to float %fabs = call float @llvm.fabs.f32(float %bc) store float %fabs, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}s_fabsf_f32: -; R600: |{{(PV|T[0-9])\.[XYZW]}}| - -; GCN: s_bitset0_b32 s{{[0-9]+}}, 31 define amdgpu_kernel void @s_fabsf_f32(ptr addrspace(1) %out, float %in) { +; SI-LABEL: s_fabsf_f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; SI-NEXT: s_load_dword s0, s[0:1], 0xb +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_bitset0_b32 s0, 31 +; SI-NEXT: v_mov_b32_e32 v0, s0 +; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: s_fabsf_f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; VI-NEXT: s_load_dword s0, s[0:1], 0x2c +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: s_bitset0_b32 s0, 31 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %fabs = call float @llvm.fabs.f32(float %in) store float %fabs, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}fabs_v2f32: -; R600: |{{(PV|T[0-9])\.[XYZW]}}| -; R600: |{{(PV|T[0-9])\.[XYZW]}}| - -; GCN: s_and_b32 s{{[0-9]+}}, s{{[0-9]+}}, 0x7fffffff -; GCN: s_and_b32 s{{[0-9]+}}, s{{[0-9]+}}, 0x7fffffff define amdgpu_kernel void @fabs_v2f32(ptr addrspace(1) %out, <2 x float> %in) { +; SI-LABEL: fabs_v2f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: s_and_b32 s0, s3, 0x7fffffff +; SI-NEXT: s_and_b32 s1, s2, 0x7fffffff +; SI-NEXT: v_mov_b32_e32 v0, s1 +; SI-NEXT: v_mov_b32_e32 v1, s0 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fabs_v2f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_bitset0_b32 s3, 31 +; VI-NEXT: s_bitset0_b32 s2, 31 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in) store <2 x float> %fabs, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}fabsf_v4f32: -; R600: |{{(PV|T[0-9])\.[XYZW]}}| -; R600: |{{(PV|T[0-9])\.[XYZW]}}| -; R600: |{{(PV|T[0-9])\.[XYZW]}}| -; R600: |{{(PV|T[0-9])\.[XYZW]}}| - -; GCN: s_bitset0_b32 -; GCN: s_bitset0_b32 -; GCN: s_bitset0_b32 -; GCN: s_bitset0_b32 define amdgpu_kernel void @fabsf_v4f32(ptr addrspace(1) %out, <4 x float> %in) { +; SI-LABEL: fabsf_v4f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_bitset0_b32 s3, 31 +; SI-NEXT: s_bitset0_b32 s2, 31 +; SI-NEXT: s_bitset0_b32 s1, 31 +; SI-NEXT: s_bitset0_b32 s0, 31 +; SI-NEXT: v_mov_b32_e32 v0, s0 +; SI-NEXT: v_mov_b32_e32 v1, s1 +; SI-NEXT: v_mov_b32_e32 v2, s2 +; SI-NEXT: v_mov_b32_e32 v3, s3 +; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fabsf_v4f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x34 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v4, s4 +; VI-NEXT: s_bitset0_b32 s3, 31 +; VI-NEXT: s_bitset0_b32 s2, 31 +; VI-NEXT: s_bitset0_b32 s1, 31 +; VI-NEXT: s_bitset0_b32 s0, 31 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: v_mov_b32_e32 v3, s3 +; VI-NEXT: v_mov_b32_e32 v5, s5 +; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; VI-NEXT: s_endpgm %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %in) store <4 x float> %fabs, ptr addrspace(1) %out ret void } -; GCN-LABEL: {{^}}fabsf_fn_fold: -; SI: s_load_dwordx4 s[[[#LOAD:]]:[[#END:]]], s[{{[0-9]+:[0-9]+}}], 0x9 -; VI: s_load_dwordx4 s[[[#LOAD:]]:[[#END:]]], s[{{[0-9]+:[0-9]+}}], 0x24 -; GCN-NOT: and -; GCN: v_mov_b32_e32 [[V_MUL_VI:v[0-9]+]], s[[#LOAD + 3]] -; GCN: v_mul_f32_e64 v{{[0-9]+}}, |s[[#LOAD + 2]]|, [[V_MUL_VI]] define amdgpu_kernel void @fabsf_fn_fold(ptr addrspace(1) %out, float %in0, float %in1) { +; SI-LABEL: fabsf_fn_fold: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: v_mov_b32_e32 v0, s3 +; SI-NEXT: v_mul_f32_e64 v0, |s2|, v0 +; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fabsf_fn_fold: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s3 +; VI-NEXT: v_mul_f32_e64 v2, |s2|, v0 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %fabs = call float @fabsf(float %in0) %fmul = fmul float %fabs, %in1 store float %fmul, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}fabs_fold: -; SI: s_load_dwordx4 s[[[#LOAD:]]:[[#END:]]], s[{{[0-9]+:[0-9]+}}], 0x9 -; VI: s_load_dwordx4 s[[[#LOAD:]]:[[#END:]]], s[{{[0-9]+:[0-9]+}}], 0x24 -; GCN-NOT: and -; GCN: v_mov_b32_e32 [[V_MUL_VI:v[0-9]+]], s[[#LOAD + 3]] -; GCN: v_mul_f32_e64 v{{[0-9]+}}, |s[[#LOAD + 2]]|, [[V_MUL_VI]] define amdgpu_kernel void @fabs_fold(ptr addrspace(1) %out, float %in0, float %in1) { +; SI-LABEL: fabs_fold: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: v_mov_b32_e32 v0, s3 +; SI-NEXT: v_mul_f32_e64 v0, |s2|, v0 +; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fabs_fold: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s3 +; VI-NEXT: v_mul_f32_e64 v2, |s2|, v0 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %fabs = call float @llvm.fabs.f32(float %in0) %fmul = fmul float %fabs, %in1 store float %fmul, ptr addrspace(1) %out ret void } -; Make sure we turn some integer operations back into fabsf -; FUNC-LABEL: {{^}}bitpreserve_fabsf_f32: -; GCN: v_add_f32_e64 v{{[0-9]+}}, |s{{[0-9]+}}|, 1.0 define amdgpu_kernel void @bitpreserve_fabsf_f32(ptr addrspace(1) %out, float %in) { +; SI-LABEL: bitpreserve_fabsf_f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; SI-NEXT: s_load_dword s0, s[0:1], 0xb +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_add_f32_e64 v0, |s0|, 1.0 +; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: bitpreserve_fabsf_f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; VI-NEXT: s_load_dword s0, s[0:1], 0x2c +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_add_f32_e64 v2, |s0|, 1.0 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %in.bc = bitcast float %in to i32 %int.abs = and i32 %in.bc, 2147483647 %bc = bitcast i32 %int.abs to float @@ -111,3 +263,5 @@ declare float @fabsf(float) readnone declare float @llvm.fabs.f32(float) readnone declare <2 x float> @llvm.fabs.v2f32(<2 x float>) readnone declare <4 x float> @llvm.fabs.v4f32(<4 x float>) readnone +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GCN: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs-r600.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs-r600.ll new file mode 100644 index 000000000000..4f5271ed2325 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs-r600.ll @@ -0,0 +1,180 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=redwood < %s | FileCheck --check-prefix=R600 %s + +define amdgpu_kernel void @fneg_fabsf_fadd_f32(ptr addrspace(1) %out, float %x, float %y) { +; R600-LABEL: fneg_fabsf_fadd_f32: +; R600: ; %bb.0: +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: ADD * T1.X, KC0[2].W, -|KC0[2].Z|, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %fabs = call float @llvm.fabs.f32(float %x) + %fsub = fsub float -0.000000e+00, %fabs + %fadd = fadd float %y, %fsub + store float %fadd, ptr addrspace(1) %out, align 4 + ret void +} + +define amdgpu_kernel void @fneg_fabsf_fmul_f32(ptr addrspace(1) %out, float %x, float %y) { +; R600-LABEL: fneg_fabsf_fmul_f32: +; R600: ; %bb.0: +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: MUL_IEEE * T1.X, KC0[2].W, -|KC0[2].Z|, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %fabs = call float @llvm.fabs.f32(float %x) + %fsub = fsub float -0.000000e+00, %fabs + %fmul = fmul float %y, %fsub + store float %fmul, ptr addrspace(1) %out, align 4 + ret void +} + +define amdgpu_kernel void @fneg_fabsf_free_f32(ptr addrspace(1) %out, i32 %in) { +; R600-LABEL: fneg_fabsf_free_f32: +; R600: ; %bb.0: +; R600-NEXT: ALU 4, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV * T0.W, KC0[2].Z, +; R600-NEXT: MOV * T0.W, |PV.W|, +; R600-NEXT: MOV T0.X, -PV.W, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %bc = bitcast i32 %in to float + %fabs = call float @llvm.fabs.f32(float %bc) + %fsub = fsub float -0.000000e+00, %fabs + store float %fsub, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @fneg_fabsf_fn_free_f32(ptr addrspace(1) %out, i32 %in) { +; R600-LABEL: fneg_fabsf_fn_free_f32: +; R600: ; %bb.0: +; R600-NEXT: ALU 4, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV * T0.W, KC0[2].Z, +; R600-NEXT: MOV * T0.W, |PV.W|, +; R600-NEXT: MOV T0.X, -PV.W, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %bc = bitcast i32 %in to float + %fabs = call float @fabsf(float %bc) + %fsub = fsub float -0.000000e+00, %fabs + store float %fsub, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @fneg_fabsf_f32(ptr addrspace(1) %out, float %in) { +; R600-LABEL: fneg_fabsf_f32: +; R600: ; %bb.0: +; R600-NEXT: ALU 4, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV * T0.W, KC0[2].Z, +; R600-NEXT: MOV * T0.W, |PV.W|, +; R600-NEXT: MOV T0.X, -PV.W, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %fabs = call float @llvm.fabs.f32(float %in) + %fsub = fsub float -0.000000e+00, %fabs + store float %fsub, ptr addrspace(1) %out, align 4 + ret void +} + +define amdgpu_kernel void @v_fneg_fabsf_f32(ptr addrspace(1) %out, ptr addrspace(1) %in) { +; R600-LABEL: v_fneg_fabsf_f32: +; R600: ; %bb.0: +; R600-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 0 @6 +; R600-NEXT: ALU 3, @9, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: Fetch clause starting at 6: +; R600-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1 +; R600-NEXT: ALU clause starting at 8: +; R600-NEXT: MOV * T0.X, KC0[2].Z, +; R600-NEXT: ALU clause starting at 9: +; R600-NEXT: MOV * T0.W, |T0.X|, +; R600-NEXT: MOV T0.X, -PV.W, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %val = load float, ptr addrspace(1) %in, align 4 + %fabs = call float @llvm.fabs.f32(float %val) + %fsub = fsub float -0.000000e+00, %fabs + store float %fsub, ptr addrspace(1) %out, align 4 + ret void +} + +define amdgpu_kernel void @fneg_fabsf_v2f32(ptr addrspace(1) %out, <2 x float> %in) { +; R600-LABEL: fneg_fabsf_v2f32: +; R600: ; %bb.0: +; R600-NEXT: ALU 7, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV T0.W, KC0[3].X, +; R600-NEXT: MOV * T1.W, KC0[2].W, +; R600-NEXT: MOV * T0.W, |PV.W|, +; R600-NEXT: MOV T0.Y, -PV.W, +; R600-NEXT: MOV * T0.W, |T1.W|, +; R600-NEXT: MOV T0.X, -PV.W, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in) + %fsub = fsub <2 x float> , %fabs + store <2 x float> %fsub, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @fneg_fabsf_v4f32(ptr addrspace(1) %out, <4 x float> %in) { +; R600-LABEL: fneg_fabsf_v4f32: +; R600: ; %bb.0: +; R600-NEXT: ALU 13, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T2.XYZW, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV * T0.W, KC0[4].X, +; R600-NEXT: MOV T0.W, |PV.W|, +; R600-NEXT: MOV * T1.W, KC0[3].W, +; R600-NEXT: MOV T0.Z, KC0[3].Z, +; R600-NEXT: MOV T1.W, |PS|, +; R600-NEXT: MOV * T2.W, -PV.W, +; R600-NEXT: MOV T2.Z, -PV.W, +; R600-NEXT: MOV T0.W, KC0[3].Y, +; R600-NEXT: MOV * T1.W, |PV.Z|, +; R600-NEXT: MOV T2.Y, -PS, +; R600-NEXT: MOV * T0.W, |PV.W|, +; R600-NEXT: MOV T2.X, -PV.W, +; R600-NEXT: LSHR * T0.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %in) + %fsub = fsub <4 x float> , %fabs + store <4 x float> %fsub, ptr addrspace(1) %out + ret void +} + +declare float @fabsf(float) readnone +declare float @llvm.fabs.f32(float) readnone +declare <2 x float> @llvm.fabs.v2f32(<2 x float>) readnone +declare <4 x float> @llvm.fabs.v4f32(<4 x float>) readnone + +!llvm.module.flags = !{!0} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.f64.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.f64.ll index 5f1d232daabe..2c9042ec17da 100644 --- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.f64.ll +++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.f64.ll @@ -1,12 +1,35 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN %s -; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -march=amdgcn < %s | FileCheck -check-prefixes=SI,GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefixes=VI,GCN %s -; FIXME: Check something here. Currently it seems fabs + fneg aren't -; into 2 modifiers, although theoretically that should work. - -; GCN-LABEL: {{^}}fneg_fabs_fadd_f64: -; GCN: v_add_f64 {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, -|v{{\[[0-9]+:[0-9]+\]}}| define amdgpu_kernel void @fneg_fabs_fadd_f64(ptr addrspace(1) %out, double %x, double %y) { +; SI-LABEL: fneg_fabs_fadd_f64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 +; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s0, s4 +; SI-NEXT: s_mov_b32 s1, s5 +; SI-NEXT: v_mov_b32_e32 v0, s6 +; SI-NEXT: v_mov_b32_e32 v1, s7 +; SI-NEXT: v_add_f64 v[0:1], s[8:9], -|v[0:1]| +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabs_fadd_f64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s6 +; VI-NEXT: v_mov_b32_e32 v1, s7 +; VI-NEXT: v_add_f64 v[0:1], s[0:1], -|v[0:1]| +; VI-NEXT: v_mov_b32_e32 v2, s4 +; VI-NEXT: v_mov_b32_e32 v3, s5 +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm %fabs = call double @llvm.fabs.f64(double %x) %fsub = fsub double -0.000000e+00, %fabs %fadd = fadd double %y, %fsub @@ -15,6 +38,29 @@ define amdgpu_kernel void @fneg_fabs_fadd_f64(ptr addrspace(1) %out, double %x, } define amdgpu_kernel void @v_fneg_fabs_fadd_f64(ptr addrspace(1) %out, ptr addrspace(1) %xptr, ptr addrspace(1) %yptr) { +; SI-LABEL: v_fneg_fabs_fadd_f64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_add_f64 v[0:1], s[4:5], -|s[4:5]| +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: v_fneg_fabs_fadd_f64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_add_f64 v[0:1], s[2:3], -|s[2:3]| +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm %x = load double, ptr addrspace(1) %xptr, align 8 %y = load double, ptr addrspace(1) %xptr, align 8 %fabs = call double @llvm.fabs.f64(double %x) @@ -24,9 +70,34 @@ define amdgpu_kernel void @v_fneg_fabs_fadd_f64(ptr addrspace(1) %out, ptr addrs ret void } -; GCN-LABEL: {{^}}fneg_fabs_fmul_f64: -; GCN: v_mul_f64 {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, -|v{{\[[0-9]+:[0-9]+\]}}| define amdgpu_kernel void @fneg_fabs_fmul_f64(ptr addrspace(1) %out, double %x, double %y) { +; SI-LABEL: fneg_fabs_fmul_f64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 +; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s0, s4 +; SI-NEXT: s_mov_b32 s1, s5 +; SI-NEXT: v_mov_b32_e32 v0, s6 +; SI-NEXT: v_mov_b32_e32 v1, s7 +; SI-NEXT: v_mul_f64 v[0:1], s[8:9], -|v[0:1]| +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabs_fmul_f64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s6 +; VI-NEXT: v_mov_b32_e32 v1, s7 +; VI-NEXT: v_mul_f64 v[0:1], s[0:1], -|v[0:1]| +; VI-NEXT: v_mov_b32_e32 v2, s4 +; VI-NEXT: v_mov_b32_e32 v3, s5 +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm %fabs = call double @llvm.fabs.f64(double %x) %fsub = fsub double -0.000000e+00, %fabs %fmul = fmul double %y, %fsub @@ -34,8 +105,32 @@ define amdgpu_kernel void @fneg_fabs_fmul_f64(ptr addrspace(1) %out, double %x, ret void } -; GCN-LABEL: {{^}}fneg_fabs_free_f64: define amdgpu_kernel void @fneg_fabs_free_f64(ptr addrspace(1) %out, i64 %in) { +; SI-LABEL: fneg_fabs_free_f64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_bitset1_b32 s3, 31 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: v_mov_b32_e32 v0, s2 +; SI-NEXT: v_mov_b32_e32 v1, s3 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabs_free_f64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: s_or_b32 s0, s3, 0x80000000 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: v_mov_b32_e32 v3, s0 +; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3] +; VI-NEXT: s_endpgm %bc = bitcast i64 %in to double %fabs = call double @llvm.fabs.f64(double %bc) %fsub = fsub double -0.000000e+00, %fabs @@ -43,10 +138,32 @@ define amdgpu_kernel void @fneg_fabs_free_f64(ptr addrspace(1) %out, i64 %in) { ret void } -; GCN-LABEL: {{^}}fneg_fabs_fn_free_f64: -; SI: s_bitset1_b32 -; VI: s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80000000 define amdgpu_kernel void @fneg_fabs_fn_free_f64(ptr addrspace(1) %out, i64 %in) { +; SI-LABEL: fneg_fabs_fn_free_f64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_bitset1_b32 s3, 31 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: v_mov_b32_e32 v0, s2 +; SI-NEXT: v_mov_b32_e32 v1, s3 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabs_fn_free_f64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: s_or_b32 s0, s3, 0x80000000 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: v_mov_b32_e32 v3, s0 +; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3] +; VI-NEXT: s_endpgm %bc = bitcast i64 %in to double %fabs = call double @fabs(double %bc) %fsub = fsub double -0.000000e+00, %fabs @@ -54,38 +171,126 @@ define amdgpu_kernel void @fneg_fabs_fn_free_f64(ptr addrspace(1) %out, i64 %in) ret void } -; GCN-LABEL: {{^}}fneg_fabs_f64: -; SI-DAG: s_load_dwordx2 s[[[LO_X:[0-9]+]]:[[HI_X:[0-9]+]]], s[{{[0-9]+:[0-9]+}}], 0x13 -; VI-DAG: s_load_dwordx2 s[[[LO_X:[0-9]+]]:[[HI_X:[0-9]+]]], s[{{[0-9]+:[0-9]+}}], 0x4c -; GCN-DAG: s_bitset1_b32 s[[HI_X]], 31 -; GCN-DAG: v_mov_b32_e32 v[[LO_V:[0-9]+]], s[[LO_X]] -; GCN-DAG: v_mov_b32_e32 v[[HI_V:[0-9]+]], s[[HI_X]] -; GCN: buffer_store_dwordx2 v[[[LO_V]]:[[HI_V]]] define amdgpu_kernel void @fneg_fabs_f64(ptr addrspace(1) %out, [8 x i32], double %in) { +; SI-LABEL: fneg_fabs_f64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x13 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_bitset1_b32 s5, 31 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: v_mov_b32_e32 v1, s5 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabs_f64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x4c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_bitset1_b32 s3, 31 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm %fabs = call double @llvm.fabs.f64(double %in) %fsub = fsub double -0.000000e+00, %fabs store double %fsub, ptr addrspace(1) %out, align 8 ret void } -; GCN-LABEL: {{^}}fneg_fabs_v2f64: -; GCN-NOT: 0x80000000 -; GCN: s_bitset1_b32 s{{[0-9]+}}, 31 -; GCN: s_bitset1_b32 s{{[0-9]+}}, 31 define amdgpu_kernel void @fneg_fabs_v2f64(ptr addrspace(1) %out, <2 x double> %in) { +; SI-LABEL: fneg_fabs_v2f64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xd +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_bitset1_b32 s7, 31 +; SI-NEXT: s_bitset1_b32 s5, 31 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: v_mov_b32_e32 v2, s6 +; SI-NEXT: v_mov_b32_e32 v1, s5 +; SI-NEXT: v_mov_b32_e32 v3, s7 +; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabs_v2f64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x34 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_or_b32 s2, s7, 0x80000000 +; VI-NEXT: s_or_b32 s3, s5, 0x80000000 +; VI-NEXT: v_mov_b32_e32 v5, s1 +; VI-NEXT: v_mov_b32_e32 v0, s4 +; VI-NEXT: v_mov_b32_e32 v2, s6 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v3, s2 +; VI-NEXT: v_mov_b32_e32 v4, s0 +; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; VI-NEXT: s_endpgm %fabs = call <2 x double> @llvm.fabs.v2f64(<2 x double> %in) %fsub = fsub <2 x double> , %fabs store <2 x double> %fsub, ptr addrspace(1) %out ret void } -; GCN-LABEL: {{^}}fneg_fabs_v4f64: -; GCN-NOT: 0x80000000 -; GCN: s_bitset1_b32 s{{[0-9]+}}, 31 -; GCN: s_bitset1_b32 s{{[0-9]+}}, 31 -; GCN: s_bitset1_b32 s{{[0-9]+}}, 31 -; GCN: s_bitset1_b32 s{{[0-9]+}}, 31 define amdgpu_kernel void @fneg_fabs_v4f64(ptr addrspace(1) %out, <4 x double> %in) { +; SI-LABEL: fneg_fabs_v4f64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x11 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_bitset1_b32 s7, 31 +; SI-NEXT: s_bitset1_b32 s11, 31 +; SI-NEXT: s_bitset1_b32 s9, 31 +; SI-NEXT: s_bitset1_b32 s5, 31 +; SI-NEXT: v_mov_b32_e32 v0, s8 +; SI-NEXT: v_mov_b32_e32 v2, s10 +; SI-NEXT: v_mov_b32_e32 v4, s4 +; SI-NEXT: v_mov_b32_e32 v6, s6 +; SI-NEXT: v_mov_b32_e32 v1, s9 +; SI-NEXT: v_mov_b32_e32 v3, s11 +; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16 +; SI-NEXT: v_mov_b32_e32 v5, s5 +; SI-NEXT: v_mov_b32_e32 v7, s7 +; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabs_v4f64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x44 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_bitset1_b32 s7, 31 +; VI-NEXT: s_bitset1_b32 s5, 31 +; VI-NEXT: s_or_b32 s2, s11, 0x80000000 +; VI-NEXT: s_or_b32 s3, s9, 0x80000000 +; VI-NEXT: v_mov_b32_e32 v3, s2 +; VI-NEXT: s_add_u32 s2, s0, 16 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: s_addc_u32 s3, s1, 0 +; VI-NEXT: v_mov_b32_e32 v5, s3 +; VI-NEXT: v_mov_b32_e32 v0, s8 +; VI-NEXT: v_mov_b32_e32 v2, s10 +; VI-NEXT: v_mov_b32_e32 v4, s2 +; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; VI-NEXT: v_mov_b32_e32 v5, s1 +; VI-NEXT: v_mov_b32_e32 v0, s4 +; VI-NEXT: v_mov_b32_e32 v1, s5 +; VI-NEXT: v_mov_b32_e32 v2, s6 +; VI-NEXT: v_mov_b32_e32 v3, s7 +; VI-NEXT: v_mov_b32_e32 v4, s0 +; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; VI-NEXT: s_endpgm %fabs = call <4 x double> @llvm.fabs.v4f64(<4 x double> %in) %fsub = fsub <4 x double> , %fabs store <4 x double> %fsub, ptr addrspace(1) %out @@ -96,3 +301,5 @@ declare double @fabs(double) readnone declare double @llvm.fabs.f64(double) readnone declare <2 x double> @llvm.fabs.v2f64(<2 x double>) readnone declare <4 x double> @llvm.fabs.v4f64(<4 x double>) readnone +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GCN: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll index b0c17828cb13..3c000d4fa63a 100644 --- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll +++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll @@ -1,11 +1,31 @@ -; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -verify-machineinstrs < %s | FileCheck --check-prefixes=SI,FUNC %s -; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck --check-prefixes=VI,FUNC %s -; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=redwood < %s | FileCheck --check-prefixes=R600,FUNC %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn < %s | FileCheck --check-prefixes=SI,FUNC %s +; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga < %s | FileCheck --check-prefixes=VI,FUNC %s -; FUNC-LABEL: {{^}}fneg_fabsf_fadd_f32: -; SI-NOT: and -; SI: v_sub_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, |{{v[0-9]+}}| define amdgpu_kernel void @fneg_fabsf_fadd_f32(ptr addrspace(1) %out, float %x, float %y) { +; SI-LABEL: fneg_fabsf_fadd_f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: v_mov_b32_e32 v0, s2 +; SI-NEXT: v_sub_f32_e64 v0, s3, |v0| +; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabsf_fadd_f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_sub_f32_e64 v2, s3, |v0| +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %fabs = call float @llvm.fabs.f32(float %x) %fsub = fsub float -0.000000e+00, %fabs %fadd = fadd float %y, %fsub @@ -13,11 +33,30 @@ define amdgpu_kernel void @fneg_fabsf_fadd_f32(ptr addrspace(1) %out, float %x, ret void } -; FUNC-LABEL: {{^}}fneg_fabsf_fmul_f32: -; SI-NOT: and -; SI: v_mul_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, -|{{v[0-9]+}}| -; SI-NOT: and define amdgpu_kernel void @fneg_fabsf_fmul_f32(ptr addrspace(1) %out, float %x, float %y) { +; SI-LABEL: fneg_fabsf_fmul_f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: v_mov_b32_e32 v0, s2 +; SI-NEXT: v_mul_f32_e64 v0, s3, -|v0| +; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabsf_fmul_f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mul_f32_e64 v2, s3, -|v0| +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %fabs = call float @llvm.fabs.f32(float %x) %fsub = fsub float -0.000000e+00, %fabs %fmul = fmul float %y, %fsub @@ -25,18 +64,30 @@ define amdgpu_kernel void @fneg_fabsf_fmul_f32(ptr addrspace(1) %out, float %x, ret void } -; DAGCombiner will transform: -; (fabsf (f32 bitcast (i32 a))) => (f32 bitcast (and (i32 a), 0x7FFFFFFF)) -; unless isFabsFree returns true - -; FUNC-LABEL: {{^}}fneg_fabsf_free_f32: -; R600-NOT: AND -; R600: |PV.{{[XYZW]}}| -; R600: -PV - -; SI: s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80000000 -; VI: s_bitset1_b32 s{{[0-9]+}}, 31 define amdgpu_kernel void @fneg_fabsf_free_f32(ptr addrspace(1) %out, i32 %in) { +; SI-LABEL: fneg_fabsf_free_f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s2, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_or_b32 s4, s2, 0x80000000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabsf_free_f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s2, s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_bitset1_b32 s2, 31 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %bc = bitcast i32 %in to float %fabs = call float @llvm.fabs.f32(float %bc) %fsub = fsub float -0.000000e+00, %fabs @@ -44,13 +95,30 @@ define amdgpu_kernel void @fneg_fabsf_free_f32(ptr addrspace(1) %out, i32 %in) { ret void } -; FUNC-LABEL: {{^}}fneg_fabsf_fn_free_f32: -; R600-NOT: AND -; R600: |PV.{{[XYZW]}}| -; R600: -PV - -; SI: s_load_dwordx2 s[0:1], s[2:3], 0x9 define amdgpu_kernel void @fneg_fabsf_fn_free_f32(ptr addrspace(1) %out, i32 %in) { +; SI-LABEL: fneg_fabsf_fn_free_f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s4, s[2:3], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_bitset1_b32 s4, 31 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabsf_fn_free_f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s4, s[2:3], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_or_b32 s2, s4, 0x80000000 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %bc = bitcast i32 %in to float %fabs = call float @fabsf(float %bc) %fsub = fsub float -0.000000e+00, %fabs @@ -58,18 +126,68 @@ define amdgpu_kernel void @fneg_fabsf_fn_free_f32(ptr addrspace(1) %out, i32 %in ret void } -; FUNC-LABEL: {{^}}fneg_fabsf_f32: -; SI: s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80000000 define amdgpu_kernel void @fneg_fabsf_f32(ptr addrspace(1) %out, float %in) { +; SI-LABEL: fneg_fabsf_f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s2, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_or_b32 s4, s2, 0x80000000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabsf_f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s2, s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_bitset1_b32 s2, 31 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %fabs = call float @llvm.fabs.f32(float %in) %fsub = fsub float -0.000000e+00, %fabs store float %fsub, ptr addrspace(1) %out, align 4 ret void } -; FUNC-LABEL: {{^}}v_fneg_fabsf_f32: -; SI: v_or_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}} define amdgpu_kernel void @v_fneg_fabsf_f32(ptr addrspace(1) %out, ptr addrspace(1) %in) { +; SI-LABEL: v_fneg_fabsf_f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_mov_b32 s10, s6 +; SI-NEXT: s_mov_b32 s11, s7 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s8, s2 +; SI-NEXT: s_mov_b32 s9, s3 +; SI-NEXT: buffer_load_dword v0, off, s[8:11], 0 +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: v_or_b32_e32 v0, 0x80000000, v0 +; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: v_fneg_fabsf_f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: flat_load_dword v2, v[0:1] +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: v_or_b32_e32 v2, 0x80000000, v2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %val = load float, ptr addrspace(1) %in, align 4 %fabs = call float @llvm.fabs.f32(float %val) %fsub = fsub float -0.000000e+00, %fabs @@ -77,28 +195,76 @@ define amdgpu_kernel void @v_fneg_fabsf_f32(ptr addrspace(1) %out, ptr addrspace ret void } -; FUNC-LABEL: {{^}}fneg_fabsf_v2f32: -; R600: |{{(PV|T[0-9])\.[XYZW]}}| -; R600: -PV -; R600: |{{(PV|T[0-9])\.[XYZW]}}| -; R600: -PV - -; FIXME: In this case two uses of the constant should be folded -; SI: s_bitset1_b32 s{{[0-9]+}}, 31 -; SI: s_bitset1_b32 s{{[0-9]+}}, 31 define amdgpu_kernel void @fneg_fabsf_v2f32(ptr addrspace(1) %out, <2 x float> %in) { +; SI-LABEL: fneg_fabsf_v2f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_bitset1_b32 s3, 31 +; SI-NEXT: s_bitset1_b32 s2, 31 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: v_mov_b32_e32 v0, s2 +; SI-NEXT: v_mov_b32_e32 v1, s3 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabsf_v2f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_bitset1_b32 s3, 31 +; VI-NEXT: s_bitset1_b32 s2, 31 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in) %fsub = fsub <2 x float> , %fabs store <2 x float> %fsub, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}fneg_fabsf_v4f32: -; SI: s_bitset1_b32 s{{[0-9]+}}, 31 -; SI: s_bitset1_b32 s{{[0-9]+}}, 31 -; SI: s_bitset1_b32 s{{[0-9]+}}, 31 -; SI: s_bitset1_b32 s{{[0-9]+}}, 31 define amdgpu_kernel void @fneg_fabsf_v4f32(ptr addrspace(1) %out, <4 x float> %in) { +; SI-LABEL: fneg_fabsf_v4f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xd +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_bitset1_b32 s7, 31 +; SI-NEXT: s_bitset1_b32 s6, 31 +; SI-NEXT: s_bitset1_b32 s5, 31 +; SI-NEXT: s_bitset1_b32 s4, 31 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: v_mov_b32_e32 v1, s5 +; SI-NEXT: v_mov_b32_e32 v2, s6 +; SI-NEXT: v_mov_b32_e32 v3, s7 +; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fabsf_v4f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x34 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_or_b32 s2, s7, 0x80000000 +; VI-NEXT: s_or_b32 s3, s6, 0x80000000 +; VI-NEXT: s_bitset1_b32 s5, 31 +; VI-NEXT: s_bitset1_b32 s4, 31 +; VI-NEXT: v_mov_b32_e32 v5, s1 +; VI-NEXT: v_mov_b32_e32 v0, s4 +; VI-NEXT: v_mov_b32_e32 v1, s5 +; VI-NEXT: v_mov_b32_e32 v2, s3 +; VI-NEXT: v_mov_b32_e32 v3, s2 +; VI-NEXT: v_mov_b32_e32 v4, s0 +; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; VI-NEXT: s_endpgm %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %in) %fsub = fsub <4 x float> , %fabs store <4 x float> %fsub, ptr addrspace(1) %out @@ -112,3 +278,5 @@ declare <4 x float> @llvm.fabs.v4f32(<4 x float>) readnone !llvm.module.flags = !{!0} !0 = !{i32 1, !"amdhsa_code_object_version", i32 500} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; FUNC: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/fneg.ll b/llvm/test/CodeGen/AMDGPU/fneg.ll index 03ca780c9032..d78bdfe08772 100644 --- a/llvm/test/CodeGen/AMDGPU/fneg.ll +++ b/llvm/test/CodeGen/AMDGPU/fneg.ll @@ -1,89 +1,279 @@ -; RUN: llc -mtriple=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=FUNC,GCN,SI %s -; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=FUNC,GCN,VI %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=FUNC,GCN,GFX11 %s -; RUN: not llc -mtriple=r600 -mcpu=redwood < %s | FileCheck -enable-var-scope -check-prefixes=FUNC,R600 %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s +; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX11 %s +; RUN: not llc -mtriple=r600 -mcpu=redwood < %s -; FUNC-LABEL: {{^}}s_fneg_f32: -; R600: -PV - -; GCN: s_load_{{dword|b32}} [[VAL:s[0-9]+]] -; GCN: s_xor_b32 [[NEG_VAL:s[0-9]+]], [[VAL]], 0x80000000 -; GCN: v_mov_b32_e32 v{{[0-9]+}}, [[NEG_VAL]] define amdgpu_kernel void @s_fneg_f32(ptr addrspace(1) %out, float %in) { +; SI-LABEL: s_fneg_f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s4, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_xor_b32 s4, s4, 0x80000000 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: s_fneg_f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s2, s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_xor_b32 s2, s2, 0x80000000 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: s_fneg_f32: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x2c +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_xor_b32 s2, s2, 0x80000000 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 +; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %fneg = fsub float -0.000000e+00, %in store float %fneg, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}s_fneg_v2f32: -; R600: -PV -; R600: -PV - -; GCN: s_xor_b32 {{s[0-9]+}}, {{s[0-9]+}}, 0x80000000 -; GCN: s_xor_b32 {{s[0-9]+}}, {{s[0-9]+}}, 0x80000000 define amdgpu_kernel void @s_fneg_v2f32(ptr addrspace(1) nocapture %out, <2 x float> %in) { +; SI-LABEL: s_fneg_v2f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: s_xor_b32 s0, s3, 0x80000000 +; SI-NEXT: s_xor_b32 s1, s2, 0x80000000 +; SI-NEXT: v_mov_b32_e32 v0, s1 +; SI-NEXT: v_mov_b32_e32 v1, s0 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: s_fneg_v2f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_xor_b32 s3, s3, 0x80000000 +; VI-NEXT: s_xor_b32 s2, s2, 0x80000000 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: s_fneg_v2f32: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_xor_b32 s2, s2, 0x80000000 +; GFX11-NEXT: s_xor_b32 s3, s3, 0x80000000 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3 +; GFX11-NEXT: v_mov_b32_e32 v0, s2 +; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %fneg = fsub <2 x float> , %in store <2 x float> %fneg, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}s_fneg_v4f32: -; R600: -PV -; R600: -T -; R600: -PV -; R600: -PV - -; GCN: s_xor_b32 {{s[0-9]+}}, {{s[0-9]+}}, 0x80000000 -; GCN: s_xor_b32 {{s[0-9]+}}, {{s[0-9]+}}, 0x80000000 -; GCN: s_xor_b32 {{s[0-9]+}}, {{s[0-9]+}}, 0x80000000 -; GCN: s_xor_b32 {{s[0-9]+}}, {{s[0-9]+}}, 0x80000000 define amdgpu_kernel void @s_fneg_v4f32(ptr addrspace(1) nocapture %out, <4 x float> %in) { +; SI-LABEL: s_fneg_v4f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xd +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_xor_b32 s7, s7, 0x80000000 +; SI-NEXT: s_xor_b32 s6, s6, 0x80000000 +; SI-NEXT: s_xor_b32 s5, s5, 0x80000000 +; SI-NEXT: s_xor_b32 s4, s4, 0x80000000 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: v_mov_b32_e32 v1, s5 +; SI-NEXT: v_mov_b32_e32 v2, s6 +; SI-NEXT: v_mov_b32_e32 v3, s7 +; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: s_fneg_v4f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x34 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_xor_b32 s2, s7, 0x80000000 +; VI-NEXT: s_xor_b32 s3, s6, 0x80000000 +; VI-NEXT: s_xor_b32 s5, s5, 0x80000000 +; VI-NEXT: s_xor_b32 s4, s4, 0x80000000 +; VI-NEXT: v_mov_b32_e32 v5, s1 +; VI-NEXT: v_mov_b32_e32 v0, s4 +; VI-NEXT: v_mov_b32_e32 v1, s5 +; VI-NEXT: v_mov_b32_e32 v2, s3 +; VI-NEXT: v_mov_b32_e32 v3, s2 +; VI-NEXT: v_mov_b32_e32 v4, s0 +; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: s_fneg_v4f32: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x34 +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_xor_b32 s2, s7, 0x80000000 +; GFX11-NEXT: s_xor_b32 s3, s6, 0x80000000 +; GFX11-NEXT: s_xor_b32 s4, s4, 0x80000000 +; GFX11-NEXT: s_xor_b32 s5, s5, 0x80000000 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s5 +; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s2 +; GFX11-NEXT: v_mov_b32_e32 v2, s3 +; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %fneg = fsub <4 x float> , %in store <4 x float> %fneg, ptr addrspace(1) %out ret void } -; DAGCombiner will transform: -; (fneg (f32 bitcast (i32 a))) => (f32 bitcast (xor (i32 a), 0x80000000)) -; unless the target returns true for isNegFree() - -; FUNC-LABEL: {{^}}fsub0_f32: - -; GCN: v_sub_f32_e64 v{{[0-9]}}, 0, s{{[0-9]+$}} - -; R600-NOT: XOR -; R600: -KC0[2].Z define amdgpu_kernel void @fsub0_f32(ptr addrspace(1) %out, i32 %in) { +; SI-LABEL: fsub0_f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s4, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_sub_f32_e64 v0, 0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fsub0_f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s2, s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_sub_f32_e64 v2, 0, s2 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: fsub0_f32: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x2c +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX11-NEXT: v_mov_b32_e32 v0, 0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: v_sub_f32_e64 v1, 0, s2 +; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %bc = bitcast i32 %in to float %fsub = fsub float 0.0, %bc store float %fsub, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}fneg_free_f32: -; SI: s_load_dword [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xb -; VI: s_load_dword [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c -; GFX11: s_load_b32 [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c - -; GCN: s_xor_b32 [[RES:s[0-9]+]], [[NEG_VALUE]], 0x80000000 -; GCN: v_mov_b32_e32 [[V_RES:v[0-9]+]], [[RES]] -; GCN: buffer_store_{{dword|b32}} [[V_RES]] -; R600-NOT: XOR -; R600: -PV.W define amdgpu_kernel void @fneg_free_f32(ptr addrspace(1) %out, i32 %in) { +; SI-LABEL: fneg_free_f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s4, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_xor_b32 s4, s4, 0x80000000 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_free_f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s2, s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_xor_b32 s2, s2, 0x80000000 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: fneg_free_f32: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x2c +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_xor_b32 s2, s2, 0x80000000 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 +; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %bc = bitcast i32 %in to float %fsub = fsub float -0.0, %bc store float %fsub, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}fneg_fold_f32: -; SI: s_load_dword [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xb -; VI: s_load_dword [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c -; GFX11: s_load_{{dword|b32}} [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c -; GCN-NOT: xor -; GCN: v_mul_f32_e64 v{{[0-9]+}}, -[[NEG_VALUE]], [[NEG_VALUE]] define amdgpu_kernel void @fneg_fold_f32(ptr addrspace(1) %out, float %in) { +; SI-LABEL: fneg_fold_f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s4, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_mul_f32_e64 v0, -s4, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: fneg_fold_f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s2, s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mul_f32_e64 v2, -s2, s2 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: fneg_fold_f32: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x2c +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX11-NEXT: v_mov_b32_e32 v0, 0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: v_mul_f32_e64 v1, -s2, s2 +; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %fsub = fsub float -0.0, %in %fmul = fmul float %fsub, %in store float %fmul, ptr addrspace(1) %out @@ -91,9 +281,41 @@ define amdgpu_kernel void @fneg_fold_f32(ptr addrspace(1) %out, float %in) { } ; Make sure we turn some integer operations back into fabs -; FUNC-LABEL: {{^}}bitpreserve_fneg_f32: -; GCN: v_mul_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, -4.0 define amdgpu_kernel void @bitpreserve_fneg_f32(ptr addrspace(1) %out, float %in) { +; SI-LABEL: bitpreserve_fneg_f32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s4, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_mul_f32_e64 v0, s4, -4.0 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: bitpreserve_fneg_f32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s2, s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mul_f32_e64 v2, s2, -4.0 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: bitpreserve_fneg_f32: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x2c +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX11-NEXT: v_mov_b32_e32 v0, 0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: v_mul_f32_e64 v1, s2, -4.0 +; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %in.bc = bitcast float %in to i32 %int.abs = xor i32 %in.bc, 2147483648 %bc = bitcast i32 %int.abs to float @@ -102,29 +324,94 @@ define amdgpu_kernel void @bitpreserve_fneg_f32(ptr addrspace(1) %out, float %in ret void } -; FUNC-LABEL: {{^}}s_fneg_i32: -; GCN: s_load_{{dword|b32}} [[IN:s[0-9]+]] -; GCN: s_xor_b32 [[FNEG:s[0-9]+]], [[IN]], 0x80000000 -; GCN: v_mov_b32_e32 [[V_FNEG:v[0-9]+]], [[FNEG]] define amdgpu_kernel void @s_fneg_i32(ptr addrspace(1) %out, i32 %in) { +; SI-LABEL: s_fneg_i32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s4, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_xor_b32 s4, s4, 0x80000000 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: s_fneg_i32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s2, s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_xor_b32 s2, s2, 0x80000000 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: s_fneg_i32: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x2c +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_xor_b32 s2, s2, 0x80000000 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 +; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %fneg = xor i32 %in, -2147483648 store i32 %fneg, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}v_fneg_i32: -; GCN: s_waitcnt -; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0 -; GCN-NEXT: s_setpc_b64 define i32 @v_fneg_i32(i32 %in) { +; GCN-LABEL: v_fneg_i32: +; GCN: ; %bb.0: +; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0 +; GCN-NEXT: s_setpc_b64 s[30:31] %fneg = xor i32 %in, -2147483648 ret i32 %fneg } -; FUNC-LABEL: {{^}}s_fneg_i32_fp_use: -; GCN: s_load_{{dword|b32}} [[IN:s[0-9]+]] -; GCN: v_sub_f32_e64 v{{[0-9]+}}, 2.0, [[IN]] define amdgpu_kernel void @s_fneg_i32_fp_use(ptr addrspace(1) %out, i32 %in) { +; SI-LABEL: s_fneg_i32_fp_use: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s4, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_sub_f32_e64 v0, 2.0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: s_fneg_i32_fp_use: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s2, s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_sub_f32_e64 v2, 2.0, s2 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: s_fneg_i32_fp_use: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x2c +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX11-NEXT: v_mov_b32_e32 v0, 0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: v_sub_f32_e64 v1, 2.0, s2 +; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %fneg = xor i32 %in, -2147483648 %bitcast = bitcast i32 %fneg to float %fadd = fadd float %bitcast, 2.0 @@ -132,37 +419,105 @@ define amdgpu_kernel void @s_fneg_i32_fp_use(ptr addrspace(1) %out, i32 %in) { ret void } -; FUNC-LABEL: {{^}}v_fneg_i32_fp_use: -; GCN: s_waitcnt -; GCN-NEXT: v_sub_f32_e32 v0, 2.0, v0 -; GCN-NEXT: s_setpc_b64 define float @v_fneg_i32_fp_use(i32 %in) { +; GCN-LABEL: v_fneg_i32_fp_use: +; GCN: ; %bb.0: +; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-NEXT: v_sub_f32_e32 v0, 2.0, v0 +; GCN-NEXT: s_setpc_b64 s[30:31] %fneg = xor i32 %in, -2147483648 %bitcast = bitcast i32 %fneg to float %fadd = fadd float %bitcast, 2.0 ret float %fadd } -; FUNC-LABEL: {{^}}s_fneg_i64: -; GCN: s_xor_b32 s[[NEG_HI:[0-9]+]], s{{[0-9]+}}, 0x80000000 define amdgpu_kernel void @s_fneg_i64(ptr addrspace(1) %out, i64 %in) { +; SI-LABEL: s_fneg_i64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_xor_b32 s0, s3, 0x80000000 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: v_mov_b32_e32 v0, s2 +; SI-NEXT: v_mov_b32_e32 v1, s0 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: s_fneg_i64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: s_xor_b32 s0, s3, 0x80000000 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: v_mov_b32_e32 v3, s0 +; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3] +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: s_fneg_i64: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_xor_b32 s3, s3, 0x80000000 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3 +; GFX11-NEXT: v_mov_b32_e32 v0, s2 +; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %fneg = xor i64 %in, -9223372036854775808 store i64 %fneg, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}v_fneg_i64: -; GCN: s_waitcnt -; GCN-NEXT: v_xor_b32_e32 v1, 0x80000000, v1 -; GCN-NEXT: s_setpc_b64 define i64 @v_fneg_i64(i64 %in) { +; GCN-LABEL: v_fneg_i64: +; GCN: ; %bb.0: +; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-NEXT: v_xor_b32_e32 v1, 0x80000000, v1 +; GCN-NEXT: s_setpc_b64 s[30:31] %fneg = xor i64 %in, -9223372036854775808 ret i64 %fneg } -; FUNC-LABEL: {{^}}s_fneg_i64_fp_use: -; GCN: v_add_f64 v{{\[[0-9]+:[0-9]+\]}}, -s{{\[[0-9]+:[0-9]+\]}}, 2.0 define amdgpu_kernel void @s_fneg_i64_fp_use(ptr addrspace(1) %out, i64 %in) { +; SI-LABEL: s_fneg_i64_fp_use: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_add_f64 v[0:1], -s[2:3], 2.0 +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: s_fneg_i64_fp_use: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_add_f64 v[0:1], -s[2:3], 2.0 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: s_fneg_i64_fp_use: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 +; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: v_add_f64 v[0:1], -s[2:3], 2.0 +; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %fneg = xor i64 %in, -9223372036854775808 %bitcast = bitcast i64 %fneg to double %fadd = fadd double %bitcast, 2.0 @@ -170,34 +525,65 @@ define amdgpu_kernel void @s_fneg_i64_fp_use(ptr addrspace(1) %out, i64 %in) { ret void } -; FUNC-LABEL: {{^}}v_fneg_i64_fp_use: -; GCN: s_waitcnt -; GCN-NEXT: v_add_f64 v[0:1], -v[0:1], 2.0 -; GCN-NEXT: s_setpc_b64 define double @v_fneg_i64_fp_use(i64 %in) { +; GCN-LABEL: v_fneg_i64_fp_use: +; GCN: ; %bb.0: +; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-NEXT: v_add_f64 v[0:1], -v[0:1], 2.0 +; GCN-NEXT: s_setpc_b64 s[30:31] %fneg = xor i64 %in, -9223372036854775808 %bitcast = bitcast i64 %fneg to double %fadd = fadd double %bitcast, 2.0 ret double %fadd } -; FUNC-LABEL: {{^}}v_fneg_i16: -; GCN: s_waitcnt -; GCN-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0 -; GCN-NEXT: s_setpc_b64 define i16 @v_fneg_i16(i16 %in) { +; GCN-LABEL: v_fneg_i16: +; GCN: ; %bb.0: +; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0 +; GCN-NEXT: s_setpc_b64 s[30:31] %fneg = xor i16 %in, -32768 ret i16 %fneg } -; FUNC-LABEL: {{^}}s_fneg_i16_fp_use: -; SI: v_cvt_f32_f16_e32 [[CVT0:v[0-9]+]], s{{[0-9]+}} -; SI: v_sub_f32_e32 [[ADD:v[0-9]+]], 2.0, [[CVT0]] -; SI: v_cvt_f16_f32_e32 [[CVT1:v[0-9]+]], [[ADD]] - -; VI: s_load_dword [[IN:s[0-9]+]] -; VI: v_sub_f16_e64 v{{[0-9]+}}, 2.0, [[IN]] define amdgpu_kernel void @s_fneg_i16_fp_use(ptr addrspace(1) %out, i16 %in) { +; SI-LABEL: s_fneg_i16_fp_use: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s2, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_cvt_f32_f16_e32 v0, s2 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_sub_f32_e32 v0, 2.0, v0 +; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 +; SI-NEXT: buffer_store_short v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: s_fneg_i16_fp_use: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s2, s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_sub_f16_e64 v2, 2.0, s2 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: flat_store_short v[0:1], v2 +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: s_fneg_i16_fp_use: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x2c +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX11-NEXT: v_mov_b32_e32 v0, 0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: v_sub_f16_e64 v1, 2.0, s2 +; GFX11-NEXT: global_store_b16 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %fneg = xor i16 %in, -32768 %bitcast = bitcast i16 %fneg to half %fadd = fadd half %bitcast, 2.0 @@ -205,69 +591,157 @@ define amdgpu_kernel void @s_fneg_i16_fp_use(ptr addrspace(1) %out, i16 %in) { ret void } -; FUNC-LABEL: {{^}}v_fneg_i16_fp_use: -; SI: s_waitcnt -; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_sub_f32_e32 v0, 2.0, v0 -; SI-NEXT: s_setpc_b64 - -; VI: s_waitcnt -; VI-NEXT: v_sub_f16_e32 v0, 2.0, v0 -; VI-NEXT: s_setpc_b64 define half @v_fneg_i16_fp_use(i16 %in) { +; SI-LABEL: v_fneg_i16_fp_use: +; SI: ; %bb.0: +; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 +; SI-NEXT: v_sub_f32_e32 v0, 2.0, v0 +; SI-NEXT: s_setpc_b64 s[30:31] +; +; VI-LABEL: v_fneg_i16_fp_use: +; VI: ; %bb.0: +; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VI-NEXT: v_sub_f16_e32 v0, 2.0, v0 +; VI-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: v_fneg_i16_fp_use: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_sub_f16_e32 v0, 2.0, v0 +; GFX11-NEXT: s_setpc_b64 s[30:31] %fneg = xor i16 %in, -32768 %bitcast = bitcast i16 %fneg to half %fadd = fadd half %bitcast, 2.0 ret half %fadd } -; FUNC-LABEL: {{^}}s_fneg_v2i16: -; SI: s_xor_b32 s4, s4, 0x80008000 - -; VI: s_lshr_b32 s5, s4, 16 -; VI: s_xor_b32 s4, s4, 0x8000 -; VI: s_xor_b32 s5, s5, 0x8000 -; VI: s_and_b32 s4, s4, 0xffff -; VI: s_lshl_b32 s5, s5, 16 -; VI: s_or_b32 s4, s4, s5 define amdgpu_kernel void @s_fneg_v2i16(ptr addrspace(1) %out, i32 %arg) { +; SI-LABEL: s_fneg_v2i16: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s4, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_xor_b32 s4, s4, 0x80008000 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: s_fneg_v2i16: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s2, s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_lshr_b32 s3, s2, 16 +; VI-NEXT: s_xor_b32 s2, s2, 0x8000 +; VI-NEXT: s_xor_b32 s3, s3, 0x8000 +; VI-NEXT: s_and_b32 s2, s2, 0xffff +; VI-NEXT: s_lshl_b32 s3, s3, 16 +; VI-NEXT: s_or_b32 s2, s2, s3 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: s_fneg_v2i16: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x2c +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_xor_b32 s2, s2, 0x80008000 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 +; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %in = bitcast i32 %arg to <2 x i16> %fneg = xor <2 x i16> %in, store <2 x i16> %fneg, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}v_fneg_v2i16: -; SI: v_xor_b32_e32 v1, 0x8000, v1 -; SI: v_xor_b32_e32 v0, 0x8000, v0 -; SI: v_lshlrev_b32_e32 v2, 16, v1 -; SI: v_and_b32_e32 v0, 0xffff, v0 -; SI: v_or_b32_e32 v0, v0, v2 -; SI: v_and_b32_e32 v1, 0xffff, v1 - -; VI: s_waitcnt -; VI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0 -; VI-NEXT: s_setpc_b64 define <2 x i16> @v_fneg_v2i16(<2 x i16> %in) { +; SI-LABEL: v_fneg_v2i16: +; SI: ; %bb.0: +; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SI-NEXT: v_xor_b32_e32 v1, 0x8000, v1 +; SI-NEXT: v_xor_b32_e32 v0, 0x8000, v0 +; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1 +; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; SI-NEXT: v_or_b32_e32 v0, v0, v2 +; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1 +; SI-NEXT: s_setpc_b64 s[30:31] +; +; VI-LABEL: v_fneg_v2i16: +; VI: ; %bb.0: +; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0 +; VI-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: v_fneg_v2i16: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_xor_b32_e32 v0, 0x80008000, v0 +; GFX11-NEXT: s_setpc_b64 s[30:31] %fneg = xor <2 x i16> %in, ret <2 x i16> %fneg } -; FUNC-LABEL: {{^}}s_fneg_v2i16_fp_use: -; SI: s_lshr_b32 s3, s2, 16 -; SI: v_cvt_f32_f16_e32 v0, s3 -; SI: v_cvt_f32_f16_e32 v1, s2 -; SI: v_sub_f32_e32 v0, 2.0, v0 -; SI: v_sub_f32_e32 v1, 2.0, v1 - -; VI: s_lshr_b32 s5, s4, 16 -; VI: s_xor_b32 s5, s5, 0x8000 -; VI: s_xor_b32 s4, s4, 0x8000 -; VI: v_mov_b32_e32 v0, s5 -; VI: v_add_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; VI: v_add_f16_e64 v1, s4, 2.0 -; VI: v_or_b32_e32 v0, v1, v0 define amdgpu_kernel void @s_fneg_v2i16_fp_use(ptr addrspace(1) %out, i32 %arg) { +; SI-LABEL: s_fneg_v2i16_fp_use: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s2, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_lshr_b32 s3, s2, 16 +; SI-NEXT: v_cvt_f32_f16_e32 v0, s3 +; SI-NEXT: v_cvt_f32_f16_e32 v1, s2 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_sub_f32_e32 v0, 2.0, v0 +; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 +; SI-NEXT: v_sub_f32_e32 v1, 2.0, v1 +; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SI-NEXT: v_or_b32_e32 v0, v1, v0 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: s_fneg_v2i16_fp_use: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s2, s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: v_mov_b32_e32 v0, 0x4000 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_lshr_b32 s3, s2, 16 +; VI-NEXT: s_xor_b32 s3, s3, 0x8000 +; VI-NEXT: s_xor_b32 s2, s2, 0x8000 +; VI-NEXT: v_mov_b32_e32 v2, s3 +; VI-NEXT: v_add_f16_e64 v1, s2, 2.0 +; VI-NEXT: v_add_f16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; VI-NEXT: v_or_b32_e32 v2, v1, v0 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; GFX11-LABEL: s_fneg_v2i16_fp_use: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x2c +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX11-NEXT: v_mov_b32_e32 v0, 0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: v_pk_add_f16 v1, s2, 2.0 op_sel_hi:[1,0] neg_lo:[1,0] neg_hi:[1,0] +; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %in = bitcast i32 %arg to <2 x i16> %fneg = xor <2 x i16> %in, %bitcast = bitcast <2 x i16> %fneg to <2 x half> @@ -276,20 +750,31 @@ define amdgpu_kernel void @s_fneg_v2i16_fp_use(ptr addrspace(1) %out, i32 %arg) ret void } -; FUNC-LABEL: {{^}}v_fneg_v2i16_fp_use: -; SI: v_lshrrev_b32_e32 v1, 16, v0 -; SI: v_cvt_f32_f16_e32 v0, v0 -; SI: v_cvt_f32_f16_e32 v1, v1 -; SI: v_sub_f32_e32 v0, 2.0, v0 -; SI: v_sub_f32_e32 v1, 2.0, v1 - -; VI: s_waitcnt -; VI: v_mov_b32_e32 v1, 0x4000 -; VI: v_sub_f16_sdwa v1, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 -; VI: v_sub_f16_e32 v0, 2.0, v0 -; VI: v_or_b32_e32 v0, v0, v1 -; VI: s_setpc_b64 define <2 x half> @v_fneg_v2i16_fp_use(i32 %arg) { +; SI-LABEL: v_fneg_v2i16_fp_use: +; SI: ; %bb.0: +; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 +; SI-NEXT: v_cvt_f32_f16_e32 v1, v1 +; SI-NEXT: v_sub_f32_e32 v0, 2.0, v0 +; SI-NEXT: v_sub_f32_e32 v1, 2.0, v1 +; SI-NEXT: s_setpc_b64 s[30:31] +; +; VI-LABEL: v_fneg_v2i16_fp_use: +; VI: ; %bb.0: +; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v1, 0x4000 +; VI-NEXT: v_sub_f16_sdwa v1, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 +; VI-NEXT: v_sub_f16_e32 v0, 2.0, v0 +; VI-NEXT: v_or_b32_e32 v0, v0, v1 +; VI-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: v_fneg_v2i16_fp_use: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_pk_add_f16 v0, v0, 2.0 op_sel_hi:[1,0] neg_lo:[1,0] neg_hi:[1,0] +; GFX11-NEXT: s_setpc_b64 s[30:31] %in = bitcast i32 %arg to <2 x i16> %fneg = xor <2 x i16> %in, %bitcast = bitcast <2 x i16> %fneg to <2 x half> diff --git a/llvm/test/CodeGen/AMDGPU/llvm.r600.read.local.size.ll b/llvm/test/CodeGen/AMDGPU/llvm.r600.read.local.size.ll index ab035b9de04b..826862e12492 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.r600.read.local.size.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.r600.read.local.size.ll @@ -1,63 +1,160 @@ -; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck --check-prefixes=SI,GCN,SI-NOHSA,FUNC %s -; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck --check-prefixes=VI,VI-NOHSA,GCN,FUNC %s -; RUN: llc -mtriple=r600 -mcpu=redwood < %s | FileCheck --check-prefixes=EG,FUNC %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=amdgcn < %s | FileCheck --check-prefixes=SI,GCN,FUNC %s +; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck --check-prefixes=VI,GCN,FUNC %s +; RUN: llc -mtriple=r600 -mcpu=redwood < %s | FileCheck --check-prefixes=R600,FUNC %s - -; FUNC-LABEL: {{^}}local_size_x: -; EG: MEM_RAT_CACHELESS STORE_RAW [[VAL:T[0-9]+\.X]] -; EG: MOV * [[VAL]], KC0[1].Z - -; SI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x6 -; VI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x18 -; CI-HSA: s_load_dword [[XY:s[0-9]+]], s[4:5], 0x1 -; VI-HSA: s_load_dword [[XY:s[0-9]+]], s[4:5], 0x4 - -; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN: buffer_store_dword [[VVAL]] define amdgpu_kernel void @local_size_x(ptr addrspace(1) %out) { +; SI-LABEL: local_size_x: +; SI: ; %bb.0: ; %entry +; SI-NEXT: s_load_dword s4, s[0:1], 0x6 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: local_size_x: +; VI: ; %bb.0: ; %entry +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; VI-NEXT: s_load_dword s0, s[0:1], 0x18 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; R600-LABEL: local_size_x: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: MOV * T1.X, KC0[1].Z, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) entry: %0 = call i32 @llvm.r600.read.local.size.x() #0 store i32 %0, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}local_size_y: -; EG: MEM_RAT_CACHELESS STORE_RAW [[VAL:T[0-9]+\.X]] -; EG: MOV * [[VAL]], KC0[1].W - -; SI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x7 -; VI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x1c -; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN: buffer_store_dword [[VVAL]] define amdgpu_kernel void @local_size_y(ptr addrspace(1) %out) { +; SI-LABEL: local_size_y: +; SI: ; %bb.0: ; %entry +; SI-NEXT: s_load_dword s4, s[0:1], 0x7 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: local_size_y: +; VI: ; %bb.0: ; %entry +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; VI-NEXT: s_load_dword s0, s[0:1], 0x1c +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; R600-LABEL: local_size_y: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: MOV * T1.X, KC0[1].W, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) entry: %0 = call i32 @llvm.r600.read.local.size.y() #0 store i32 %0, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}local_size_z: -; EG: MEM_RAT_CACHELESS STORE_RAW [[VAL:T[0-9]+\.X]] -; EG: MOV * [[VAL]], KC0[2].X - -; SI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x8 -; VI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x20 -; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN: buffer_store_dword [[VVAL]] define amdgpu_kernel void @local_size_z(ptr addrspace(1) %out) { +; SI-LABEL: local_size_z: +; SI: ; %bb.0: ; %entry +; SI-NEXT: s_load_dword s4, s[0:1], 0x8 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: local_size_z: +; VI: ; %bb.0: ; %entry +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; VI-NEXT: s_load_dword s0, s[0:1], 0x20 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; R600-LABEL: local_size_z: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: MOV * T1.X, KC0[2].X, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) entry: %0 = call i32 @llvm.r600.read.local.size.z() #0 store i32 %0, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}local_size_xy: -; SI-NOHSA-DAG: s_load_dwordx2 s[[[X:[0-9]+]]:[[Y:[0-9+]]]], s[0:1], 0x6 -; VI-NOHSA-DAG: s_load_dwordx2 s[[[X:[0-9]+]]:[[Y:[0-9+]]]], s[0:1], 0x18 -; GCN: s_mul_i32 [[VAL:s[0-9]+]], s[[X]], s[[Y]] -; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN: buffer_store_dword [[VVAL]] define amdgpu_kernel void @local_size_xy(ptr addrspace(1) %out) { +; SI-LABEL: local_size_xy: +; SI: ; %bb.0: ; %entry +; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x6 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mul_i32 s4, s4, s5 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: local_size_xy: +; VI: ; %bb.0: ; %entry +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x18 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_mul_i32 s2, s2, s3 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; R600-LABEL: local_size_xy: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: MULLO_INT * T1.X, KC0[1].Z, KC0[1].W, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) entry: %x = call i32 @llvm.r600.read.local.size.x() #0 %y = call i32 @llvm.r600.read.local.size.y() #0 @@ -66,17 +163,43 @@ entry: ret void } -; FUNC-LABEL: {{^}}local_size_xz: - -; SI-NOHSA-DAG: s_load_dword [[X:s[0-9]+]], s[0:1], 0x6 -; SI-NOHSA-DAG: s_load_dword [[Z:s[0-9]+]], s[0:1], 0x8 -; VI-NOHSA-DAG: s_load_dword [[X:s[0-9]+]], s[0:1], 0x18 -; VI-NOHSA-DAG: s_load_dword [[Z:s[0-9]+]], s[0:1], 0x20 -; HSA-DAG: s_and_b32 [[X:s[0-9]+]], [[XY]], 0xffff -; GCN: s_mul_i32 [[VAL:s[0-9]+]], [[X]], [[Z]] -; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN: buffer_store_dword [[VVAL]] define amdgpu_kernel void @local_size_xz(ptr addrspace(1) %out) { +; SI-LABEL: local_size_xz: +; SI: ; %bb.0: ; %entry +; SI-NEXT: s_load_dword s2, s[0:1], 0x6 +; SI-NEXT: s_load_dword s4, s[0:1], 0x8 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mul_i32 s4, s2, s4 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: local_size_xz: +; VI: ; %bb.0: ; %entry +; VI-NEXT: s_load_dword s2, s[0:1], 0x18 +; VI-NEXT: s_load_dword s3, s[0:1], 0x20 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_mul_i32 s2, s2, s3 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; R600-LABEL: local_size_xz: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: MULLO_INT * T1.X, KC0[1].Z, KC0[2].X, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) entry: %x = call i32 @llvm.r600.read.local.size.x() #0 %z = call i32 @llvm.r600.read.local.size.z() #0 @@ -85,16 +208,41 @@ entry: ret void } -; FUNC-LABEL: {{^}}local_size_yz: -; HSA: enable_sgpr_private_segment_buffer = 1 -; HSA: enable_sgpr_dispatch_ptr = 1 - -; SI-NOHSA-DAG: s_load_dwordx4 s[[[#LOAD:]]:{{[0-9]+}}], s[0:1], 0x7 -; VI-NOHSA-DAG: s_load_dwordx4 s[[[#LOAD:]]:{{[0-9]+}}], s[0:1], 0x1c -; GCN: s_mul_i32 [[VAL:s[0-9]+]], s[[#LOAD + 0]], s[[#LOAD + 1]] -; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN: buffer_store_dword [[VVAL]] define amdgpu_kernel void @local_size_yz(ptr addrspace(1) %out) { +; SI-LABEL: local_size_yz: +; SI: ; %bb.0: ; %entry +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x7 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mul_i32 s0, s0, s1 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_mov_b32 s4, s2 +; SI-NEXT: s_mov_b32 s5, s3 +; SI-NEXT: v_mov_b32_e32 v0, s0 +; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: local_size_yz: +; VI: ; %bb.0: ; %entry +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x1c +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_mul_i32 s0, s0, s1 +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; R600-LABEL: local_size_yz: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: MULLO_INT * T1.X, KC0[1].W, KC0[2].X, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) entry: %y = call i32 @llvm.r600.read.local.size.y() #0 %z = call i32 @llvm.r600.read.local.size.z() #0 @@ -103,19 +251,46 @@ entry: ret void } -; FUNC-LABEL: {{^}}local_size_xyz: -; HSA: enable_sgpr_private_segment_buffer = 1 -; HSA: enable_sgpr_dispatch_ptr = 1 - -; SI-NOHSA-DAG: s_load_dwordx2 s[[[X:[0-9]+]]:[[Y:[0-9]+]]], s[0:1], 0x6 -; SI-NOHSA-DAG: s_load_dword s[[Z:[0-9]+]], s[0:1], 0x8 -; VI-NOHSA-DAG: s_load_dwordx2 s[[[X:[0-9]+]]:[[Y:[0-9]+]]], s[0:1], 0x18 -; VI-NOHSA-DAG: s_load_dword s[[Z:[0-9]+]], s[0:1], 0x20 -; GCN: s_mul_i32 [[M:s[0-9]+]], s[[X]], s[[Y]] -; GCN: s_add_i32 [[VAL:s[0-9]+]], [[M]], s[[Z]] -; GCN-DAG: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN: buffer_store_dword [[VVAL]] define amdgpu_kernel void @local_size_xyz(ptr addrspace(1) %out) { +; SI-LABEL: local_size_xyz: +; SI: ; %bb.0: ; %entry +; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x6 +; SI-NEXT: s_load_dword s2, s[0:1], 0x8 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mul_i32 s4, s4, s5 +; SI-NEXT: s_add_i32 s4, s4, s2 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: local_size_xyz: +; VI: ; %bb.0: ; %entry +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x18 +; VI-NEXT: s_load_dword s4, s[0:1], 0x20 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_mul_i32 s2, s2, s3 +; VI-NEXT: s_add_i32 s2, s2, s4 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; R600-LABEL: local_size_xyz: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MULLO_INT * T0.X, KC0[1].Z, KC0[1].W, +; R600-NEXT: ADD_INT T0.X, PS, KC0[2].X, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) entry: %x = call i32 @llvm.r600.read.local.size.x() #0 %y = call i32 @llvm.r600.read.local.size.y() #0 @@ -126,13 +301,39 @@ entry: ret void } -; FUNC-LABEL: {{^}}local_size_x_known_bits: -; SI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x6 -; VI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x18 -; GCN-NOT: 0xffff -; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN-NEXT: buffer_store_dword [[VVAL]] define amdgpu_kernel void @local_size_x_known_bits(ptr addrspace(1) %out) { +; SI-LABEL: local_size_x_known_bits: +; SI: ; %bb.0: ; %entry +; SI-NEXT: s_load_dword s4, s[0:1], 0x6 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: local_size_x_known_bits: +; VI: ; %bb.0: ; %entry +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; VI-NEXT: s_load_dword s0, s[0:1], 0x18 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; R600-LABEL: local_size_x_known_bits: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: AND_INT * T1.X, KC0[1].Z, literal.y, +; R600-NEXT: 2(2.802597e-45), 65535(9.183409e-41) entry: %size = call i32 @llvm.r600.read.local.size.x() #0 %shl = shl i32 %size, 16 @@ -141,13 +342,39 @@ entry: ret void } -; FUNC-LABEL: {{^}}local_size_y_known_bits: -; SI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x7 -; VI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x1c -; GCN-NOT: 0xffff -; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN-NEXT: buffer_store_dword [[VVAL]] define amdgpu_kernel void @local_size_y_known_bits(ptr addrspace(1) %out) { +; SI-LABEL: local_size_y_known_bits: +; SI: ; %bb.0: ; %entry +; SI-NEXT: s_load_dword s4, s[0:1], 0x7 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: local_size_y_known_bits: +; VI: ; %bb.0: ; %entry +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; VI-NEXT: s_load_dword s0, s[0:1], 0x1c +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; R600-LABEL: local_size_y_known_bits: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: AND_INT * T1.X, KC0[1].W, literal.y, +; R600-NEXT: 2(2.802597e-45), 65535(9.183409e-41) entry: %size = call i32 @llvm.r600.read.local.size.y() #0 %shl = shl i32 %size, 16 @@ -156,13 +383,39 @@ entry: ret void } -; FUNC-LABEL: {{^}}local_size_z_known_bits: -; SI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x8 -; VI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x20 -; GCN-NOT: 0xffff -; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN-NEXT: buffer_store_dword [[VVAL]] define amdgpu_kernel void @local_size_z_known_bits(ptr addrspace(1) %out) { +; SI-LABEL: local_size_z_known_bits: +; SI: ; %bb.0: ; %entry +; SI-NEXT: s_load_dword s4, s[0:1], 0x8 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: local_size_z_known_bits: +; VI: ; %bb.0: ; %entry +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; VI-NEXT: s_load_dword s0, s[0:1], 0x20 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm +; +; R600-LABEL: local_size_z_known_bits: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: AND_INT * T1.X, KC0[2].X, literal.y, +; R600-NEXT: 2(2.802597e-45), 65535(9.183409e-41) entry: %size = call i32 @llvm.r600.read.local.size.z() #0 %shl = shl i32 %size, 16 @@ -176,3 +429,6 @@ declare i32 @llvm.r600.read.local.size.y() #0 declare i32 @llvm.r600.read.local.size.z() #0 attributes #0 = { nounwind readnone } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; FUNC: {{.*}} +; GCN: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll index 4d6adc7cc941..a3f7906a05f6 100644 --- a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll +++ b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll @@ -1,17 +1,21 @@ -; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -enable-var-scope -check-prefix=GCN %s -; GCN-LABEL: {{^}}fma_vector_vector_scalar_lo: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[SCALAR0]] op_sel_hi:[1,1,0]{{$}} define amdgpu_kernel void @fma_vector_vector_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 { +; GCN-LABEL: fma_vector_vector_scalar_lo: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: v_mov_b32_e32 v1, s3 +; GCN-NEXT: ds_read_b32 v2, v0 +; GCN-NEXT: ds_read_b32 v0, v0 offset:4 +; GCN-NEXT: ds_read_u16 v1, v1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 @@ -28,18 +32,21 @@ bb: } ; Apply fneg to broadcasted vector -; GCN-LABEL: {{^}}fma_vector_vector_neg_broadcast_scalar_lo: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[SCALAR0]] op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1]{{$}} define amdgpu_kernel void @fma_vector_vector_neg_broadcast_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 { +; GCN-LABEL: fma_vector_vector_neg_broadcast_scalar_lo: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: v_mov_b32_e32 v1, s3 +; GCN-NEXT: ds_read_b32 v2, v0 +; GCN-NEXT: ds_read_b32 v0, v0 offset:4 +; GCN-NEXT: ds_read_u16 v1, v1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 @@ -57,18 +64,21 @@ bb: } ; Apply fneg before broadcast -; GCN-LABEL: {{^}}fma_vector_vector_neg_scalar_lo: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[SCALAR0]] op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1]{{$}} define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 { +; GCN-LABEL: fma_vector_vector_neg_scalar_lo: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: v_mov_b32_e32 v1, s3 +; GCN-NEXT: ds_read_b32 v2, v0 +; GCN-NEXT: ds_read_b32 v0, v0 offset:4 +; GCN-NEXT: ds_read_u16 v1, v1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 @@ -86,18 +96,21 @@ bb: } ; Apply fneg before and after broadcast, and should cancel out. -; GCN-LABEL: {{^}}fma_vector_vector_neg_broadcast_neg_scalar_lo: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[SCALAR0]] op_sel_hi:[1,1,0]{{$}} define amdgpu_kernel void @fma_vector_vector_neg_broadcast_neg_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 { +; GCN-LABEL: fma_vector_vector_neg_broadcast_neg_scalar_lo: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: v_mov_b32_e32 v1, s3 +; GCN-NEXT: ds_read_b32 v2, v0 +; GCN-NEXT: ds_read_b32 v0, v0 offset:4 +; GCN-NEXT: ds_read_u16 v1, v1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 @@ -116,18 +129,21 @@ bb: } ; Add scalar, but negate low component -; GCN-LABEL: {{^}}fma_vector_vector_scalar_neg_lo: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[SCALAR0]] op_sel_hi:[1,1,0] neg_lo:[0,0,1]{{$}} define amdgpu_kernel void @fma_vector_vector_scalar_neg_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 { +; GCN-LABEL: fma_vector_vector_scalar_neg_lo: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: v_mov_b32_e32 v1, s3 +; GCN-NEXT: ds_read_b32 v2, v0 +; GCN-NEXT: ds_read_b32 v0, v0 offset:4 +; GCN-NEXT: ds_read_u16 v1, v1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] neg_lo:[0,0,1] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 @@ -144,18 +160,21 @@ bb: } ; Add scalar, but negate high component -; GCN-LABEL: {{^}}fma_vector_vector_scalar_neg_hi: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[SCALAR0]] op_sel_hi:[1,1,0] neg_hi:[0,0,1]{{$}} define amdgpu_kernel void @fma_vector_vector_scalar_neg_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 { +; GCN-LABEL: fma_vector_vector_scalar_neg_hi: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: v_mov_b32_e32 v1, s3 +; GCN-NEXT: ds_read_b32 v2, v0 +; GCN-NEXT: ds_read_b32 v0, v0 offset:4 +; GCN-NEXT: ds_read_u16 v1, v1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] neg_hi:[0,0,1] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 @@ -172,17 +191,20 @@ bb: } ; Apply fneg before broadcast with bitcast -; GCN-LABEL: {{^}}add_vector_neg_bitcast_scalar_lo: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or - -; GCN: v_pk_add_u16 v{{[0-9]+}}, [[VEC0]], [[SCALAR0]] op_sel_hi:[1,0] neg_lo:[0,1] neg_hi:[0,1]{{$}} define amdgpu_kernel void @add_vector_neg_bitcast_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 { +; GCN-LABEL: add_vector_neg_bitcast_scalar_lo: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; GCN-NEXT: v_mov_b32_e32 v2, 0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: v_mov_b32_e32 v1, s3 +; GCN-NEXT: ds_read_b32 v0, v0 +; GCN-NEXT: ds_read_u16 v1, v1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_u16 v0, v0, v1 op_sel_hi:[1,0] neg_lo:[0,1] neg_hi:[0,1] +; GCN-NEXT: global_store_dword v2, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %vec0 = load volatile <2 x i16>, ptr addrspace(3) %lds, align 4 %scalar0 = load volatile half, ptr addrspace(3) %arg2, align 2 @@ -197,19 +219,26 @@ bb: ret void } -; GCN-LABEL: {{^}}fma_vector_vector_scalar_lo_neg_scalar_hi: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]] -; GCN: ds_read_u16 [[SCALAR1:v[0-9]+]] - -; FIXME: Remove and -; GCN-DAG: v_and_b32_e32 [[SCALAR0]], 0xffff, [[SCALAR0]] -; GCN-DAG: v_xor_b32_e32 [[SCALAR1]], 0x8000, [[SCALAR1]] -; GCN: v_lshl_or_b32 [[PACKED:v[0-9]+]], [[SCALAR1]], 16, [[SCALAR0]] - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[PACKED]]{{$}} define amdgpu_kernel void @fma_vector_vector_scalar_lo_neg_scalar_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 { +; GCN-LABEL: fma_vector_vector_scalar_lo_neg_scalar_hi: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; GCN-NEXT: v_mov_b32_e32 v4, 0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: v_mov_b32_e32 v1, s3 +; GCN-NEXT: ds_read_b32 v2, v0 +; GCN-NEXT: ds_read_b32 v0, v0 offset:4 +; GCN-NEXT: ds_read_u16 v3, v1 +; GCN-NEXT: ds_read_u16 v1, v1 offset:4 +; GCN-NEXT: s_waitcnt lgkmcnt(1) +; GCN-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_xor_b32_e32 v1, 0x8000, v1 +; GCN-NEXT: v_lshl_or_b32 v1, v1, 16, v3 +; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1 +; GCN-NEXT: global_store_dword v4, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %arg2.gep = getelementptr inbounds half, ptr addrspace(3) %arg2, i32 2 @@ -229,15 +258,23 @@ bb: } ; FIXME: Can we avoid waitcnt between the two halves? -; GCN-LABEL: {{^}}fma_vector_vector_neg_scalar_lo_scalar_hi: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_u16 [[PACKED:v[0-9]+]] -; GCN: s_waitcnt -; GCN: ds_read_u16_d16_hi [[PACKED]] - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[PACKED]] neg_lo:[0,0,1] neg_hi:[0,0,1]{{$}} define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo_scalar_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 { +; GCN-LABEL: fma_vector_vector_neg_scalar_lo_scalar_hi: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: v_mov_b32_e32 v1, s3 +; GCN-NEXT: ds_read_b32 v2, v0 +; GCN-NEXT: ds_read_b32 v0, v0 offset:4 +; GCN-NEXT: ds_read_u16 v3, v1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: ds_read_u16_d16_hi v3, v1 offset:4 +; GCN-NEXT: v_mov_b32_e32 v1, 0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v3 neg_lo:[0,0,1] neg_hi:[0,0,1] +; GCN-NEXT: global_store_dword v1, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %arg2.gep = getelementptr inbounds half, ptr addrspace(3) %arg2, i32 2 @@ -257,18 +294,21 @@ bb: ret void } -; GCN-LABEL: {{^}}fma_vector_vector_neg_vector_hi: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_b32 [[VEC2:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] op_sel:[0,0,1] neg_lo:[0,0,1] neg_hi:[0,0,1]{{$}} define amdgpu_kernel void @fma_vector_vector_neg_vector_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: fma_vector_vector_neg_vector_hi: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: ds_read_b32 v2, v0 offset:4 +; GCN-NEXT: ds_read_b32 v0, v0 offset:8 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] neg_lo:[0,0,1] neg_hi:[0,0,1] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2 @@ -285,18 +325,21 @@ bb: ret void } -; GCN-LABEL: {{^}}fma_vector_vector_vector_neg_hi: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_b32 [[VEC2:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] neg_hi:[0,0,1]{{$}} define amdgpu_kernel void @fma_vector_vector_vector_neg_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: fma_vector_vector_vector_neg_hi: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: ds_read_b32 v2, v0 offset:4 +; GCN-NEXT: ds_read_b32 v0, v0 offset:8 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v1, v2, v0 neg_hi:[0,0,1] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2 @@ -314,17 +357,20 @@ bb: ret void } -; GCN-LABEL: {{^}}add_vector_scalar_hi: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or - -; GCN: v_pk_add_u16 v{{[0-9]+}}, [[VEC0]], [[VEC1]] op_sel:[0,1]{{$}} define amdgpu_kernel void @add_vector_scalar_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: add_vector_scalar_hi: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v2, 0 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: ds_read_b32 v0, v0 offset:4 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_u16 v0, v1, v0 op_sel:[0,1] +; GCN-NEXT: global_store_dword v2, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x i16>, ptr addrspace(3) %lds, i32 1 @@ -338,18 +384,21 @@ bb: ret void } -; GCN-LABEL: {{^}}fma_vector_vector_scalar_hi: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_b32 [[VEC2:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] op_sel:[0,0,1]{{$}} define amdgpu_kernel void @fma_vector_vector_scalar_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: fma_vector_vector_scalar_hi: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: ds_read_b32 v2, v0 offset:4 +; GCN-NEXT: ds_read_b32 v0, v0 offset:8 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2 @@ -366,18 +415,21 @@ bb: ret void } -; GCN-LABEL: {{^}}fma_vector_vector_neg_vector_lo_neg_hi: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_b32 [[VEC2:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]]{{$}} define amdgpu_kernel void @fma_vector_vector_neg_vector_lo_neg_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: fma_vector_vector_neg_vector_lo_neg_hi: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: ds_read_b32 v2, v0 offset:4 +; GCN-NEXT: ds_read_b32 v0, v0 offset:8 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v1, v2, v0 +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2 @@ -396,18 +448,21 @@ bb: ret void } -; GCN-LABEL: {{^}}fma_vector_vector_swap_vector: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_b32 [[VEC2:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] op_sel:[0,0,1] op_sel_hi:[1,1,0]{{$}} define amdgpu_kernel void @fma_vector_vector_swap_vector(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: fma_vector_vector_swap_vector: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: ds_read_b32 v2, v0 offset:4 +; GCN-NEXT: ds_read_b32 v0, v0 offset:8 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] op_sel_hi:[1,1,0] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2 @@ -423,19 +478,21 @@ bb: ret void } -; GCN-LABEL: {{^}}fma_vector_vector_swap_neg_vector: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_b32 [[VEC2:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or -; GCN-NOT: xor - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] op_sel:[0,0,1] op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1]{{$}} define amdgpu_kernel void @fma_vector_vector_swap_neg_vector(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: fma_vector_vector_swap_neg_vector: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: ds_read_b32 v2, v0 offset:4 +; GCN-NEXT: ds_read_b32 v0, v0 offset:8 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2 @@ -452,19 +509,21 @@ bb: ret void } -; GCN-LABEL: {{^}}fma_vector_vector_blend_vector_neg_vector_0: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_b32 [[VEC2:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or -; GCN-NOT: xor - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] op_sel:[0,0,1] op_sel_hi:[1,1,0] neg_lo:[0,0,1]{{$}} define amdgpu_kernel void @fma_vector_vector_blend_vector_neg_vector_0(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: fma_vector_vector_blend_vector_neg_vector_0: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: ds_read_b32 v2, v0 offset:4 +; GCN-NEXT: ds_read_b32 v0, v0 offset:8 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] op_sel_hi:[1,1,0] neg_lo:[0,0,1] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2 @@ -480,19 +539,21 @@ bb: ret void } -; GCN-LABEL: {{^}}fma_vector_vector_blend_vector_neg_vector_1: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_b32 [[VEC2:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or -; GCN-NOT: xor - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] neg_lo:[0,0,1]{{$}} define amdgpu_kernel void @fma_vector_vector_blend_vector_neg_vector_1(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: fma_vector_vector_blend_vector_neg_vector_1: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: ds_read_b32 v2, v0 offset:4 +; GCN-NEXT: ds_read_b32 v0, v0 offset:8 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v1, v2, v0 neg_lo:[0,0,1] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2 @@ -508,19 +569,21 @@ bb: ret void } -; GCN-LABEL: {{^}}fma_vector_vector_blend_vector_neg_vector_2: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_b32 [[VEC2:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or -; GCN-NOT: xor - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] neg_hi:[0,0,1]{{$}} define amdgpu_kernel void @fma_vector_vector_blend_vector_neg_vector_2(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: fma_vector_vector_blend_vector_neg_vector_2: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: ds_read_b32 v2, v0 offset:4 +; GCN-NEXT: ds_read_b32 v0, v0 offset:8 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v1, v2, v0 neg_hi:[0,0,1] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2 @@ -536,19 +599,21 @@ bb: ret void } -; GCN-LABEL: {{^}}fma_vector_vector_blend_vector_neg_vector_3: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_b32 [[VEC2:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: or -; GCN-NOT: xor - -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[VEC2]] op_sel:[0,0,1] neg_lo:[0,0,1]{{$}} define amdgpu_kernel void @fma_vector_vector_blend_vector_neg_vector_3(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: fma_vector_vector_blend_vector_neg_vector_3: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: ds_read_b32 v2, v0 offset:4 +; GCN-NEXT: ds_read_b32 v0, v0 offset:8 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] neg_lo:[0,0,1] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2 @@ -564,9 +629,22 @@ bb: ret void } -; GCN-LABEL: {{^}}bitcast_fneg_f32: -; GCN: v_pk_add_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+$}} define amdgpu_kernel void @bitcast_fneg_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: bitcast_fneg_f32: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v2, 0 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v0, v0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_xor_b32_e32 v1, 0x80000000, v1 +; GCN-NEXT: v_pk_add_f16 v0, v0, v1 +; GCN-NEXT: global_store_dword v2, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %vec0 = load volatile <2 x half>, ptr addrspace(3) %lds, align 4 %f32 = load volatile float, ptr addrspace(3) undef, align 4 @@ -578,9 +656,22 @@ bb: ret void } -; GCN-LABEL: {{^}}shuffle_bitcast_fneg_f32: -; GCN: v_pk_add_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} op_sel:[0,1] op_sel_hi:[1,0]{{$}} define amdgpu_kernel void @shuffle_bitcast_fneg_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: shuffle_bitcast_fneg_f32: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v2, 0 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v0, v0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_xor_b32_e32 v1, 0x80000000, v1 +; GCN-NEXT: v_pk_add_f16 v0, v0, v1 op_sel:[0,1] op_sel_hi:[1,0] +; GCN-NEXT: global_store_dword v2, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %vec0 = load volatile <2 x half>, ptr addrspace(3) %lds, align 4 @@ -593,10 +684,24 @@ bb: ret void } -; GCN-LABEL: {{^}}extract_from_i64: -; GCN: v_lshl_or_b32 -; GCN: v_pk_add_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+$}} define amdgpu_kernel void @extract_from_i64(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: extract_from_i64: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: v_mov_b32_e32 v3, 0xffff +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v2, v0 +; GCN-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v1, 0 +; GCN-NEXT: v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 +; GCN-NEXT: v_lshl_or_b32 v0, v0, 16, v3 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_u16 v0, v2, v0 +; GCN-NEXT: global_store_dword v1, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %vec0 = load volatile <2 x i16>, ptr addrspace(3) %lds, align 4 %i64 = load volatile i64, ptr addrspace(1) undef @@ -612,21 +717,24 @@ bb: ret void } - -; Bitcast is final obstacle to identifying same source register -; GCN-LABEL: {{^}}bitcast_lo_elt_op_sel: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_b32 [[VEC2:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: _or - -; GCN: v_pk_add_f16 [[FADD:v[0-9]+]] -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[FADD]] op_sel:[0,0,1] op_sel_hi:[1,1,0]{{$}} define amdgpu_kernel void @bitcast_lo_elt_op_sel(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: bitcast_lo_elt_op_sel: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: ds_read_b32 v2, v0 offset:4 +; GCN-NEXT: ds_read_b32 v0, v0 offset:8 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: global_load_ushort v3, v[0:1], off glc +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: v_pk_add_f16 v0, v0, 2.0 op_sel_hi:[1,0] +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] op_sel_hi:[1,1,0] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2 @@ -647,21 +755,29 @@ bb: ret void } - -; Bitcast is final obstacle to identifying same source register -; GCN-LABEL: {{^}}mix_elt_types_op_sel: -; GCN: ds_read_b32 [[VEC0:v[0-9]+]] -; GCN: ds_read_b32 [[VEC1:v[0-9]+]] -; GCN: ds_read_b32 [[VEC2:v[0-9]+]] - -; GCN-NOT: pack -; GCN-NOT: and -; GCN-NOT: shl -; GCN-NOT: _or - -; GCN: v_pk_add_f16 [[FADD:v[0-9]+]] -; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[FADD]] op_sel:[0,0,1] op_sel_hi:[1,1,0]{{$}} define amdgpu_kernel void @mix_elt_types_op_sel(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 { +; GCN-LABEL: mix_elt_types_op_sel: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: ds_read_b32 v2, v0 offset:4 +; GCN-NEXT: ds_read_b32 v0, v0 offset:8 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GCN-NEXT: ; kill: killed $vgpr0_vgpr1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: global_load_ushort v3, v[0:1], off glc +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: global_load_ushort v3, v[0:1], off glc +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: ; kill: killed $vgpr0_vgpr1 +; GCN-NEXT: v_pk_add_f16 v0, v0, 2.0 op_sel_hi:[1,0] +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] op_sel_hi:[1,1,0] +; GCN-NEXT: global_store_dword v3, v0, s[0:1] +; GCN-NEXT: s_endpgm bb: %lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1 %lds.gep2 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 2 diff --git a/llvm/test/CodeGen/AMDGPU/xor-r600.ll b/llvm/test/CodeGen/AMDGPU/xor-r600.ll new file mode 100644 index 000000000000..3fb11f4484bd --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/xor-r600.ll @@ -0,0 +1,478 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=r600 -mcpu=redwood < %s | FileCheck -enable-var-scope -check-prefixes=R600 %s + +define amdgpu_kernel void @xor_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; R600-LABEL: xor_v2i32: +; R600: ; %bb.0: +; R600-NEXT: ALU 1, @10, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 1 @6 +; R600-NEXT: ALU 3, @12, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: Fetch clause starting at 6: +; R600-NEXT: VTX_READ_64 T1.XY, T1.X, 0, #1 +; R600-NEXT: VTX_READ_64 T0.XY, T0.X, 0, #1 +; R600-NEXT: ALU clause starting at 10: +; R600-NEXT: MOV T0.X, KC0[2].Z, +; R600-NEXT: MOV * T1.X, KC0[2].W, +; R600-NEXT: ALU clause starting at 12: +; R600-NEXT: XOR_INT * T0.Y, T0.Y, T1.Y, +; R600-NEXT: XOR_INT T0.X, T0.X, T1.X, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %a = load <2 x i32>, ptr addrspace(1) %in0 + %b = load <2 x i32>, ptr addrspace(1) %in1 + %result = xor <2 x i32> %a, %b + store <2 x i32> %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @xor_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; R600-LABEL: xor_v4i32: +; R600: ; %bb.0: +; R600-NEXT: ALU 1, @10, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 1 @6 +; R600-NEXT: ALU 5, @12, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: Fetch clause starting at 6: +; R600-NEXT: VTX_READ_128 T1.XYZW, T1.X, 0, #1 +; R600-NEXT: VTX_READ_128 T0.XYZW, T0.X, 0, #1 +; R600-NEXT: ALU clause starting at 10: +; R600-NEXT: MOV T0.X, KC0[2].Z, +; R600-NEXT: MOV * T1.X, KC0[2].W, +; R600-NEXT: ALU clause starting at 12: +; R600-NEXT: XOR_INT * T0.W, T0.W, T1.W, +; R600-NEXT: XOR_INT * T0.Z, T0.Z, T1.Z, +; R600-NEXT: XOR_INT * T0.Y, T0.Y, T1.Y, +; R600-NEXT: XOR_INT T0.X, T0.X, T1.X, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %a = load <4 x i32>, ptr addrspace(1) %in0 + %b = load <4 x i32>, ptr addrspace(1) %in1 + %result = xor <4 x i32> %a, %b + store <4 x i32> %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @xor_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; R600-LABEL: xor_i1: +; R600: ; %bb.0: +; R600-NEXT: ALU 0, @12, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 0 @8 +; R600-NEXT: ALU 0, @13, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 0 @10 +; R600-NEXT: ALU 5, @14, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: Fetch clause starting at 8: +; R600-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1 +; R600-NEXT: Fetch clause starting at 10: +; R600-NEXT: VTX_READ_32 T1.X, T1.X, 0, #1 +; R600-NEXT: ALU clause starting at 12: +; R600-NEXT: MOV * T0.X, KC0[2].W, +; R600-NEXT: ALU clause starting at 13: +; R600-NEXT: MOV * T1.X, KC0[2].Z, +; R600-NEXT: ALU clause starting at 14: +; R600-NEXT: SETGE_DX10 T0.W, T0.X, 1.0, +; R600-NEXT: SETGE_DX10 * T1.W, T1.X, 0.0, +; R600-NEXT: XOR_INT * T0.W, PS, PV.W, +; R600-NEXT: CNDE_INT T0.X, PV.W, T0.X, T1.X, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %a = load float, ptr addrspace(1) %in0 + %b = load float, ptr addrspace(1) %in1 + %acmp = fcmp oge float %a, 0.000000e+00 + %bcmp = fcmp oge float %b, 1.000000e+00 + %xor = xor i1 %acmp, %bcmp + %result = select i1 %xor, float %a, float %b + store float %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @v_xor_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; R600-LABEL: v_xor_i1: +; R600: ; %bb.0: +; R600-NEXT: ALU 0, @12, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 0 @8 +; R600-NEXT: ALU 0, @13, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 0 @10 +; R600-NEXT: ALU 12, @14, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT MSKOR T0.XW, T1.X +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: Fetch clause starting at 8: +; R600-NEXT: VTX_READ_8 T0.X, T0.X, 0, #1 +; R600-NEXT: Fetch clause starting at 10: +; R600-NEXT: VTX_READ_8 T1.X, T1.X, 0, #1 +; R600-NEXT: ALU clause starting at 12: +; R600-NEXT: MOV * T0.X, KC0[2].Z, +; R600-NEXT: ALU clause starting at 13: +; R600-NEXT: MOV * T1.X, KC0[2].W, +; R600-NEXT: ALU clause starting at 14: +; R600-NEXT: AND_INT T0.W, KC0[2].Y, literal.x, +; R600-NEXT: XOR_INT * T1.W, T0.X, T1.X, +; R600-NEXT: 3(4.203895e-45), 0(0.000000e+00) +; R600-NEXT: AND_INT T1.W, PS, 1, +; R600-NEXT: LSHL * T0.W, PV.W, literal.x, +; R600-NEXT: 3(4.203895e-45), 0(0.000000e+00) +; R600-NEXT: LSHL T0.X, PV.W, PS, +; R600-NEXT: LSHL * T0.W, literal.x, PS, +; R600-NEXT: 255(3.573311e-43), 0(0.000000e+00) +; R600-NEXT: MOV T0.Y, 0.0, +; R600-NEXT: MOV * T0.Z, 0.0, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %a = load volatile i1, ptr addrspace(1) %in0 + %b = load volatile i1, ptr addrspace(1) %in1 + %xor = xor i1 %a, %b + store i1 %xor, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @vector_xor_i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; R600-LABEL: vector_xor_i32: +; R600: ; %bb.0: +; R600-NEXT: ALU 1, @10, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 1 @6 +; R600-NEXT: ALU 2, @12, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: Fetch clause starting at 6: +; R600-NEXT: VTX_READ_32 T1.X, T1.X, 0, #1 +; R600-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1 +; R600-NEXT: ALU clause starting at 10: +; R600-NEXT: MOV T0.X, KC0[2].Z, +; R600-NEXT: MOV * T1.X, KC0[2].W, +; R600-NEXT: ALU clause starting at 12: +; R600-NEXT: XOR_INT T0.X, T0.X, T1.X, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %a = load i32, ptr addrspace(1) %in0 + %b = load i32, ptr addrspace(1) %in1 + %result = xor i32 %a, %b + store i32 %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @scalar_xor_i32(ptr addrspace(1) %out, i32 %a, i32 %b) { +; R600-LABEL: scalar_xor_i32: +; R600: ; %bb.0: +; R600-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; R600-NEXT: NOT_INT * T1.X, KC0[2].Z, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %result = xor i32 %a, -1 + store i32 %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @vector_not_i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; R600-LABEL: vector_not_i32: +; R600: ; %bb.0: +; R600-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 0 @6 +; R600-NEXT: ALU 2, @9, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: Fetch clause starting at 6: +; R600-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1 +; R600-NEXT: ALU clause starting at 8: +; R600-NEXT: MOV * T0.X, KC0[2].Z, +; R600-NEXT: ALU clause starting at 9: +; R600-NEXT: NOT_INT T0.X, T0.X, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %a = load i32, ptr addrspace(1) %in0 + %b = load i32, ptr addrspace(1) %in1 + %result = xor i32 %a, -1 + store i32 %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @vector_xor_i64(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; R600-LABEL: vector_xor_i64: +; R600: ; %bb.0: +; R600-NEXT: ALU 1, @10, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 1 @6 +; R600-NEXT: ALU 3, @12, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: Fetch clause starting at 6: +; R600-NEXT: VTX_READ_64 T1.XY, T1.X, 0, #1 +; R600-NEXT: VTX_READ_64 T0.XY, T0.X, 0, #1 +; R600-NEXT: ALU clause starting at 10: +; R600-NEXT: MOV T0.X, KC0[2].Z, +; R600-NEXT: MOV * T1.X, KC0[2].W, +; R600-NEXT: ALU clause starting at 12: +; R600-NEXT: XOR_INT * T0.Y, T0.Y, T1.Y, +; R600-NEXT: XOR_INT T0.X, T0.X, T1.X, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %a = load i64, ptr addrspace(1) %in0 + %b = load i64, ptr addrspace(1) %in1 + %result = xor i64 %a, %b + store i64 %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @scalar_xor_i64(ptr addrspace(1) %out, i64 %a, i64 %b) { +; R600-LABEL: scalar_xor_i64: +; R600: ; %bb.0: +; R600-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: XOR_INT * T0.Y, KC0[3].X, KC0[3].Z, +; R600-NEXT: XOR_INT * T0.X, KC0[2].W, KC0[3].Y, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %result = xor i64 %a, %b + store i64 %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @scalar_not_i64(ptr addrspace(1) %out, i64 %a) { +; R600-LABEL: scalar_not_i64: +; R600: ; %bb.0: +; R600-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: NOT_INT * T0.Y, KC0[3].X, +; R600-NEXT: NOT_INT T0.X, KC0[2].W, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %result = xor i64 %a, -1 + store i64 %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @vector_not_i64(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; R600-LABEL: vector_not_i64: +; R600: ; %bb.0: +; R600-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 0 @6 +; R600-NEXT: ALU 3, @9, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: Fetch clause starting at 6: +; R600-NEXT: VTX_READ_64 T0.XY, T0.X, 0, #1 +; R600-NEXT: ALU clause starting at 8: +; R600-NEXT: MOV * T0.X, KC0[2].Z, +; R600-NEXT: ALU clause starting at 9: +; R600-NEXT: NOT_INT * T0.Y, T0.Y, +; R600-NEXT: NOT_INT T0.X, T0.X, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) + %a = load i64, ptr addrspace(1) %in0 + %b = load i64, ptr addrspace(1) %in1 + %result = xor i64 %a, -1 + store i64 %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @xor_cf(ptr addrspace(1) %out, ptr addrspace(1) %in, i64 %a, i64 %b) { +; R600-LABEL: xor_cf: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU_PUSH_BEFORE 4, @14, KC0[CB0:0-32], KC1[] +; R600-NEXT: JUMP @5 POP:1 +; R600-NEXT: ALU 0, @19, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 0 @12 +; R600-NEXT: ALU_POP_AFTER 1, @20, KC0[], KC1[] +; R600-NEXT: ALU_PUSH_BEFORE 2, @22, KC0[CB0:0-32], KC1[] +; R600-NEXT: JUMP @8 POP:1 +; R600-NEXT: ALU_POP_AFTER 5, @25, KC0[CB0:0-32], KC1[] +; R600-NEXT: ALU 1, @31, KC0[], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: Fetch clause starting at 12: +; R600-NEXT: VTX_READ_64 T0.XY, T0.X, 0, #1 +; R600-NEXT: ALU clause starting at 14: +; R600-NEXT: OR_INT T0.W, KC0[2].W, KC0[3].X, +; R600-NEXT: MOV * T1.W, literal.x, +; R600-NEXT: 1(1.401298e-45), 0(0.000000e+00) +; R600-NEXT: SETNE_INT * T0.W, PV.W, 0.0, +; R600-NEXT: PRED_SETNE_INT * ExecMask,PredicateBit (MASKED), PV.W, 0.0, +; R600-NEXT: ALU clause starting at 19: +; R600-NEXT: MOV * T0.X, KC0[2].Z, +; R600-NEXT: ALU clause starting at 20: +; R600-NEXT: MOV * T1.W, literal.x, +; R600-NEXT: 0(0.000000e+00), 0(0.000000e+00) +; R600-NEXT: ALU clause starting at 22: +; R600-NEXT: MOV T0.W, KC0[2].Y, +; R600-NEXT: SETE_INT * T1.W, T1.W, 0.0, +; R600-NEXT: PRED_SETE_INT * ExecMask,PredicateBit (MASKED), PS, 0.0, +; R600-NEXT: ALU clause starting at 25: +; R600-NEXT: MOV T1.W, KC0[2].W, +; R600-NEXT: MOV * T2.W, KC0[3].Y, +; R600-NEXT: XOR_INT T0.X, PV.W, PS, +; R600-NEXT: MOV T1.W, KC0[3].X, +; R600-NEXT: MOV * T2.W, KC0[3].Z, +; R600-NEXT: XOR_INT * T0.Y, PV.W, PS, +; R600-NEXT: ALU clause starting at 31: +; R600-NEXT: LSHR * T1.X, T0.W, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) +entry: + %0 = icmp eq i64 %a, 0 + br i1 %0, label %if, label %else + +if: + %1 = xor i64 %a, %b + br label %endif + +else: + %2 = load i64, ptr addrspace(1) %in + br label %endif + +endif: + %3 = phi i64 [%1, %if], [%2, %else] + store i64 %3, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @scalar_xor_literal_i64(ptr addrspace(1) %out, [8 x i32], i64 %a) { +; R600-LABEL: scalar_xor_literal_i64: +; R600: ; %bb.0: +; R600-NEXT: ALU 4, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: XOR_INT * T0.Y, KC0[5].X, literal.x, +; R600-NEXT: 992123(1.390260e-39), 0(0.000000e+00) +; R600-NEXT: XOR_INT T0.X, KC0[4].W, literal.x, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.y, +; R600-NEXT: 12345(1.729903e-41), 2(2.802597e-45) + %or = xor i64 %a, 4261135838621753 + store i64 %or, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @scalar_xor_literal_multi_use_i64(ptr addrspace(1) %out, [8 x i32], i64 %a, i64 %b) { +; R600-LABEL: scalar_xor_literal_multi_use_i64: +; R600: ; %bb.0: +; R600-NEXT: ALU 12, @6, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T3.XY, T4.X, 0 +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T2.X, 0 +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T2.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 6: +; R600-NEXT: ADDC_UINT * T0.W, KC0[5].Y, literal.x, +; R600-NEXT: 12345(1.729903e-41), 0(0.000000e+00) +; R600-NEXT: ADD_INT T0.X, KC0[5].Y, literal.x, +; R600-NEXT: ADD_INT * T0.W, KC0[5].Z, PV.W, +; R600-NEXT: 12345(1.729903e-41), 0(0.000000e+00) +; R600-NEXT: ADD_INT T1.X, PV.W, literal.x, +; R600-NEXT: MOV * T2.X, literal.y, +; R600-NEXT: 992123(1.390260e-39), 0(0.000000e+00) +; R600-NEXT: XOR_INT * T3.Y, KC0[5].X, literal.x, +; R600-NEXT: 992123(1.390260e-39), 0(0.000000e+00) +; R600-NEXT: XOR_INT T3.X, KC0[4].W, literal.x, +; R600-NEXT: LSHR * T4.X, KC0[2].Y, literal.y, +; R600-NEXT: 12345(1.729903e-41), 2(2.802597e-45) + %or = xor i64 %a, 4261135838621753 + store i64 %or, ptr addrspace(1) %out + + %foo = add i64 %b, 4261135838621753 + store volatile i64 %foo, ptr addrspace(1) undef + ret void +} + +define amdgpu_kernel void @scalar_xor_inline_imm_i64(ptr addrspace(1) %out, [8 x i32], i64 %a) { +; R600-LABEL: scalar_xor_inline_imm_i64: +; R600: ; %bb.0: +; R600-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV * T0.Y, KC0[5].X, +; R600-NEXT: XOR_INT T0.X, KC0[4].W, literal.x, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.y, +; R600-NEXT: 63(8.828180e-44), 2(2.802597e-45) + %or = xor i64 %a, 63 + store i64 %or, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @scalar_xor_neg_inline_imm_i64(ptr addrspace(1) %out, [8 x i32], i64 %a) { +; R600-LABEL: scalar_xor_neg_inline_imm_i64: +; R600: ; %bb.0: +; R600-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: NOT_INT * T0.Y, KC0[5].X, +; R600-NEXT: XOR_INT T0.X, KC0[4].W, literal.x, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.y, +; R600-NEXT: -8(nan), 2(2.802597e-45) + %or = xor i64 %a, -8 + store i64 %or, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @vector_xor_i64_neg_inline_imm(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) { +; R600-LABEL: vector_xor_i64_neg_inline_imm: +; R600: ; %bb.0: +; R600-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 0 @6 +; R600-NEXT: ALU 3, @9, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: Fetch clause starting at 6: +; R600-NEXT: VTX_READ_64 T0.XY, T0.X, 0, #1 +; R600-NEXT: ALU clause starting at 8: +; R600-NEXT: MOV * T0.X, KC0[2].Z, +; R600-NEXT: ALU clause starting at 9: +; R600-NEXT: NOT_INT * T0.Y, T0.Y, +; R600-NEXT: XOR_INT T0.X, T0.X, literal.x, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.y, +; R600-NEXT: -8(nan), 2(2.802597e-45) + %loada = load i64, ptr addrspace(1) %a, align 8 + %or = xor i64 %loada, -8 + store i64 %or, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @vector_xor_literal_i64(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) { +; R600-LABEL: vector_xor_literal_i64: +; R600: ; %bb.0: +; R600-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] +; R600-NEXT: TEX 0 @6 +; R600-NEXT: ALU 4, @9, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: Fetch clause starting at 6: +; R600-NEXT: VTX_READ_64 T0.XY, T0.X, 0, #1 +; R600-NEXT: ALU clause starting at 8: +; R600-NEXT: MOV * T0.X, KC0[2].Z, +; R600-NEXT: ALU clause starting at 9: +; R600-NEXT: XOR_INT * T0.Y, T0.Y, literal.x, +; R600-NEXT: 5231(7.330192e-42), 0(0.000000e+00) +; R600-NEXT: XOR_INT T0.X, T0.X, literal.x, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.y, +; R600-NEXT: -545810305(-1.784115e+19), 2(2.802597e-45) + %loada = load i64, ptr addrspace(1) %a, align 8 + %or = xor i64 %loada, 22470723082367 + store i64 %or, ptr addrspace(1) %out + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/xor.ll b/llvm/test/CodeGen/AMDGPU/xor.ll index 1315c0b52af4..e15fd7f29671 100644 --- a/llvm/test/CodeGen/AMDGPU/xor.ll +++ b/llvm/test/CodeGen/AMDGPU/xor.ll @@ -1,16 +1,49 @@ -; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=r600 -mcpu=redwood < %s | FileCheck -enable-var-scope -check-prefix=EG -check-prefix=FUNC %s - - -; FUNC-LABEL: {{^}}xor_v2i32: -; EG: XOR_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} -; EG: XOR_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} - -; SI: v_xor_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} -; SI: v_xor_b32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=verde < %s | FileCheck -enable-var-scope -check-prefixes=SI,GCN %s +; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -enable-var-scope -check-prefixes=VI,GCN %s define amdgpu_kernel void @xor_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; SI-LABEL: xor_v2i32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 +; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_mov_b32 s10, s2 +; SI-NEXT: s_mov_b32 s11, s3 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s12, s6 +; SI-NEXT: s_mov_b32 s13, s7 +; SI-NEXT: s_mov_b32 s14, s2 +; SI-NEXT: s_mov_b32 s15, s3 +; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0 +; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[12:15], 0 +; SI-NEXT: s_mov_b32 s0, s4 +; SI-NEXT: s_mov_b32 s1, s5 +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: v_xor_b32_e32 v1, v3, v1 +; SI-NEXT: v_xor_b32_e32 v0, v2, v0 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: xor_v2i32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s6 +; VI-NEXT: v_mov_b32_e32 v1, s7 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] +; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3] +; VI-NEXT: v_mov_b32_e32 v4, s4 +; VI-NEXT: v_mov_b32_e32 v5, s5 +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: v_xor_b32_e32 v1, v1, v3 +; VI-NEXT: v_xor_b32_e32 v0, v0, v2 +; VI-NEXT: flat_store_dwordx2 v[4:5], v[0:1] +; VI-NEXT: s_endpgm %a = load <2 x i32>, ptr addrspace(1) %in0 %b = load <2 x i32>, ptr addrspace(1) %in1 %result = xor <2 x i32> %a, %b @@ -18,18 +51,52 @@ define amdgpu_kernel void @xor_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %in ret void } -; FUNC-LABEL: {{^}}xor_v4i32: -; EG: XOR_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} -; EG: XOR_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} -; EG: XOR_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} -; EG: XOR_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} - -; SI: v_xor_b32_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}} -; SI: v_xor_b32_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}} -; SI: v_xor_b32_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}} -; SI: v_xor_b32_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}} - define amdgpu_kernel void @xor_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; SI-LABEL: xor_v4i32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 +; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_mov_b32 s10, s2 +; SI-NEXT: s_mov_b32 s11, s3 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s12, s6 +; SI-NEXT: s_mov_b32 s13, s7 +; SI-NEXT: s_mov_b32 s14, s2 +; SI-NEXT: s_mov_b32 s15, s3 +; SI-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 +; SI-NEXT: buffer_load_dwordx4 v[4:7], off, s[12:15], 0 +; SI-NEXT: s_mov_b32 s0, s4 +; SI-NEXT: s_mov_b32 s1, s5 +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: v_xor_b32_e32 v3, v7, v3 +; SI-NEXT: v_xor_b32_e32 v2, v6, v2 +; SI-NEXT: v_xor_b32_e32 v1, v5, v1 +; SI-NEXT: v_xor_b32_e32 v0, v4, v0 +; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: xor_v4i32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s6 +; VI-NEXT: v_mov_b32_e32 v1, s7 +; VI-NEXT: v_mov_b32_e32 v4, s0 +; VI-NEXT: v_mov_b32_e32 v5, s1 +; VI-NEXT: flat_load_dwordx4 v[0:3], v[0:1] +; VI-NEXT: flat_load_dwordx4 v[4:7], v[4:5] +; VI-NEXT: v_mov_b32_e32 v8, s4 +; VI-NEXT: v_mov_b32_e32 v9, s5 +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: v_xor_b32_e32 v3, v3, v7 +; VI-NEXT: v_xor_b32_e32 v2, v2, v6 +; VI-NEXT: v_xor_b32_e32 v1, v1, v5 +; VI-NEXT: v_xor_b32_e32 v0, v0, v4 +; VI-NEXT: flat_store_dwordx4 v[8:9], v[0:3] +; VI-NEXT: s_endpgm %a = load <4 x i32>, ptr addrspace(1) %in0 %b = load <4 x i32>, ptr addrspace(1) %in1 %result = xor <4 x i32> %a, %b @@ -37,16 +104,54 @@ define amdgpu_kernel void @xor_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %in ret void } -; FUNC-LABEL: {{^}}xor_i1: -; EG: XOR_INT {{\** *}}{{T[0-9]+\.[XYZW]}}, {{PS|PV\.[XYZW]}}, {{PS|PV\.[XYZW]}} - -; SI-DAG: v_cmp_le_f32_e32 [[CMP0:vcc]], 1.0, {{v[0-9]+}} -; SI-DAG: v_cmp_le_f32_e64 [[CMP1:s\[[0-9]+:[0-9]+\]]], 0, {{v[0-9]+}} -; SI: s_xor_b64 [[XOR:vcc]], [[CMP1]], [[CMP0]] -; SI: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}} -; SI: buffer_store_dword [[RESULT]] -; SI: s_endpgm define amdgpu_kernel void @xor_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; SI-LABEL: xor_i1: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd +; SI-NEXT: s_mov_b32 s11, 0xf000 +; SI-NEXT: s_mov_b32 s10, -1 +; SI-NEXT: s_mov_b32 s2, s10 +; SI-NEXT: s_mov_b32 s3, s11 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s12, s6 +; SI-NEXT: s_mov_b32 s13, s7 +; SI-NEXT: s_mov_b32 s14, s10 +; SI-NEXT: s_mov_b32 s15, s11 +; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 +; SI-NEXT: buffer_load_dword v1, off, s[12:15], 0 +; SI-NEXT: s_mov_b32 s8, s4 +; SI-NEXT: s_mov_b32 s9, s5 +; SI-NEXT: s_waitcnt vmcnt(1) +; SI-NEXT: v_cmp_le_f32_e32 vcc, 1.0, v0 +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: v_cmp_le_f32_e64 s[0:1], 0, v1 +; SI-NEXT: s_xor_b64 vcc, s[0:1], vcc +; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: xor_i1: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s6 +; VI-NEXT: v_mov_b32_e32 v1, s7 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: flat_load_dword v4, v[0:1] +; VI-NEXT: flat_load_dword v2, v[2:3] +; VI-NEXT: v_mov_b32_e32 v0, s4 +; VI-NEXT: v_mov_b32_e32 v1, s5 +; VI-NEXT: s_waitcnt vmcnt(1) +; VI-NEXT: v_cmp_le_f32_e32 vcc, 0, v4 +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: v_cmp_le_f32_e64 s[0:1], 1.0, v2 +; VI-NEXT: s_xor_b64 vcc, vcc, s[0:1] +; VI-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %a = load float, ptr addrspace(1) %in0 %b = load float, ptr addrspace(1) %in1 %acmp = fcmp oge float %a, 0.000000e+00 @@ -57,13 +162,50 @@ define amdgpu_kernel void @xor_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ret void } -; FUNC-LABEL: {{^}}v_xor_i1: -; SI: buffer_load_ubyte [[B:v[0-9]+]] -; SI: buffer_load_ubyte [[A:v[0-9]+]] -; SI: v_xor_b32_e32 [[XOR:v[0-9]+]], [[B]], [[A]] -; SI: v_and_b32_e32 [[RESULT:v[0-9]+]], 1, [[XOR]] -; SI: buffer_store_byte [[RESULT]] define amdgpu_kernel void @v_xor_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; SI-LABEL: v_xor_i1: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 +; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_mov_b32 s14, s2 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s12, s6 +; SI-NEXT: s_mov_b32 s13, s7 +; SI-NEXT: s_mov_b32 s15, s3 +; SI-NEXT: s_mov_b32 s10, s2 +; SI-NEXT: s_mov_b32 s11, s3 +; SI-NEXT: buffer_load_ubyte v0, off, s[12:15], 0 glc +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 glc +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: s_mov_b32 s0, s4 +; SI-NEXT: s_mov_b32 s1, s5 +; SI-NEXT: v_xor_b32_e32 v0, v0, v1 +; SI-NEXT: v_and_b32_e32 v0, 1, v0 +; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: v_xor_i1: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s6 +; VI-NEXT: v_mov_b32_e32 v1, s7 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: flat_load_ubyte v4, v[0:1] glc +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: flat_load_ubyte v2, v[2:3] glc +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s4 +; VI-NEXT: v_mov_b32_e32 v1, s5 +; VI-NEXT: v_xor_b32_e32 v2, v4, v2 +; VI-NEXT: v_and_b32_e32 v2, 1, v2 +; VI-NEXT: flat_store_byte v[0:1], v2 +; VI-NEXT: s_endpgm %a = load volatile i1, ptr addrspace(1) %in0 %b = load volatile i1, ptr addrspace(1) %in1 %xor = xor i1 %a, %b @@ -71,9 +213,46 @@ define amdgpu_kernel void @v_xor_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0 ret void } -; FUNC-LABEL: {{^}}vector_xor_i32: -; SI: v_xor_b32_e32 define amdgpu_kernel void @vector_xor_i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; SI-LABEL: vector_xor_i32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 +; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_mov_b32 s14, s2 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s12, s6 +; SI-NEXT: s_mov_b32 s13, s7 +; SI-NEXT: s_mov_b32 s15, s3 +; SI-NEXT: s_mov_b32 s10, s2 +; SI-NEXT: s_mov_b32 s11, s3 +; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0 +; SI-NEXT: buffer_load_dword v1, off, s[8:11], 0 +; SI-NEXT: s_mov_b32 s0, s4 +; SI-NEXT: s_mov_b32 s1, s5 +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: v_xor_b32_e32 v0, v0, v1 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: vector_xor_i32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s6 +; VI-NEXT: v_mov_b32_e32 v1, s7 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: flat_load_dword v4, v[0:1] +; VI-NEXT: flat_load_dword v2, v[2:3] +; VI-NEXT: v_mov_b32_e32 v0, s4 +; VI-NEXT: v_mov_b32_e32 v1, s5 +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: v_xor_b32_e32 v2, v4, v2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %a = load i32, ptr addrspace(1) %in0 %b = load i32, ptr addrspace(1) %in1 %result = xor i32 %a, %b @@ -81,25 +260,96 @@ define amdgpu_kernel void @vector_xor_i32(ptr addrspace(1) %out, ptr addrspace(1 ret void } -; FUNC-LABEL: {{^}}scalar_xor_i32: -; SI: s_xor_b32 define amdgpu_kernel void @scalar_xor_i32(ptr addrspace(1) %out, i32 %a, i32 %b) { +; SI-LABEL: scalar_xor_i32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_xor_b32 s0, s2, s3 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: v_mov_b32_e32 v0, s0 +; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: scalar_xor_i32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_xor_b32 s2, s2, s3 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %result = xor i32 %a, %b store i32 %result, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}scalar_not_i32: -; SI: s_not_b32 define amdgpu_kernel void @scalar_not_i32(ptr addrspace(1) %out, i32 %a) { +; SI-LABEL: scalar_not_i32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dword s4, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_not_b32 s4, s4 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: scalar_not_i32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dword s2, s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_not_b32 s2, s2 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %result = xor i32 %a, -1 store i32 %result, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}vector_not_i32: -; SI: v_not_b32 define amdgpu_kernel void @vector_not_i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; SI-LABEL: vector_not_i32: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_mov_b32 s10, s6 +; SI-NEXT: s_mov_b32 s11, s7 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s8, s2 +; SI-NEXT: s_mov_b32 s9, s3 +; SI-NEXT: buffer_load_dword v0, off, s[8:11], 0 +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: v_not_b32_e32 v0, v0 +; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: vector_not_i32: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: flat_load_dword v2, v[0:1] +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: v_not_b32_e32 v2, v2 +; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: s_endpgm %a = load i32, ptr addrspace(1) %in0 %b = load i32, ptr addrspace(1) %in1 %result = xor i32 %a, -1 @@ -107,11 +357,48 @@ define amdgpu_kernel void @vector_not_i32(ptr addrspace(1) %out, ptr addrspace(1 ret void } -; FUNC-LABEL: {{^}}vector_xor_i64: -; SI: v_xor_b32_e32 -; SI: v_xor_b32_e32 -; SI: s_endpgm define amdgpu_kernel void @vector_xor_i64(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; SI-LABEL: vector_xor_i64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 +; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_mov_b32 s10, s2 +; SI-NEXT: s_mov_b32 s11, s3 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s12, s6 +; SI-NEXT: s_mov_b32 s13, s7 +; SI-NEXT: s_mov_b32 s14, s2 +; SI-NEXT: s_mov_b32 s15, s3 +; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0 +; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[12:15], 0 +; SI-NEXT: s_mov_b32 s0, s4 +; SI-NEXT: s_mov_b32 s1, s5 +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: v_xor_b32_e32 v0, v2, v0 +; SI-NEXT: v_xor_b32_e32 v1, v3, v1 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: vector_xor_i64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s6 +; VI-NEXT: v_mov_b32_e32 v1, s7 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] +; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3] +; VI-NEXT: v_mov_b32_e32 v4, s4 +; VI-NEXT: v_mov_b32_e32 v5, s5 +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: v_xor_b32_e32 v0, v0, v2 +; VI-NEXT: v_xor_b32_e32 v1, v1, v3 +; VI-NEXT: flat_store_dwordx2 v[4:5], v[0:1] +; VI-NEXT: s_endpgm %a = load i64, ptr addrspace(1) %in0 %b = load i64, ptr addrspace(1) %in1 %result = xor i64 %a, %b @@ -119,27 +406,104 @@ define amdgpu_kernel void @vector_xor_i64(ptr addrspace(1) %out, ptr addrspace(1 ret void } -; FUNC-LABEL: {{^}}scalar_xor_i64: -; SI: s_xor_b64 -; SI: s_endpgm define amdgpu_kernel void @scalar_xor_i64(ptr addrspace(1) %out, i64 %a, i64 %b) { +; SI-LABEL: scalar_xor_i64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 +; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s0, s4 +; SI-NEXT: s_mov_b32 s1, s5 +; SI-NEXT: s_xor_b64 s[4:5], s[6:7], s[8:9] +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: v_mov_b32_e32 v1, s5 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: scalar_xor_i64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s4 +; VI-NEXT: s_xor_b64 s[0:1], s[6:7], s[0:1] +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: v_mov_b32_e32 v1, s5 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3] +; VI-NEXT: s_endpgm %result = xor i64 %a, %b store i64 %result, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}scalar_not_i64: -; SI: s_not_b64 define amdgpu_kernel void @scalar_not_i64(ptr addrspace(1) %out, i64 %a) { +; SI-LABEL: scalar_not_i64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: s_not_b64 s[0:1], s[2:3] +; SI-NEXT: v_mov_b32_e32 v0, s0 +; SI-NEXT: v_mov_b32_e32 v1, s1 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: scalar_not_i64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: s_not_b64 s[0:1], s[2:3] +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3] +; VI-NEXT: s_endpgm %result = xor i64 %a, -1 store i64 %result, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}vector_not_i64: -; SI: v_not_b32 -; SI: v_not_b32 define amdgpu_kernel void @vector_not_i64(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) { +; SI-LABEL: vector_not_i64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_mov_b32 s10, s6 +; SI-NEXT: s_mov_b32 s11, s7 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s8, s2 +; SI-NEXT: s_mov_b32 s9, s3 +; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0 +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: v_not_b32_e32 v0, v0 +; SI-NEXT: v_not_b32_e32 v1, v1 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: vector_not_i64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: v_not_b32_e32 v0, v0 +; VI-NEXT: v_not_b32_e32 v1, v1 +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm %a = load i64, ptr addrspace(1) %in0 %b = load i64, ptr addrspace(1) %in1 %result = xor i64 %a, -1 @@ -147,13 +511,65 @@ define amdgpu_kernel void @vector_not_i64(ptr addrspace(1) %out, ptr addrspace(1 ret void } -; Test that we have a pattern to match xor inside a branch. -; Note that in the future the backend may be smart enough to -; use an SALU instruction for this. - -; FUNC-LABEL: {{^}}xor_cf: -; SI: s_xor_b64 define amdgpu_kernel void @xor_cf(ptr addrspace(1) %out, ptr addrspace(1) %in, i64 %a, i64 %b) { +; SI-LABEL: xor_cf: +; SI: ; %bb.0: ; %entry +; SI-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x9 +; SI-NEXT: s_mov_b64 s[8:9], 0 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: v_cmp_ne_u64_e64 s[10:11], s[4:5], 0 +; SI-NEXT: s_and_b64 vcc, exec, s[10:11] +; SI-NEXT: s_cbranch_vccz .LBB12_4 +; SI-NEXT: ; %bb.1: ; %else +; SI-NEXT: s_mov_b32 s15, 0xf000 +; SI-NEXT: s_mov_b32 s14, -1 +; SI-NEXT: s_mov_b32 s12, s2 +; SI-NEXT: s_mov_b32 s13, s3 +; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[12:15], 0 +; SI-NEXT: s_andn2_b64 vcc, exec, s[8:9] +; SI-NEXT: s_cbranch_vccnz .LBB12_3 +; SI-NEXT: .LBB12_2: ; %if +; SI-NEXT: s_xor_b64 s[2:3], s[4:5], s[6:7] +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: v_mov_b32_e32 v0, s2 +; SI-NEXT: v_mov_b32_e32 v1, s3 +; SI-NEXT: .LBB12_3: ; %endif +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; SI-NEXT: .LBB12_4: +; SI-NEXT: ; implicit-def: $vgpr0_vgpr1 +; SI-NEXT: s_branch .LBB12_2 +; +; VI-LABEL: xor_cf: +; VI: ; %bb.0: ; %entry +; VI-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24 +; VI-NEXT: s_mov_b64 s[8:9], 0 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_cmp_lg_u64 s[4:5], 0 +; VI-NEXT: s_cbranch_scc0 .LBB12_4 +; VI-NEXT: ; %bb.1: ; %else +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] +; VI-NEXT: s_andn2_b64 vcc, exec, s[8:9] +; VI-NEXT: s_cbranch_vccnz .LBB12_3 +; VI-NEXT: .LBB12_2: ; %if +; VI-NEXT: s_xor_b64 s[2:3], s[4:5], s[6:7] +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: .LBB12_3: ; %endif +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm +; VI-NEXT: .LBB12_4: +; VI-NEXT: ; implicit-def: $vgpr0_vgpr1 +; VI-NEXT: s_branch .LBB12_2 entry: %0 = icmp eq i64 %a, 0 br i1 %0, label %if, label %else @@ -172,27 +588,82 @@ endif: ret void } -; FUNC-LABEL: {{^}}scalar_xor_literal_i64: -; SI: s_load_dwordx2 s[[[LO:[0-9]+]]:[[HI:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, {{0x9|0x24}} -; SI-DAG: s_xor_b32 s[[RES_HI:[0-9]+]], s{{[0-9]+}}, 0xf237b -; SI-DAG: s_xor_b32 s[[RES_LO:[0-9]+]], s{{[0-9]+}}, 0x3039 -; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[RES_LO]] -; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[RES_HI]] define amdgpu_kernel void @scalar_xor_literal_i64(ptr addrspace(1) %out, [8 x i32], i64 %a) { +; SI-LABEL: scalar_xor_literal_i64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x13 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_xor_b32 s5, s5, 0xf237b +; SI-NEXT: s_xor_b32 s4, s4, 0x3039 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: v_mov_b32_e32 v1, s5 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: scalar_xor_literal_i64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x4c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_xor_b32 s3, s3, 0xf237b +; VI-NEXT: s_xor_b32 s2, s2, 0x3039 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm %or = xor i64 %a, 4261135838621753 store i64 %or, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}scalar_xor_literal_multi_use_i64: -; SI: s_load_dwordx4 s[[[LO:[0-9]+]]:[[HI:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, {{0x13|0x4c}} -; SI-DAG: s_mov_b32 s[[K_HI:[0-9]+]], 0xf237b -; SI-DAG: s_movk_i32 s[[K_LO:[0-9]+]], 0x3039 -; SI: s_xor_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s[[[K_LO]]:[[K_HI]]] - -; SI: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, 0x3039 -; SI: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, 0xf237b define amdgpu_kernel void @scalar_xor_literal_multi_use_i64(ptr addrspace(1) %out, [8 x i32], i64 %a, i64 %b) { +; SI-LABEL: scalar_xor_literal_multi_use_i64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x13 +; SI-NEXT: s_movk_i32 s8, 0x3039 +; SI-NEXT: s_mov_b32 s9, 0xf237b +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_xor_b64 s[0:1], s[0:1], s[8:9] +; SI-NEXT: v_mov_b32_e32 v0, s0 +; SI-NEXT: v_mov_b32_e32 v1, s1 +; SI-NEXT: s_add_u32 s0, s2, 0x3039 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: s_addc_u32 s1, s3, 0xf237b +; SI-NEXT: s_waitcnt expcnt(0) +; SI-NEXT: v_mov_b32_e32 v0, s0 +; SI-NEXT: v_mov_b32_e32 v1, s1 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: s_endpgm +; +; VI-LABEL: scalar_xor_literal_multi_use_i64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x4c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_movk_i32 s2, 0x3039 +; VI-NEXT: s_mov_b32 s3, 0xf237b +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_xor_b64 s[2:3], s[4:5], s[2:3] +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v3, s3 +; VI-NEXT: s_add_u32 s0, s6, 0x3039 +; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3] +; VI-NEXT: s_addc_u32 s1, s7, 0xf237b +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: flat_store_dwordx2 v[0:1], v[0:1] +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: s_endpgm %or = xor i64 %a, 4261135838621753 store i64 %or, ptr addrspace(1) %out @@ -201,51 +672,146 @@ define amdgpu_kernel void @scalar_xor_literal_multi_use_i64(ptr addrspace(1) %ou ret void } -; FUNC-LABEL: {{^}}scalar_xor_inline_imm_i64: -; SI: s_load_dwordx2 s[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, {{0x13|0x4c}} -; SI-NOT: xor_b32 -; SI: s_xor_b32 s[[VAL_LO]], s{{[0-9]+}}, 63 -; SI-NOT: xor_b32 -; SI: v_mov_b32_e32 v[[VLO:[0-9]+]], s{{[0-9]+}} -; SI-NOT: xor_b32 -; SI: v_mov_b32_e32 v[[VHI:[0-9]+]], s{{[0-9]+}} -; SI-NOT: xor_b32 -; SI: buffer_store_dwordx2 v[[[VLO]]:[[VHI]]] define amdgpu_kernel void @scalar_xor_inline_imm_i64(ptr addrspace(1) %out, [8 x i32], i64 %a) { +; SI-LABEL: scalar_xor_inline_imm_i64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x13 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_xor_b32 s4, s4, 63 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: v_mov_b32_e32 v1, s5 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: scalar_xor_inline_imm_i64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x4c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_xor_b32 s2, s2, 63 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm %or = xor i64 %a, 63 store i64 %or, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}scalar_xor_neg_inline_imm_i64: -; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], s{{\[[0-9]+:[0-9]+\]}}, {{0x13|0x4c}} -; SI: s_xor_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, -8 define amdgpu_kernel void @scalar_xor_neg_inline_imm_i64(ptr addrspace(1) %out, [8 x i32], i64 %a) { +; SI-LABEL: scalar_xor_neg_inline_imm_i64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x13 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_xor_b64 s[4:5], s[4:5], -8 +; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: v_mov_b32_e32 v1, s5 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: scalar_xor_neg_inline_imm_i64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x4c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: s_xor_b64 s[2:3], s[2:3], -8 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v3, s3 +; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3] +; VI-NEXT: s_endpgm %or = xor i64 %a, -8 store i64 %or, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}vector_xor_i64_neg_inline_imm: -; SI: buffer_load_dwordx2 v[[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]], -; SI: v_xor_b32_e32 {{v[0-9]+}}, -8, v[[LO_VREG]] -; SI: v_xor_b32_e32 {{v[0-9]+}}, -1, {{.*}} -; SI: s_endpgm define amdgpu_kernel void @vector_xor_i64_neg_inline_imm(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) { +; SI-LABEL: vector_xor_i64_neg_inline_imm: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_mov_b32 s10, s6 +; SI-NEXT: s_mov_b32 s11, s7 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s8, s2 +; SI-NEXT: s_mov_b32 s9, s3 +; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0 +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: v_xor_b32_e32 v0, -8, v0 +; SI-NEXT: v_xor_b32_e32 v1, -1, v1 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: vector_xor_i64_neg_inline_imm: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: v_xor_b32_e32 v0, -8, v0 +; VI-NEXT: v_xor_b32_e32 v1, -1, v1 +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm %loada = load i64, ptr addrspace(1) %a, align 8 %or = xor i64 %loada, -8 store i64 %or, ptr addrspace(1) %out ret void } -; FUNC-LABEL: {{^}}vector_xor_literal_i64: -; SI-DAG: buffer_load_dwordx2 v[[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]], -; SI-DAG: v_xor_b32_e32 {{v[0-9]+}}, 0xdf77987f, v[[LO_VREG]] -; SI-DAG: v_xor_b32_e32 {{v[0-9]+}}, 0x146f, v[[HI_VREG]] -; SI: s_endpgm define amdgpu_kernel void @vector_xor_literal_i64(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) { +; SI-LABEL: vector_xor_literal_i64: +; SI: ; %bb.0: +; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: s_mov_b32 s10, s6 +; SI-NEXT: s_mov_b32 s11, s7 +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_mov_b32 s8, s2 +; SI-NEXT: s_mov_b32 s9, s3 +; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0 +; SI-NEXT: s_mov_b32 s4, s0 +; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: v_xor_b32_e32 v1, 0x146f, v1 +; SI-NEXT: v_xor_b32_e32 v0, 0xdf77987f, v0 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: s_endpgm +; +; VI-LABEL: vector_xor_literal_i64: +; VI: ; %bb.0: +; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_waitcnt lgkmcnt(0) +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] +; VI-NEXT: v_mov_b32_e32 v2, s0 +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: v_xor_b32_e32 v1, 0x146f, v1 +; VI-NEXT: v_xor_b32_e32 v0, 0xdf77987f, v0 +; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] +; VI-NEXT: s_endpgm %loada = load i64, ptr addrspace(1) %a, align 8 %or = xor i64 %loada, 22470723082367 store i64 %or, ptr addrspace(1) %out ret void } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GCN: {{.*}} -- GitLab From 1bc8b3258e6d42f702fb11eb60d84d0e23935e3e Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 7 Jun 2024 12:22:42 +0800 Subject: [PATCH 187/462] [NewPM][CodeGen] Port `regallocfast` to new pass manager (#94426) This pull request port `regallocfast` to new pass manager. It exposes the parameter `filter` to handle different register classes for AMDGPU. IIUC AMDGPU need to allocate different register classes separately so it need implement its own `---regalloc`. Now users can use e.g. `-passe=regallocfast` to allocate specific register class. The command line option `--regalloc-npm` is still in work progress, plan to reuse the syntax of passes, e.g. use `--regalloc-npm=regallocfast,greedy` to replace `--sgpr-regalloc` and `--vgpr-regalloc`. --- .../include/llvm/CodeGen/MachinePassManager.h | 6 +- llvm/include/llvm/CodeGen/RegAllocFast.h | 57 +++++ llvm/include/llvm/Passes/CodeGenPassBuilder.h | 3 +- .../llvm/Passes/MachinePassRegistry.def | 14 +- llvm/include/llvm/Passes/PassBuilder.h | 15 ++ llvm/lib/CodeGen/RegAllocFast.cpp | 201 +++++++++++------- llvm/lib/Passes/PassBuilder.cpp | 56 +++++ .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 9 + .../fast-regalloc-empty-bb-with-liveins.mir | 1 + .../test/CodeGen/AMDGPU/fast-ra-kills-vcc.mir | 1 + .../CodeGen/AMDGPU/fast-regalloc-bundles.mir | 1 + .../fastregalloc-illegal-subreg-physreg.mir | 1 + .../fastregalloc-self-loop-heuristic.mir | 1 + llvm/test/CodeGen/AMDGPU/spill-agpr.mir | 2 + llvm/test/CodeGen/AMDGPU/spill192.mir | 1 + llvm/test/CodeGen/AMDGPU/spill224.mir | 1 + llvm/test/CodeGen/AMDGPU/spill288.mir | 1 + llvm/test/CodeGen/AMDGPU/spill320.mir | 1 + llvm/test/CodeGen/AMDGPU/spill352.mir | 1 + llvm/test/CodeGen/AMDGPU/spill384.mir | 1 + .../AMDGPU/unexpected-reg-unit-state.mir | 1 + .../ARM/regalloc-fast-rewrite-implicits.mir | 1 + llvm/test/CodeGen/MIR/Generic/runPass.mir | 1 + llvm/test/CodeGen/PowerPC/spill-nor0.mir | 1 + .../regalloc-fast-invalid-kill-flag.mir | 1 + llvm/test/CodeGen/Thumb/high-reg-clobber.mir | 1 + llvm/test/CodeGen/Thumb2/high-reg-spill.mir | 1 + .../CodeGen/X86/bug47278-eflags-error.mir | 1 + llvm/test/CodeGen/X86/bug47278.mir | 1 + .../X86/callbr-asm-outputs-regallocfast.mir | 1 + .../CodeGen/X86/fastregalloc-selfloop.mir | 1 + .../CodeGen/X86/fastregalloc-tied-undef.mir | 1 + .../regalloc-fast-missing-live-out-spill.mir | 1 + .../CodeGen/X86/statepoint-fastregalloc.mir | 1 + .../X86/virtreg-physreg-def-regallocfast.mir | 1 + .../test/tools/llc/new-pm/regalloc-amdgpu.mir | 10 + 36 files changed, 317 insertions(+), 82 deletions(-) create mode 100644 llvm/include/llvm/CodeGen/RegAllocFast.h create mode 100644 llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h index 852b1a0f4161..7d15664fbe75 100644 --- a/llvm/include/llvm/CodeGen/MachinePassManager.h +++ b/llvm/include/llvm/CodeGen/MachinePassManager.h @@ -61,7 +61,7 @@ struct MachinePassModel #ifndef NDEBUG if constexpr (is_detected::value) { auto &MFProps = IR.getProperties(); - auto RequiredProperties = PassT::getRequiredProperties(); + auto RequiredProperties = this->Pass.getRequiredProperties(); if (!MFProps.verifyRequiredProperties(RequiredProperties)) { errs() << "MachineFunctionProperties required by " << PassT::name() << " pass are not met by function " << IR.getName() << ".\n" @@ -78,9 +78,9 @@ struct MachinePassModel auto PA = this->Pass.run(IR, AM); if constexpr (is_detected::value) - IR.getProperties().set(PassT::getSetProperties()); + IR.getProperties().set(this->Pass.getSetProperties()); if constexpr (is_detected::value) - IR.getProperties().reset(PassT::getClearedProperties()); + IR.getProperties().reset(this->Pass.getClearedProperties()); return PA; } diff --git a/llvm/include/llvm/CodeGen/RegAllocFast.h b/llvm/include/llvm/CodeGen/RegAllocFast.h new file mode 100644 index 000000000000..c50deccabd99 --- /dev/null +++ b/llvm/include/llvm/CodeGen/RegAllocFast.h @@ -0,0 +1,57 @@ +//==- RegAllocFast.h ----------- fast register allocator ----------*-C++-*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_REGALLOCFAST_H +#define LLVM_CODEGEN_REGALLOCFAST_H + +#include "llvm/CodeGen/MachinePassManager.h" +#include "llvm/CodeGen/RegAllocCommon.h" + +namespace llvm { + +struct RegAllocFastPassOptions { + RegClassFilterFunc Filter = allocateAllRegClasses; + StringRef FilterName = "all"; + bool ClearVRegs = true; +}; + +class RegAllocFastPass : public PassInfoMixin { + RegAllocFastPassOptions Opts; + +public: + RegAllocFastPass(RegAllocFastPassOptions Opts = RegAllocFastPassOptions()) + : Opts(Opts) {} + + MachineFunctionProperties getRequiredProperties() { + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::NoPHIs); + } + + MachineFunctionProperties getSetProperties() { + if (Opts.ClearVRegs) { + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::NoVRegs); + } + + return MachineFunctionProperties(); + } + + MachineFunctionProperties getClearedProperties() { + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::IsSSA); + } + + PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &); + + void printPipeline(raw_ostream &OS, + function_ref MapClassName2PassName); +}; + +} // namespace llvm + +#endif // LLVM_CODEGEN_REGALLOCFAST_H diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index 7542986837ec..eef1709bce5d 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -43,6 +43,7 @@ #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/PreISelIntrinsicLowering.h" +#include "llvm/CodeGen/RegAllocFast.h" #include "llvm/CodeGen/ReplaceWithVeclib.h" #include "llvm/CodeGen/SafeStack.h" #include "llvm/CodeGen/SelectOptimize.h" @@ -1038,7 +1039,7 @@ void CodeGenPassBuilder::addTargetRegisterAllocator( if (Optimized) addPass(RAGreedyPass()); else - addPass(RAFastPass()); + addPass(RegAllocFastPass()); } /// Find and instantiate the register allocation pass requested by this target diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index 4152c35ded6c..e59591e473b7 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -133,6 +133,19 @@ MACHINE_FUNCTION_PASS("require-all-machine-function-properties", MACHINE_FUNCTION_PASS("trigger-verifier-error", TriggerVerifierErrorPass()) #undef MACHINE_FUNCTION_PASS +#ifndef MACHINE_FUNCTION_PASS_WITH_PARAMS +#define MACHINE_FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, \ + PARAMS) +#endif +MACHINE_FUNCTION_PASS_WITH_PARAMS( + "regallocfast", "RegAllocFast", + [](RegAllocFastPassOptions Opts) { return RegAllocFastPass(Opts); }, + [PB = this](StringRef Params) { + return parseRegAllocFastPassOptions(*PB, Params); + }, + "filter=reg-filter;no-clear-vregs") +#undef MACHINE_FUNCTION_PASS_WITH_PARAMS + // After a pass is converted to new pass manager, its entry should be moved from // dummy table to the normal one. For example, for a machine function pass, // DUMMY_MACHINE_FUNCTION_PASS to MACHINE_FUNCTION_PASS. @@ -211,7 +224,6 @@ DUMMY_MACHINE_FUNCTION_PASS("processimpdefs", ProcessImplicitDefsPass) DUMMY_MACHINE_FUNCTION_PASS("prologepilog", PrologEpilogInserterPass) DUMMY_MACHINE_FUNCTION_PASS("prologepilog-code", PrologEpilogCodeInserterPass) DUMMY_MACHINE_FUNCTION_PASS("ra-basic", RABasicPass) -DUMMY_MACHINE_FUNCTION_PASS("ra-fast", RAFastPass) DUMMY_MACHINE_FUNCTION_PASS("ra-greedy", RAGreedyPass) DUMMY_MACHINE_FUNCTION_PASS("ra-pbqp", RAPBQPPass) DUMMY_MACHINE_FUNCTION_PASS("reg-usage-collector", RegUsageInfoCollectorPass) diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 8de0f024a4b1..ed817127c3db 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -17,6 +17,7 @@ #include "llvm/Analysis/CGSCCPassManager.h" #include "llvm/CodeGen/MachinePassManager.h" +#include "llvm/CodeGen/RegAllocCommon.h" #include "llvm/IR/PassManager.h" #include "llvm/Passes/OptimizationLevel.h" #include "llvm/Support/Error.h" @@ -388,6 +389,9 @@ public: /// returns false. Error parseAAPipeline(AAManager &AA, StringRef PipelineText); + /// Parse RegClassFilterName to get RegClassFilterFunc. + RegClassFilterFunc parseRegAllocFilter(StringRef RegClassFilterName); + /// Print pass names. void printPassNames(raw_ostream &OS); @@ -576,6 +580,14 @@ public: } /// @}} + /// Register callbacks to parse target specific filter field if regalloc pass + /// needs it. E.g. AMDGPU requires regalloc passes can handle sgpr and vgpr + /// separately. + void registerRegClassFilterParsingCallback( + const std::function &C) { + RegClassFilterParsingCallbacks.push_back(C); + } + /// Register a callback for a top-level pipeline entry. /// /// If the PassManager type is not given at the top level of the pipeline @@ -792,6 +804,9 @@ private: ArrayRef)>, 2> MachineFunctionPipelineParsingCallbacks; + // Callbacks to parse `filter` parameter in register allocation passes + SmallVector, 2> + RegClassFilterParsingCallbacks; }; /// This utility template takes care of adding require<> and invalidate<> diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp index 6740e1f0edb4..09ce8c42a385 100644 --- a/llvm/lib/CodeGen/RegAllocFast.cpp +++ b/llvm/lib/CodeGen/RegAllocFast.cpp @@ -12,6 +12,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/CodeGen/RegAllocFast.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IndexedMap.h" @@ -174,14 +175,12 @@ private: DenseMap Instr2PosIndex; }; -class RegAllocFast : public MachineFunctionPass { +class RegAllocFastImpl { public: - static char ID; - - RegAllocFast(const RegClassFilterFunc F = allocateAllRegClasses, - bool ClearVirtRegs_ = true) - : MachineFunctionPass(ID), ShouldAllocateClass(F), - StackSlotForVirtReg(-1), ClearVirtRegs(ClearVirtRegs_) {} + RegAllocFastImpl(const RegClassFilterFunc F = allocateAllRegClasses, + bool ClearVirtRegs_ = true) + : ShouldAllocateClass(F), StackSlotForVirtReg(-1), + ClearVirtRegs(ClearVirtRegs_) {} private: MachineFrameInfo *MFI = nullptr; @@ -197,8 +196,6 @@ private: /// Maps virtual regs to the frame index where these values are spilled. IndexedMap StackSlotForVirtReg; - bool ClearVirtRegs; - /// Everything we know about a live virtual register. struct LiveReg { MachineInstr *LastUse = nullptr; ///< Last instr to use reg. @@ -318,35 +315,11 @@ private: }; public: - StringRef getPassName() const override { return "Fast Register Allocator"; } - - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.setPreservesCFG(); - MachineFunctionPass::getAnalysisUsage(AU); - } - - MachineFunctionProperties getRequiredProperties() const override { - return MachineFunctionProperties().set( - MachineFunctionProperties::Property::NoPHIs); - } - - MachineFunctionProperties getSetProperties() const override { - if (ClearVirtRegs) { - return MachineFunctionProperties().set( - MachineFunctionProperties::Property::NoVRegs); - } - - return MachineFunctionProperties(); - } + bool ClearVirtRegs; - MachineFunctionProperties getClearedProperties() const override { - return MachineFunctionProperties().set( - MachineFunctionProperties::Property::IsSSA); - } + bool runOnMachineFunction(MachineFunction &MF); private: - bool runOnMachineFunction(MachineFunction &MF) override; - void allocateBasicBlock(MachineBasicBlock &MBB); void addRegClassDefCounts(std::vector &RegClassDefCounts, @@ -408,6 +381,47 @@ private: void dumpState() const; }; +class RegAllocFast : public MachineFunctionPass { + RegAllocFastImpl Impl; + +public: + static char ID; + + RegAllocFast(const RegClassFilterFunc F = allocateAllRegClasses, + bool ClearVirtRegs_ = true) + : MachineFunctionPass(ID), Impl(F, ClearVirtRegs_) {} + + bool runOnMachineFunction(MachineFunction &MF) override { + return Impl.runOnMachineFunction(MF); + } + + StringRef getPassName() const override { return "Fast Register Allocator"; } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + MachineFunctionProperties getRequiredProperties() const override { + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::NoPHIs); + } + + MachineFunctionProperties getSetProperties() const override { + if (Impl.ClearVirtRegs) { + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::NoVRegs); + } + + return MachineFunctionProperties(); + } + + MachineFunctionProperties getClearedProperties() const override { + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::IsSSA); + } +}; + } // end anonymous namespace char RegAllocFast::ID = 0; @@ -415,18 +429,18 @@ char RegAllocFast::ID = 0; INITIALIZE_PASS(RegAllocFast, "regallocfast", "Fast Register Allocator", false, false) -bool RegAllocFast::shouldAllocateRegister(const Register Reg) const { +bool RegAllocFastImpl::shouldAllocateRegister(const Register Reg) const { assert(Reg.isVirtual()); const TargetRegisterClass &RC = *MRI->getRegClass(Reg); return ShouldAllocateClass(*TRI, RC); } -void RegAllocFast::setPhysRegState(MCPhysReg PhysReg, unsigned NewState) { +void RegAllocFastImpl::setPhysRegState(MCPhysReg PhysReg, unsigned NewState) { for (MCRegUnit Unit : TRI->regunits(PhysReg)) RegUnitStates[Unit] = NewState; } -bool RegAllocFast::isPhysRegFree(MCPhysReg PhysReg) const { +bool RegAllocFastImpl::isPhysRegFree(MCPhysReg PhysReg) const { for (MCRegUnit Unit : TRI->regunits(PhysReg)) { if (RegUnitStates[Unit] != regFree) return false; @@ -436,7 +450,7 @@ bool RegAllocFast::isPhysRegFree(MCPhysReg PhysReg) const { /// This allocates space for the specified virtual register to be held on the /// stack. -int RegAllocFast::getStackSpaceFor(Register VirtReg) { +int RegAllocFastImpl::getStackSpaceFor(Register VirtReg) { // Find the location Reg would belong... int SS = StackSlotForVirtReg[VirtReg]; // Already has space allocated? @@ -464,7 +478,7 @@ static bool dominates(InstrPosIndexes &PosIndexes, const MachineInstr &A, } /// Returns false if \p VirtReg is known to not live out of the current block. -bool RegAllocFast::mayLiveOut(Register VirtReg) { +bool RegAllocFastImpl::mayLiveOut(Register VirtReg) { if (MayLiveAcrossBlocks.test(Register::virtReg2Index(VirtReg))) { // Cannot be live-out if there are no successors. return !MBB->succ_empty(); @@ -517,7 +531,7 @@ bool RegAllocFast::mayLiveOut(Register VirtReg) { } /// Returns false if \p VirtReg is known to not be live into the current block. -bool RegAllocFast::mayLiveIn(Register VirtReg) { +bool RegAllocFastImpl::mayLiveIn(Register VirtReg) { if (MayLiveAcrossBlocks.test(Register::virtReg2Index(VirtReg))) return !MBB->pred_empty(); @@ -536,8 +550,9 @@ bool RegAllocFast::mayLiveIn(Register VirtReg) { /// Insert spill instruction for \p AssignedReg before \p Before. Update /// DBG_VALUEs with \p VirtReg operands with the stack slot. -void RegAllocFast::spill(MachineBasicBlock::iterator Before, Register VirtReg, - MCPhysReg AssignedReg, bool Kill, bool LiveOut) { +void RegAllocFastImpl::spill(MachineBasicBlock::iterator Before, + Register VirtReg, MCPhysReg AssignedReg, bool Kill, + bool LiveOut) { LLVM_DEBUG(dbgs() << "Spilling " << printReg(VirtReg, TRI) << " in " << printReg(AssignedReg, TRI)); int FI = getStackSpaceFor(VirtReg); @@ -596,8 +611,8 @@ void RegAllocFast::spill(MachineBasicBlock::iterator Before, Register VirtReg, } /// Insert reload instruction for \p PhysReg before \p Before. -void RegAllocFast::reload(MachineBasicBlock::iterator Before, Register VirtReg, - MCPhysReg PhysReg) { +void RegAllocFastImpl::reload(MachineBasicBlock::iterator Before, + Register VirtReg, MCPhysReg PhysReg) { LLVM_DEBUG(dbgs() << "Reloading " << printReg(VirtReg, TRI) << " into " << printReg(PhysReg, TRI) << '\n'); int FI = getStackSpaceFor(VirtReg); @@ -610,7 +625,7 @@ void RegAllocFast::reload(MachineBasicBlock::iterator Before, Register VirtReg, /// This is not just MBB.begin() because surprisingly we have EH_LABEL /// instructions marking the begin of a basic block. This means we must insert /// new instructions after such labels... -MachineBasicBlock::iterator RegAllocFast::getMBBBeginInsertionPoint( +MachineBasicBlock::iterator RegAllocFastImpl::getMBBBeginInsertionPoint( MachineBasicBlock &MBB, SmallSet &PrologLiveIns) const { MachineBasicBlock::iterator I = MBB.begin(); while (I != MBB.end()) { @@ -637,7 +652,7 @@ MachineBasicBlock::iterator RegAllocFast::getMBBBeginInsertionPoint( } /// Reload all currently assigned virtual registers. -void RegAllocFast::reloadAtBegin(MachineBasicBlock &MBB) { +void RegAllocFastImpl::reloadAtBegin(MachineBasicBlock &MBB) { if (LiveVirtRegs.empty()) return; @@ -680,7 +695,7 @@ void RegAllocFast::reloadAtBegin(MachineBasicBlock &MBB) { /// Handle the direct use of a physical register. Check that the register is /// not used by a virtreg. Kill the physreg, marking it free. This may add /// implicit kills to MO->getParent() and invalidate MO. -bool RegAllocFast::usePhysReg(MachineInstr &MI, MCPhysReg Reg) { +bool RegAllocFastImpl::usePhysReg(MachineInstr &MI, MCPhysReg Reg) { assert(Register::isPhysicalRegister(Reg) && "expected physreg"); bool displacedAny = displacePhysReg(MI, Reg); setPhysRegState(Reg, regPreAssigned); @@ -688,7 +703,7 @@ bool RegAllocFast::usePhysReg(MachineInstr &MI, MCPhysReg Reg) { return displacedAny; } -bool RegAllocFast::definePhysReg(MachineInstr &MI, MCPhysReg Reg) { +bool RegAllocFastImpl::definePhysReg(MachineInstr &MI, MCPhysReg Reg) { bool displacedAny = displacePhysReg(MI, Reg); setPhysRegState(Reg, regPreAssigned); return displacedAny; @@ -697,7 +712,7 @@ bool RegAllocFast::definePhysReg(MachineInstr &MI, MCPhysReg Reg) { /// Mark PhysReg as reserved or free after spilling any virtregs. This is very /// similar to defineVirtReg except the physreg is reserved instead of /// allocated. -bool RegAllocFast::displacePhysReg(MachineInstr &MI, MCPhysReg PhysReg) { +bool RegAllocFastImpl::displacePhysReg(MachineInstr &MI, MCPhysReg PhysReg) { bool displacedAny = false; for (MCRegUnit Unit : TRI->regunits(PhysReg)) { @@ -726,7 +741,7 @@ bool RegAllocFast::displacePhysReg(MachineInstr &MI, MCPhysReg PhysReg) { return displacedAny; } -void RegAllocFast::freePhysReg(MCPhysReg PhysReg) { +void RegAllocFastImpl::freePhysReg(MCPhysReg PhysReg) { LLVM_DEBUG(dbgs() << "Freeing " << printReg(PhysReg, TRI) << ':'); MCRegister FirstUnit = *TRI->regunits(PhysReg).begin(); @@ -753,7 +768,7 @@ void RegAllocFast::freePhysReg(MCPhysReg PhysReg) { /// for allocation. Returns 0 when PhysReg is free or disabled with all aliases /// disabled - it can be allocated directly. /// \returns spillImpossible when PhysReg or an alias can't be spilled. -unsigned RegAllocFast::calcSpillCost(MCPhysReg PhysReg) const { +unsigned RegAllocFastImpl::calcSpillCost(MCPhysReg PhysReg) const { for (MCRegUnit Unit : TRI->regunits(PhysReg)) { switch (unsigned VirtReg = RegUnitStates[Unit]) { case regFree: @@ -772,8 +787,9 @@ unsigned RegAllocFast::calcSpillCost(MCPhysReg PhysReg) const { return 0; } -void RegAllocFast::assignDanglingDebugValues(MachineInstr &Definition, - Register VirtReg, MCPhysReg Reg) { +void RegAllocFastImpl::assignDanglingDebugValues(MachineInstr &Definition, + Register VirtReg, + MCPhysReg Reg) { auto UDBGValIter = DanglingDbgValues.find(VirtReg); if (UDBGValIter == DanglingDbgValues.end()) return; @@ -809,8 +825,8 @@ void RegAllocFast::assignDanglingDebugValues(MachineInstr &Definition, /// This method updates local state so that we know that PhysReg is the /// proper container for VirtReg now. The physical register must not be used /// for anything else when this is called. -void RegAllocFast::assignVirtToPhysReg(MachineInstr &AtMI, LiveReg &LR, - MCPhysReg PhysReg) { +void RegAllocFastImpl::assignVirtToPhysReg(MachineInstr &AtMI, LiveReg &LR, + MCPhysReg PhysReg) { Register VirtReg = LR.VirtReg; LLVM_DEBUG(dbgs() << "Assigning " << printReg(VirtReg, TRI) << " to " << printReg(PhysReg, TRI) << '\n'); @@ -824,7 +840,7 @@ void RegAllocFast::assignVirtToPhysReg(MachineInstr &AtMI, LiveReg &LR, static bool isCoalescable(const MachineInstr &MI) { return MI.isFullCopy(); } -Register RegAllocFast::traceCopyChain(Register Reg) const { +Register RegAllocFastImpl::traceCopyChain(Register Reg) const { static const unsigned ChainLengthLimit = 3; unsigned C = 0; do { @@ -843,7 +859,7 @@ Register RegAllocFast::traceCopyChain(Register Reg) const { /// Check if any of \p VirtReg's definitions is a copy. If it is follow the /// chain of copies to check whether we reach a physical register we can /// coalesce with. -Register RegAllocFast::traceCopies(Register VirtReg) const { +Register RegAllocFastImpl::traceCopies(Register VirtReg) const { static const unsigned DefLimit = 3; unsigned C = 0; for (const MachineInstr &MI : MRI->def_instructions(VirtReg)) { @@ -861,8 +877,8 @@ Register RegAllocFast::traceCopies(Register VirtReg) const { } /// Allocates a physical register for VirtReg. -void RegAllocFast::allocVirtReg(MachineInstr &MI, LiveReg &LR, Register Hint0, - bool LookAtPhysRegUses) { +void RegAllocFastImpl::allocVirtReg(MachineInstr &MI, LiveReg &LR, + Register Hint0, bool LookAtPhysRegUses) { const Register VirtReg = LR.VirtReg; assert(LR.PhysReg == 0); @@ -950,7 +966,7 @@ void RegAllocFast::allocVirtReg(MachineInstr &MI, LiveReg &LR, Register Hint0, assignVirtToPhysReg(MI, LR, BestReg); } -void RegAllocFast::allocVirtRegUndef(MachineOperand &MO) { +void RegAllocFastImpl::allocVirtRegUndef(MachineOperand &MO) { assert(MO.isUndef() && "expected undef use"); Register VirtReg = MO.getReg(); assert(VirtReg.isVirtual() && "Expected virtreg"); @@ -980,8 +996,9 @@ void RegAllocFast::allocVirtRegUndef(MachineOperand &MO) { /// Variation of defineVirtReg() with special handling for livethrough regs /// (tied or earlyclobber) that may interfere with preassigned uses. /// \return true if MI's MachineOperands were re-arranged/invalidated. -bool RegAllocFast::defineLiveThroughVirtReg(MachineInstr &MI, unsigned OpNum, - Register VirtReg) { +bool RegAllocFastImpl::defineLiveThroughVirtReg(MachineInstr &MI, + unsigned OpNum, + Register VirtReg) { if (!shouldAllocateRegister(VirtReg)) return false; LiveRegMap::iterator LRI = findLiveVirtReg(VirtReg); @@ -1016,8 +1033,8 @@ bool RegAllocFast::defineLiveThroughVirtReg(MachineInstr &MI, unsigned OpNum, /// - The value is live out and all uses are in different basic blocks. /// /// \return true if MI's MachineOperands were re-arranged/invalidated. -bool RegAllocFast::defineVirtReg(MachineInstr &MI, unsigned OpNum, - Register VirtReg, bool LookAtPhysRegUses) { +bool RegAllocFastImpl::defineVirtReg(MachineInstr &MI, unsigned OpNum, + Register VirtReg, bool LookAtPhysRegUses) { assert(VirtReg.isVirtual() && "Not a virtual register"); if (!shouldAllocateRegister(VirtReg)) return false; @@ -1094,8 +1111,8 @@ bool RegAllocFast::defineVirtReg(MachineInstr &MI, unsigned OpNum, /// Allocates a register for a VirtReg use. /// \return true if MI's MachineOperands were re-arranged/invalidated. -bool RegAllocFast::useVirtReg(MachineInstr &MI, MachineOperand &MO, - Register VirtReg) { +bool RegAllocFastImpl::useVirtReg(MachineInstr &MI, MachineOperand &MO, + Register VirtReg) { assert(VirtReg.isVirtual() && "Not a virtual register"); if (!shouldAllocateRegister(VirtReg)) return false; @@ -1150,8 +1167,8 @@ bool RegAllocFast::useVirtReg(MachineInstr &MI, MachineOperand &MO, /// Changes operand OpNum in MI the refer the PhysReg, considering subregs. /// \return true if MI's MachineOperands were re-arranged/invalidated. -bool RegAllocFast::setPhysReg(MachineInstr &MI, MachineOperand &MO, - MCPhysReg PhysReg) { +bool RegAllocFastImpl::setPhysReg(MachineInstr &MI, MachineOperand &MO, + MCPhysReg PhysReg) { if (!MO.getSubReg()) { MO.setReg(PhysReg); MO.setIsRenamable(true); @@ -1190,7 +1207,7 @@ bool RegAllocFast::setPhysReg(MachineInstr &MI, MachineOperand &MO, #ifndef NDEBUG -void RegAllocFast::dumpState() const { +void RegAllocFastImpl::dumpState() const { for (unsigned Unit = 1, UnitE = TRI->getNumRegUnits(); Unit != UnitE; ++Unit) { switch (unsigned VirtReg = RegUnitStates[Unit]) { @@ -1235,7 +1252,7 @@ void RegAllocFast::dumpState() const { #endif /// Count number of defs consumed from each register class by \p Reg -void RegAllocFast::addRegClassDefCounts( +void RegAllocFastImpl::addRegClassDefCounts( std::vector &RegClassDefCounts, Register Reg) const { assert(RegClassDefCounts.size() == TRI->getNumRegClasses()); @@ -1269,7 +1286,7 @@ void RegAllocFast::addRegClassDefCounts( /// Compute \ref DefOperandIndexes so it contains the indices of "def" operands /// that are to be allocated. Those are ordered in a way that small classes, /// early clobbers and livethroughs are allocated first. -void RegAllocFast::findAndSortDefOperandIndexes(const MachineInstr &MI) { +void RegAllocFastImpl::findAndSortDefOperandIndexes(const MachineInstr &MI) { DefOperandIndexes.clear(); // Track number of defs which may consume a register from the class. @@ -1343,7 +1360,7 @@ static bool isTiedToNotUndef(const MachineOperand &MO) { return !TiedMO.isUndef(); } -void RegAllocFast::allocateInstruction(MachineInstr &MI) { +void RegAllocFastImpl::allocateInstruction(MachineInstr &MI) { // The basic algorithm here is: // 1. Mark registers of def operands as free // 2. Allocate registers to use operands and place reload instructions for @@ -1605,7 +1622,7 @@ void RegAllocFast::allocateInstruction(MachineInstr &MI) { } } -void RegAllocFast::handleDebugValue(MachineInstr &MI) { +void RegAllocFastImpl::handleDebugValue(MachineInstr &MI) { // Ignore DBG_VALUEs that aren't based on virtual registers. These are // mostly constants and frame indices. assert(MI.isDebugValue() && "not a DBG_VALUE*"); @@ -1648,7 +1665,7 @@ void RegAllocFast::handleDebugValue(MachineInstr &MI) { } } -void RegAllocFast::handleBundle(MachineInstr &MI) { +void RegAllocFastImpl::handleBundle(MachineInstr &MI) { MachineBasicBlock::instr_iterator BundledMI = MI.getIterator(); ++BundledMI; while (BundledMI->isBundledWithPred()) { @@ -1671,7 +1688,7 @@ void RegAllocFast::handleBundle(MachineInstr &MI) { } } -void RegAllocFast::allocateBasicBlock(MachineBasicBlock &MBB) { +void RegAllocFastImpl::allocateBasicBlock(MachineBasicBlock &MBB) { this->MBB = &MBB; LLVM_DEBUG(dbgs() << "\nAllocating " << MBB); @@ -1732,7 +1749,7 @@ void RegAllocFast::allocateBasicBlock(MachineBasicBlock &MBB) { LLVM_DEBUG(MBB.dump()); } -bool RegAllocFast::runOnMachineFunction(MachineFunction &MF) { +bool RegAllocFastImpl::runOnMachineFunction(MachineFunction &MF) { LLVM_DEBUG(dbgs() << "********** FAST REGISTER ALLOCATION **********\n" << "********** Function: " << MF.getName() << '\n'); MRI = &MF.getRegInfo(); @@ -1771,6 +1788,36 @@ bool RegAllocFast::runOnMachineFunction(MachineFunction &MF) { return true; } +PreservedAnalyses RegAllocFastPass::run(MachineFunction &MF, + MachineFunctionAnalysisManager &) { + RegAllocFastImpl Impl(Opts.Filter, Opts.ClearVRegs); + bool Changed = Impl.runOnMachineFunction(MF); + if (!Changed) + return PreservedAnalyses::all(); + auto PA = getMachineFunctionPassPreservedAnalyses(); + PA.preserveSet(); + return PA; +} + +void RegAllocFastPass::printPipeline( + raw_ostream &OS, function_ref MapClassName2PassName) { + bool PrintFilterName = Opts.FilterName != "all"; + bool PrintNoClearVRegs = !Opts.ClearVRegs; + bool PrintSemicolon = PrintFilterName && PrintNoClearVRegs; + + OS << "regallocfast"; + if (PrintFilterName || PrintNoClearVRegs) { + OS << '<'; + if (PrintFilterName) + OS << "filter=" << Opts.FilterName; + if (PrintSemicolon) + OS << ';'; + if (PrintNoClearVRegs) + OS << "no-clear-vregs"; + OS << '>'; + } +} + FunctionPass *llvm::createFastRegisterAllocator() { return new RegAllocFast(); } FunctionPass *llvm::createFastRegisterAllocator(RegClassFilterFunc Ftor, diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 8dd060d0151a..2c56b04a1d9c 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -97,6 +97,7 @@ #include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PreISelIntrinsicLowering.h" +#include "llvm/CodeGen/RegAllocFast.h" #include "llvm/CodeGen/SafeStack.h" #include "llvm/CodeGen/SelectOptimize.h" #include "llvm/CodeGen/ShadowStackGCLowering.h" @@ -1163,6 +1164,38 @@ Expected> parseInternalizeGVs(StringRef Params) { return Expected>(std::move(PreservedGVs)); } +Expected +parseRegAllocFastPassOptions(PassBuilder &PB, StringRef Params) { + RegAllocFastPassOptions Opts; + while (!Params.empty()) { + StringRef ParamName; + std::tie(ParamName, Params) = Params.split(';'); + + if (ParamName.consume_front("filter=")) { + RegClassFilterFunc Filter = PB.parseRegAllocFilter(ParamName); + if (!Filter) { + return make_error( + formatv("invalid regallocfast register filter '{0}' ", ParamName) + .str(), + inconvertibleErrorCode()); + } + Opts.Filter = Filter; + Opts.FilterName = ParamName; + continue; + } + + if (ParamName == "no-clear-vregs") { + Opts.ClearVRegs = false; + continue; + } + + return make_error( + formatv("invalid regallocfast pass parameter '{0}' ", ParamName).str(), + inconvertibleErrorCode()); + } + return Opts; +} + } // namespace /// Tests whether a pass name starts with a valid prefix for a default pipeline @@ -1296,6 +1329,11 @@ static bool isMachineFunctionPassName(StringRef Name, CallbacksT &Callbacks) { #define MACHINE_FUNCTION_PASS(NAME, CREATE_PASS) \ if (Name == NAME) \ return true; +#define MACHINE_FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, \ + PARAMS) \ + if (PassBuilder::checkParametrizedPassName(Name, NAME)) \ + return true; + #define MACHINE_FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ return true; @@ -1925,6 +1963,15 @@ Error PassBuilder::parseMachinePass(MachineFunctionPassManager &MFPM, MFPM.addPass(CREATE_PASS); \ return Error::success(); \ } +#define MACHINE_FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, \ + PARAMS) \ + if (checkParametrizedPassName(Name, NAME)) { \ + auto Params = parsePassParameters(PARSER, Name, NAME); \ + if (!Params) \ + return Params.takeError(); \ + MFPM.addPass(CREATE_PASS(Params.get())); \ + return Error::success(); \ + } #include "llvm/Passes/MachinePassRegistry.def" for (auto &C : MachineFunctionPipelineParsingCallbacks) @@ -2172,6 +2219,15 @@ Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) { return Error::success(); } +RegClassFilterFunc PassBuilder::parseRegAllocFilter(StringRef FilterName) { + if (FilterName == "all") + return allocateAllRegClasses; + for (auto &C : RegClassFilterParsingCallbacks) + if (auto F = C(FilterName)) + return F; + return nullptr; +} + static void printPassName(StringRef PassName, raw_ostream &OS) { OS << " " << PassName << "\n"; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index 17c961578382..ce997c659094 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -740,6 +740,15 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks( if (EnableLowerModuleLDS) PM.addPass(AMDGPULowerModuleLDSPass(*this)); }); + + PB.registerRegClassFilterParsingCallback( + [](StringRef FilterName) -> RegClassFilterFunc { + if (FilterName == "sgpr") + return onlyAllocateSGPRs; + if (FilterName == "vgpr") + return onlyAllocateVGPRs; + return nullptr; + }); } int64_t AMDGPUTargetMachine::getNullPointerValue(unsigned AddrSpace) { diff --git a/llvm/test/CodeGen/AArch64/fast-regalloc-empty-bb-with-liveins.mir b/llvm/test/CodeGen/AArch64/fast-regalloc-empty-bb-with-liveins.mir index f620cd22901c..ff5ac24f713d 100644 --- a/llvm/test/CodeGen/AArch64/fast-regalloc-empty-bb-with-liveins.mir +++ b/llvm/test/CodeGen/AArch64/fast-regalloc-empty-bb-with-liveins.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple aarch64-apple-ios -run-pass regallocfast -o - %s | FileCheck %s +# RUN: llc -mtriple aarch64-apple-ios -passes=regallocfast -o - %s | FileCheck %s # This test used to crash the fast register alloc. # Basically, when a basic block has liveins, the fast regalloc # was deferencing the begin iterator of this block. However, diff --git a/llvm/test/CodeGen/AMDGPU/fast-ra-kills-vcc.mir b/llvm/test/CodeGen/AMDGPU/fast-ra-kills-vcc.mir index 0fa6577b0dd0..01cd5f74dcb4 100644 --- a/llvm/test/CodeGen/AMDGPU/fast-ra-kills-vcc.mir +++ b/llvm/test/CodeGen/AMDGPU/fast-ra-kills-vcc.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=regallocfast -o - %s | FileCheck %s +# RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=regallocfast -o - %s | FileCheck %s # Make sure incorrect kills aren't emitted on vcc diff --git a/llvm/test/CodeGen/AMDGPU/fast-regalloc-bundles.mir b/llvm/test/CodeGen/AMDGPU/fast-regalloc-bundles.mir index ea03f2c57ea5..8c969eb0ef9b 100644 --- a/llvm/test/CodeGen/AMDGPU/fast-regalloc-bundles.mir +++ b/llvm/test/CodeGen/AMDGPU/fast-regalloc-bundles.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=amdgcn -mcpu=gfx902 -verify-machineinstrs -run-pass=regallocfast %s -o - | FileCheck --check-prefix=GCN %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx902 -verify-machineinstrs -passes=regallocfast %s -o - | FileCheck --check-prefix=GCN %s --- name: fast_regalloc_bundle_handling diff --git a/llvm/test/CodeGen/AMDGPU/fastregalloc-illegal-subreg-physreg.mir b/llvm/test/CodeGen/AMDGPU/fastregalloc-illegal-subreg-physreg.mir index 7642775aafda..6a2a2b0c3939 100644 --- a/llvm/test/CodeGen/AMDGPU/fastregalloc-illegal-subreg-physreg.mir +++ b/llvm/test/CodeGen/AMDGPU/fastregalloc-illegal-subreg-physreg.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=regallocfast -o - %s | FileCheck %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -passes=regallocfast -o - %s | FileCheck %s # This would hit "Illegal subregister index for physical register" verifier error since # tied operands would skip dropping the subregister index. diff --git a/llvm/test/CodeGen/AMDGPU/fastregalloc-self-loop-heuristic.mir b/llvm/test/CodeGen/AMDGPU/fastregalloc-self-loop-heuristic.mir index 24b82f4862af..918242e9f996 100644 --- a/llvm/test/CodeGen/AMDGPU/fastregalloc-self-loop-heuristic.mir +++ b/llvm/test/CodeGen/AMDGPU/fastregalloc-self-loop-heuristic.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=regallocfast -o - %s | FileCheck -check-prefix=GCN %s +# RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=regallocfast -o - %s | FileCheck -check-prefix=GCN %s --- name: self_loop_single_def_use diff --git a/llvm/test/CodeGen/AMDGPU/spill-agpr.mir b/llvm/test/CodeGen/AMDGPU/spill-agpr.mir index 981a853cb097..8e6da4bf92ee 100644 --- a/llvm/test/CodeGen/AMDGPU/spill-agpr.mir +++ b/llvm/test/CodeGen/AMDGPU/spill-agpr.mir @@ -1,7 +1,9 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regallocfast -o - %s | FileCheck -check-prefix=GFX908-SPILLED %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -passes=regallocfast -o - %s | FileCheck -check-prefix=GFX908-SPILLED %s # RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regallocfast,prologepilog -o - %s | FileCheck -check-prefix=GFX908-EXPANDED %s # RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=regallocfast -o - %s | FileCheck -check-prefix=GFX90A-SPILLED %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -passes=regallocfast -o - %s | FileCheck -check-prefix=GFX90A-SPILLED %s # RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=regallocfast,prologepilog -o - %s | FileCheck -check-prefix=GFX90A-EXPANDED %s --- diff --git a/llvm/test/CodeGen/AMDGPU/spill192.mir b/llvm/test/CodeGen/AMDGPU/spill192.mir index 56882cd83106..42c792b1538a 100644 --- a/llvm/test/CodeGen/AMDGPU/spill192.mir +++ b/llvm/test/CodeGen/AMDGPU/spill192.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s +# RUN: llc -mtriple=amdgcn -mcpu=tahiti -passes=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast,si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=EXPANDED %s # Make sure spill/restore of 192 bit registers works. We have to diff --git a/llvm/test/CodeGen/AMDGPU/spill224.mir b/llvm/test/CodeGen/AMDGPU/spill224.mir index 4525fc1cb70e..5e53f93df95f 100644 --- a/llvm/test/CodeGen/AMDGPU/spill224.mir +++ b/llvm/test/CodeGen/AMDGPU/spill224.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s +# RUN: llc -mtriple=amdgcn -mcpu=tahiti -passes=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast,si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=EXPANDED %s # Make sure spill/restore of 224 bit registers works. diff --git a/llvm/test/CodeGen/AMDGPU/spill288.mir b/llvm/test/CodeGen/AMDGPU/spill288.mir index 173056e6b913..3d5404a9c1ad 100644 --- a/llvm/test/CodeGen/AMDGPU/spill288.mir +++ b/llvm/test/CodeGen/AMDGPU/spill288.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s +# RUN: llc -mtriple=amdgcn -mcpu=tahiti -passes=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast,si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=EXPANDED %s # Make sure spill/restore of 288 bit registers works. diff --git a/llvm/test/CodeGen/AMDGPU/spill320.mir b/llvm/test/CodeGen/AMDGPU/spill320.mir index 828f524bb067..4473a4d6648e 100644 --- a/llvm/test/CodeGen/AMDGPU/spill320.mir +++ b/llvm/test/CodeGen/AMDGPU/spill320.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s +# RUN: llc -mtriple=amdgcn -mcpu=tahiti -passes=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast,si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=EXPANDED %s # Make sure spill/restore of 320 bit registers works. diff --git a/llvm/test/CodeGen/AMDGPU/spill352.mir b/llvm/test/CodeGen/AMDGPU/spill352.mir index ef620fae5b10..8fa053a908b6 100644 --- a/llvm/test/CodeGen/AMDGPU/spill352.mir +++ b/llvm/test/CodeGen/AMDGPU/spill352.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s +# RUN: llc -mtriple=amdgcn -mcpu=tahiti -passes=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast,si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=EXPANDED %s # Make sure spill/restore of 352 bit registers works. diff --git a/llvm/test/CodeGen/AMDGPU/spill384.mir b/llvm/test/CodeGen/AMDGPU/spill384.mir index 37cf5d0fc83c..cd604e4483b9 100644 --- a/llvm/test/CodeGen/AMDGPU/spill384.mir +++ b/llvm/test/CodeGen/AMDGPU/spill384.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s +# RUN: llc -mtriple=amdgcn -mcpu=tahiti -passes=regallocfast -o - %s | FileCheck -check-prefix=SPILLED %s # RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=regallocfast,si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=EXPANDED %s # Make sure spill/restore of 384 bit registers works. diff --git a/llvm/test/CodeGen/AMDGPU/unexpected-reg-unit-state.mir b/llvm/test/CodeGen/AMDGPU/unexpected-reg-unit-state.mir index 1652f3133474..0f0dd2374156 100644 --- a/llvm/test/CodeGen/AMDGPU/unexpected-reg-unit-state.mir +++ b/llvm/test/CodeGen/AMDGPU/unexpected-reg-unit-state.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=regallocfast -o - %s | FileCheck %s +# RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=regallocfast -o - %s | FileCheck %s --- name: bar diff --git a/llvm/test/CodeGen/ARM/regalloc-fast-rewrite-implicits.mir b/llvm/test/CodeGen/ARM/regalloc-fast-rewrite-implicits.mir index b7a007f7e3ff..7de911486ddf 100644 --- a/llvm/test/CodeGen/ARM/regalloc-fast-rewrite-implicits.mir +++ b/llvm/test/CodeGen/ARM/regalloc-fast-rewrite-implicits.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=armv7-apple-ios -run-pass=regallocfast -o - %s | FileCheck %s +# RUN: llc -mtriple=armv7-apple-ios -passes=regallocfast -o - %s | FileCheck %s # tBX_RET uses an implicit vreg with a sub-register. That implicit use will diff --git a/llvm/test/CodeGen/MIR/Generic/runPass.mir b/llvm/test/CodeGen/MIR/Generic/runPass.mir index 54c1dd221bdb..75763c5389b0 100644 --- a/llvm/test/CodeGen/MIR/Generic/runPass.mir +++ b/llvm/test/CodeGen/MIR/Generic/runPass.mir @@ -1,6 +1,7 @@ # RUN: llc -run-pass=greedy -debug-pass=Arguments -o - %s | FileCheck %s # RUN: llc -run-pass=regallocbasic -debug-pass=Arguments -o - %s | FileCheck %s # RUN: llc -run-pass=regallocfast -debug-pass=Arguments -o - %s | FileCheck %s +# RUN: llc -passes=regallocfast -o - %s | FileCheck %s # Check that passes are initialized correctly, so that it's possible to # use -run-pass. diff --git a/llvm/test/CodeGen/PowerPC/spill-nor0.mir b/llvm/test/CodeGen/PowerPC/spill-nor0.mir index 2f50ff3701d1..6658ca156d4d 100644 --- a/llvm/test/CodeGen/PowerPC/spill-nor0.mir +++ b/llvm/test/CodeGen/PowerPC/spill-nor0.mir @@ -1,4 +1,5 @@ # RUN: llc -o - %s -mtriple=powerpc64-- -run-pass=regallocfast | FileCheck %s +# RUN: llc -o - %s -mtriple=powerpc64-- -passes=regallocfast | FileCheck %s --- # CHECK-LABEL: name: func name: func diff --git a/llvm/test/CodeGen/SystemZ/regalloc-fast-invalid-kill-flag.mir b/llvm/test/CodeGen/SystemZ/regalloc-fast-invalid-kill-flag.mir index bcd7f51cdf78..917cc776e3ef 100644 --- a/llvm/test/CodeGen/SystemZ/regalloc-fast-invalid-kill-flag.mir +++ b/llvm/test/CodeGen/SystemZ/regalloc-fast-invalid-kill-flag.mir @@ -1,4 +1,5 @@ # RUN: llc -verify-machineinstrs -run-pass regallocfast -mtriple s390x-ibm-linux -o - %s | FileCheck %s +# RUN: llc -verify-machineinstrs -passes=regallocfast -mtriple s390x-ibm-linux -o - %s | FileCheck %s --- | @g_167 = external global [5 x i64], align 8 diff --git a/llvm/test/CodeGen/Thumb/high-reg-clobber.mir b/llvm/test/CodeGen/Thumb/high-reg-clobber.mir index ab73f1448d71..1402c7c2cbca 100644 --- a/llvm/test/CodeGen/Thumb/high-reg-clobber.mir +++ b/llvm/test/CodeGen/Thumb/high-reg-clobber.mir @@ -2,6 +2,7 @@ # RUN: llc -mtriple thumbv6m-arm-none-eabi -run-pass regallocbasic %s -o - | FileCheck %s # RUN: llc -mtriple thumbv6m-arm-none-eabi -run-pass greedy %s -o - | FileCheck %s # RUN: llc -mtriple thumbv6m-arm-none-eabi -run-pass regallocfast %s -o - | FileCheck %s --check-prefix=FAST +# RUN: llc -mtriple thumbv6m-arm-none-eabi -passes=regallocfast %s -o - | FileCheck %s --check-prefix=FAST ... --- diff --git a/llvm/test/CodeGen/Thumb2/high-reg-spill.mir b/llvm/test/CodeGen/Thumb2/high-reg-spill.mir index 1cf58da63fe8..8c92020fb051 100644 --- a/llvm/test/CodeGen/Thumb2/high-reg-spill.mir +++ b/llvm/test/CodeGen/Thumb2/high-reg-spill.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -run-pass regallocfast %s -o - | FileCheck %s +# RUN: llc -passes=regallocfast %s -o - | FileCheck %s # This test examines register allocation and spilling with Fast Register # Allocator. The test uses inline assembler that requests an input variable to diff --git a/llvm/test/CodeGen/X86/bug47278-eflags-error.mir b/llvm/test/CodeGen/X86/bug47278-eflags-error.mir index e4e68451850b..65d368ee224d 100644 --- a/llvm/test/CodeGen/X86/bug47278-eflags-error.mir +++ b/llvm/test/CodeGen/X86/bug47278-eflags-error.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=i386-unknown-linux-musl -verify-machineinstrs -run-pass=regallocfast -o - %s | FileCheck %s +# RUN: llc -mtriple=i386-unknown-linux-musl -verify-machineinstrs -passes=regallocfast -o - %s | FileCheck %s # Test for correct management of allocatable and non-allocatable # live-ins in fastregalloc diff --git a/llvm/test/CodeGen/X86/bug47278.mir b/llvm/test/CodeGen/X86/bug47278.mir index 0b01c9de19c0..ea7bb0961698 100644 --- a/llvm/test/CodeGen/X86/bug47278.mir +++ b/llvm/test/CodeGen/X86/bug47278.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=i386-unknown-linux-musl -verify-machineinstrs -run-pass=regallocfast -o - %s | FileCheck %s +# RUN: llc -mtriple=i386-unknown-linux-musl -verify-machineinstrs -passes=regallocfast -o - %s | FileCheck %s # Make sure this case doesn't assert or try to assign $ecx to %1 on # SHRD32rrCL diff --git a/llvm/test/CodeGen/X86/callbr-asm-outputs-regallocfast.mir b/llvm/test/CodeGen/X86/callbr-asm-outputs-regallocfast.mir index 6417538b269c..6263a3f09b0d 100644 --- a/llvm/test/CodeGen/X86/callbr-asm-outputs-regallocfast.mir +++ b/llvm/test/CodeGen/X86/callbr-asm-outputs-regallocfast.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -run-pass=regallocfast -verify-machineinstrs %s -o - | FileCheck %s +# RUN: llc -passes=regallocfast -verify-machineinstrs %s -o - | FileCheck %s --- | ; ModuleID = 'x.c' source_filename = "x.c" diff --git a/llvm/test/CodeGen/X86/fastregalloc-selfloop.mir b/llvm/test/CodeGen/X86/fastregalloc-selfloop.mir index d60632a98fb0..1872e703d6cd 100644 --- a/llvm/test/CodeGen/X86/fastregalloc-selfloop.mir +++ b/llvm/test/CodeGen/X86/fastregalloc-selfloop.mir @@ -1,4 +1,5 @@ # RUN: llc -mtriple=x86_64-- -run-pass=regallocfast -o - %s | FileCheck %s +# RUN: llc -mtriple=x86_64-- -passes=regallocfast -o - %s | FileCheck %s ... --- diff --git a/llvm/test/CodeGen/X86/fastregalloc-tied-undef.mir b/llvm/test/CodeGen/X86/fastregalloc-tied-undef.mir index 8a6311eabaca..18de9c11439c 100644 --- a/llvm/test/CodeGen/X86/fastregalloc-tied-undef.mir +++ b/llvm/test/CodeGen/X86/fastregalloc-tied-undef.mir @@ -1,4 +1,5 @@ # RUN: llc -mtriple=x86_64-- -run-pass=regallocfast -o - %s | FileCheck %s +# RUN: llc -mtriple=x86_64-- -passes=regallocfast -o - %s | FileCheck %s # If the tied use is undef value, fastregalloc should free the def register. # There is no reload needed for the undef value. diff --git a/llvm/test/CodeGen/X86/regalloc-fast-missing-live-out-spill.mir b/llvm/test/CodeGen/X86/regalloc-fast-missing-live-out-spill.mir index 6a2eae9dbbc4..143b6864b119 100644 --- a/llvm/test/CodeGen/X86/regalloc-fast-missing-live-out-spill.mir +++ b/llvm/test/CodeGen/X86/regalloc-fast-missing-live-out-spill.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=x86_64-grtev4-linux-gnu -run-pass=regallocfast -o - %s | FileCheck %s +# RUN: llc -mtriple=x86_64-grtev4-linux-gnu -passes=regallocfast -o - %s | FileCheck %s # Bug 41973. Make sure %12 is detected as live out of %bb.0, even # though the use is allocated before the def block %bb.3. Previously diff --git a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir index 8bac14018a7d..f683e9ebec81 100644 --- a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir +++ b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir @@ -1,4 +1,5 @@ # RUN: llc -mtriple=x86_64-- -run-pass=regallocfast -o - %s | FileCheck %s +# RUN: llc -mtriple=x86_64-- -passes=regallocfast -o - %s | FileCheck %s # Check that fastregalloc does not displace register assigned to tied def when # RegMask operand is present. STATEPOINT is an example of such instruction. diff --git a/llvm/test/CodeGen/X86/virtreg-physreg-def-regallocfast.mir b/llvm/test/CodeGen/X86/virtreg-physreg-def-regallocfast.mir index 6cdac85f96fe..5222b3d9fdb8 100644 --- a/llvm/test/CodeGen/X86/virtreg-physreg-def-regallocfast.mir +++ b/llvm/test/CodeGen/X86/virtreg-physreg-def-regallocfast.mir @@ -1,4 +1,5 @@ # RUN: llc -o - -mtriple=x86_64-- -run-pass=regallocfast %s | FileCheck %s +# RUN: llc -o - -mtriple=x86_64-- -passes=regallocfast %s | FileCheck %s # Fast regalloc used to not collect physical register definitions # before walking and assigning the virtual definition. # Therefore it was possible for a virtual definition to end up diff --git a/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir b/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir new file mode 100644 index 000000000000..872c35237c20 --- /dev/null +++ b/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir @@ -0,0 +1,10 @@ +# REQUIRES: amdgpu-registered-target +# RUN: llc -mtriple=amdgcn --passes='regallocfast' --print-pipeline-passes %s | FileCheck %s --check-prefix=PASS +# RUN: not llc -mtriple=amdgcn --passes='regallocfast' --print-pipeline-passes %s 2>&1 | FileCheck %s --check-prefix=BAD-FILTER + +# PASS: regallocfast +# BAD-FILTER: invalid regallocfast register filter 'bad-filter' + +--- +name: f +... -- GitLab From 566befca28f5df9490ad4fcce31042d92526ba1e Mon Sep 17 00:00:00 2001 From: WANG Rui Date: Fri, 7 Jun 2024 11:23:08 +0800 Subject: [PATCH 188/462] [NFC][LoongArch] Pre-commit tests for hoisting of PseudoLA* --- .../LoongArch/machinelicm-address-pseudos.ll | 458 ++++++++++++++++++ 1 file changed, 458 insertions(+) create mode 100644 llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll diff --git a/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll b/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll new file mode 100644 index 000000000000..30a19adcfdce --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll @@ -0,0 +1,458 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch32 --relocation-model=pic \ +; RUN: --verify-machineinstrs < %s | FileCheck -check-prefixes=LA32 %s +; RUN: llc --mtriple=loongarch64 --relocation-model=pic \ +; RUN: --verify-machineinstrs < %s | FileCheck -check-prefixes=LA64 %s +; RUN: llc --mtriple=loongarch64 --relocation-model=pic --code-model=large \ +; RUN: --verify-machineinstrs < %s | FileCheck -check-prefixes=LA64LARGE %s + +; Verifies that MachineLICM can hoist address generation pseudos out of loops. + +@l = protected global i32 0, align 4 + +define void @test_la_pcrel(i32 signext %n) { +; LA32-LABEL: test_la_pcrel: +; LA32: # %bb.0: # %entry +; LA32-NEXT: move $a1, $zero +; LA32-NEXT: pcalau12i $a2, %pc_hi20(l) +; LA32-NEXT: addi.w $a2, $a2, %pc_lo12(l) +; LA32-NEXT: .p2align 4, , 16 +; LA32-NEXT: .LBB0_1: # %loop +; LA32-NEXT: # =>This Inner Loop Header: Depth=1 +; LA32-NEXT: ld.w $a3, $a2, 0 +; LA32-NEXT: addi.w $a1, $a1, 1 +; LA32-NEXT: blt $a1, $a0, .LBB0_1 +; LA32-NEXT: # %bb.2: # %ret +; LA32-NEXT: ret +; +; LA64-LABEL: test_la_pcrel: +; LA64: # %bb.0: # %entry +; LA64-NEXT: move $a1, $zero +; LA64-NEXT: pcalau12i $a2, %pc_hi20(l) +; LA64-NEXT: addi.d $a2, $a2, %pc_lo12(l) +; LA64-NEXT: .p2align 4, , 16 +; LA64-NEXT: .LBB0_1: # %loop +; LA64-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64-NEXT: ld.w $a3, $a2, 0 +; LA64-NEXT: addi.w $a1, $a1, 1 +; LA64-NEXT: blt $a1, $a0, .LBB0_1 +; LA64-NEXT: # %bb.2: # %ret +; LA64-NEXT: ret +; +; LA64LARGE-LABEL: test_la_pcrel: +; LA64LARGE: # %bb.0: # %entry +; LA64LARGE-NEXT: move $a1, $zero +; LA64LARGE-NEXT: pcalau12i $a2, %pc_hi20(l) +; LA64LARGE-NEXT: addi.d $t8, $zero, %pc_lo12(l) +; LA64LARGE-NEXT: lu32i.d $t8, %pc64_lo20(l) +; LA64LARGE-NEXT: lu52i.d $t8, $t8, %pc64_hi12(l) +; LA64LARGE-NEXT: add.d $a2, $t8, $a2 +; LA64LARGE-NEXT: .p2align 4, , 16 +; LA64LARGE-NEXT: .LBB0_1: # %loop +; LA64LARGE-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64LARGE-NEXT: ld.w $a3, $a2, 0 +; LA64LARGE-NEXT: addi.w $a1, $a1, 1 +; LA64LARGE-NEXT: blt $a1, $a0, .LBB0_1 +; LA64LARGE-NEXT: # %bb.2: # %ret +; LA64LARGE-NEXT: ret +entry: + br label %loop + +loop: + %i = phi i32 [ %inc, %loop ], [ 0, %entry ] + %0 = load volatile i32, ptr @l, align 4 + %inc = add nuw nsw i32 %i, 1 + %cmp = icmp slt i32 %inc, %n + br i1 %cmp, label %loop, label %ret + +ret: + ret void +} + +@g = global i32 0, align 4 + +define void @test_la_got(i32 signext %n) { +; LA32-LABEL: test_la_got: +; LA32: # %bb.0: # %entry +; LA32-NEXT: move $a1, $zero +; LA32-NEXT: .p2align 4, , 16 +; LA32-NEXT: .LBB1_1: # %loop +; LA32-NEXT: # =>This Inner Loop Header: Depth=1 +; LA32-NEXT: pcalau12i $a2, %got_pc_hi20(g) +; LA32-NEXT: ld.w $a2, $a2, %got_pc_lo12(g) +; LA32-NEXT: ld.w $a2, $a2, 0 +; LA32-NEXT: addi.w $a1, $a1, 1 +; LA32-NEXT: blt $a1, $a0, .LBB1_1 +; LA32-NEXT: # %bb.2: # %ret +; LA32-NEXT: ret +; +; LA64-LABEL: test_la_got: +; LA64: # %bb.0: # %entry +; LA64-NEXT: move $a1, $zero +; LA64-NEXT: .p2align 4, , 16 +; LA64-NEXT: .LBB1_1: # %loop +; LA64-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64-NEXT: pcalau12i $a2, %got_pc_hi20(g) +; LA64-NEXT: ld.d $a2, $a2, %got_pc_lo12(g) +; LA64-NEXT: ld.w $a2, $a2, 0 +; LA64-NEXT: addi.w $a1, $a1, 1 +; LA64-NEXT: blt $a1, $a0, .LBB1_1 +; LA64-NEXT: # %bb.2: # %ret +; LA64-NEXT: ret +; +; LA64LARGE-LABEL: test_la_got: +; LA64LARGE: # %bb.0: # %entry +; LA64LARGE-NEXT: move $a1, $zero +; LA64LARGE-NEXT: .p2align 4, , 16 +; LA64LARGE-NEXT: .LBB1_1: # %loop +; LA64LARGE-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64LARGE-NEXT: pcalau12i $a2, %got_pc_hi20(g) +; LA64LARGE-NEXT: addi.d $t8, $zero, %got_pc_lo12(g) +; LA64LARGE-NEXT: lu32i.d $t8, %got64_pc_lo20(g) +; LA64LARGE-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(g) +; LA64LARGE-NEXT: ldx.d $a2, $t8, $a2 +; LA64LARGE-NEXT: ld.w $a2, $a2, 0 +; LA64LARGE-NEXT: addi.w $a1, $a1, 1 +; LA64LARGE-NEXT: blt $a1, $a0, .LBB1_1 +; LA64LARGE-NEXT: # %bb.2: # %ret +; LA64LARGE-NEXT: ret +entry: + br label %loop + +loop: + %i = phi i32 [ %inc, %loop ], [ 0, %entry ] + %0 = load volatile i32, ptr @g, align 4 + %inc = add nuw nsw i32 %i, 1 + %cmp = icmp slt i32 %inc, %n + br i1 %cmp, label %loop, label %ret + +ret: + ret void +} + +@ie = external thread_local(initialexec) global i32 + +define void @test_la_tls_ie(i32 signext %n) { +; LA32-LABEL: test_la_tls_ie: +; LA32: # %bb.0: # %entry +; LA32-NEXT: move $a1, $zero +; LA32-NEXT: .p2align 4, , 16 +; LA32-NEXT: .LBB2_1: # %loop +; LA32-NEXT: # =>This Inner Loop Header: Depth=1 +; LA32-NEXT: pcalau12i $a2, %ie_pc_hi20(ie) +; LA32-NEXT: ld.w $a2, $a2, %ie_pc_lo12(ie) +; LA32-NEXT: add.w $a2, $a2, $tp +; LA32-NEXT: ld.w $a2, $a2, 0 +; LA32-NEXT: addi.w $a1, $a1, 1 +; LA32-NEXT: blt $a1, $a0, .LBB2_1 +; LA32-NEXT: # %bb.2: # %ret +; LA32-NEXT: ret +; +; LA64-LABEL: test_la_tls_ie: +; LA64: # %bb.0: # %entry +; LA64-NEXT: move $a1, $zero +; LA64-NEXT: .p2align 4, , 16 +; LA64-NEXT: .LBB2_1: # %loop +; LA64-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64-NEXT: pcalau12i $a2, %ie_pc_hi20(ie) +; LA64-NEXT: ld.d $a2, $a2, %ie_pc_lo12(ie) +; LA64-NEXT: ldx.w $a2, $a2, $tp +; LA64-NEXT: addi.w $a1, $a1, 1 +; LA64-NEXT: blt $a1, $a0, .LBB2_1 +; LA64-NEXT: # %bb.2: # %ret +; LA64-NEXT: ret +; +; LA64LARGE-LABEL: test_la_tls_ie: +; LA64LARGE: # %bb.0: # %entry +; LA64LARGE-NEXT: move $a1, $zero +; LA64LARGE-NEXT: .p2align 4, , 16 +; LA64LARGE-NEXT: .LBB2_1: # %loop +; LA64LARGE-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64LARGE-NEXT: pcalau12i $a2, %ie_pc_hi20(ie) +; LA64LARGE-NEXT: addi.d $t8, $zero, %ie_pc_lo12(ie) +; LA64LARGE-NEXT: lu32i.d $t8, %ie64_pc_lo20(ie) +; LA64LARGE-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(ie) +; LA64LARGE-NEXT: ldx.d $a2, $t8, $a2 +; LA64LARGE-NEXT: ldx.w $a2, $a2, $tp +; LA64LARGE-NEXT: addi.w $a1, $a1, 1 +; LA64LARGE-NEXT: blt $a1, $a0, .LBB2_1 +; LA64LARGE-NEXT: # %bb.2: # %ret +; LA64LARGE-NEXT: ret +entry: + br label %loop + +loop: + %i = phi i32 [ %inc, %loop ], [ 0, %entry ] + %0 = load volatile i32, ptr @ie, align 4 + %inc = add nuw nsw i32 %i, 1 + %cmp = icmp slt i32 %inc, %n + br i1 %cmp, label %loop, label %ret + +ret: + ret void +} + +@ld = external thread_local(localdynamic) global i32 + +define void @test_la_tls_ld(i32 signext %n) { +; LA32-LABEL: test_la_tls_ld: +; LA32: # %bb.0: # %entry +; LA32-NEXT: addi.w $sp, $sp, -16 +; LA32-NEXT: .cfi_def_cfa_offset 16 +; LA32-NEXT: st.w $ra, $sp, 12 # 4-byte Folded Spill +; LA32-NEXT: st.w $fp, $sp, 8 # 4-byte Folded Spill +; LA32-NEXT: st.w $s0, $sp, 4 # 4-byte Folded Spill +; LA32-NEXT: .cfi_offset 1, -4 +; LA32-NEXT: .cfi_offset 22, -8 +; LA32-NEXT: .cfi_offset 23, -12 +; LA32-NEXT: move $fp, $a0 +; LA32-NEXT: move $s0, $zero +; LA32-NEXT: .p2align 4, , 16 +; LA32-NEXT: .LBB3_1: # %loop +; LA32-NEXT: # =>This Inner Loop Header: Depth=1 +; LA32-NEXT: pcalau12i $a0, %ld_pc_hi20(ld) +; LA32-NEXT: addi.w $a0, $a0, %got_pc_lo12(ld) +; LA32-NEXT: bl %plt(__tls_get_addr) +; LA32-NEXT: ld.w $a0, $a0, 0 +; LA32-NEXT: addi.w $s0, $s0, 1 +; LA32-NEXT: blt $s0, $fp, .LBB3_1 +; LA32-NEXT: # %bb.2: # %ret +; LA32-NEXT: ld.w $s0, $sp, 4 # 4-byte Folded Reload +; LA32-NEXT: ld.w $fp, $sp, 8 # 4-byte Folded Reload +; LA32-NEXT: ld.w $ra, $sp, 12 # 4-byte Folded Reload +; LA32-NEXT: addi.w $sp, $sp, 16 +; LA32-NEXT: ret +; +; LA64-LABEL: test_la_tls_ld: +; LA64: # %bb.0: # %entry +; LA64-NEXT: addi.d $sp, $sp, -32 +; LA64-NEXT: .cfi_def_cfa_offset 32 +; LA64-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill +; LA64-NEXT: st.d $fp, $sp, 16 # 8-byte Folded Spill +; LA64-NEXT: st.d $s0, $sp, 8 # 8-byte Folded Spill +; LA64-NEXT: .cfi_offset 1, -8 +; LA64-NEXT: .cfi_offset 22, -16 +; LA64-NEXT: .cfi_offset 23, -24 +; LA64-NEXT: move $fp, $a0 +; LA64-NEXT: move $s0, $zero +; LA64-NEXT: .p2align 4, , 16 +; LA64-NEXT: .LBB3_1: # %loop +; LA64-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64-NEXT: pcalau12i $a0, %ld_pc_hi20(ld) +; LA64-NEXT: addi.d $a0, $a0, %got_pc_lo12(ld) +; LA64-NEXT: bl %plt(__tls_get_addr) +; LA64-NEXT: ld.w $a0, $a0, 0 +; LA64-NEXT: addi.w $s0, $s0, 1 +; LA64-NEXT: blt $s0, $fp, .LBB3_1 +; LA64-NEXT: # %bb.2: # %ret +; LA64-NEXT: ld.d $s0, $sp, 8 # 8-byte Folded Reload +; LA64-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload +; LA64-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload +; LA64-NEXT: addi.d $sp, $sp, 32 +; LA64-NEXT: ret +; +; LA64LARGE-LABEL: test_la_tls_ld: +; LA64LARGE: # %bb.0: # %entry +; LA64LARGE-NEXT: addi.d $sp, $sp, -32 +; LA64LARGE-NEXT: .cfi_def_cfa_offset 32 +; LA64LARGE-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill +; LA64LARGE-NEXT: st.d $fp, $sp, 16 # 8-byte Folded Spill +; LA64LARGE-NEXT: st.d $s0, $sp, 8 # 8-byte Folded Spill +; LA64LARGE-NEXT: .cfi_offset 1, -8 +; LA64LARGE-NEXT: .cfi_offset 22, -16 +; LA64LARGE-NEXT: .cfi_offset 23, -24 +; LA64LARGE-NEXT: move $fp, $a0 +; LA64LARGE-NEXT: move $s0, $zero +; LA64LARGE-NEXT: .p2align 4, , 16 +; LA64LARGE-NEXT: .LBB3_1: # %loop +; LA64LARGE-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64LARGE-NEXT: pcalau12i $a0, %ld_pc_hi20(ld) +; LA64LARGE-NEXT: addi.d $t8, $zero, %got_pc_lo12(ld) +; LA64LARGE-NEXT: lu32i.d $t8, %got64_pc_lo20(ld) +; LA64LARGE-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(ld) +; LA64LARGE-NEXT: add.d $a0, $t8, $a0 +; LA64LARGE-NEXT: pcalau12i $ra, %pc_hi20(__tls_get_addr) +; LA64LARGE-NEXT: addi.d $t8, $zero, %pc_lo12(__tls_get_addr) +; LA64LARGE-NEXT: lu32i.d $t8, %pc64_lo20(__tls_get_addr) +; LA64LARGE-NEXT: lu52i.d $t8, $t8, %pc64_hi12(__tls_get_addr) +; LA64LARGE-NEXT: add.d $ra, $t8, $ra +; LA64LARGE-NEXT: jirl $ra, $ra, 0 +; LA64LARGE-NEXT: ld.w $a0, $a0, 0 +; LA64LARGE-NEXT: addi.w $s0, $s0, 1 +; LA64LARGE-NEXT: blt $s0, $fp, .LBB3_1 +; LA64LARGE-NEXT: # %bb.2: # %ret +; LA64LARGE-NEXT: ld.d $s0, $sp, 8 # 8-byte Folded Reload +; LA64LARGE-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload +; LA64LARGE-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload +; LA64LARGE-NEXT: addi.d $sp, $sp, 32 +; LA64LARGE-NEXT: ret +entry: + br label %loop + +loop: + %i = phi i32 [ %inc, %loop ], [ 0, %entry ] + %0 = load volatile i32, ptr @ld, align 4 + %inc = add nuw nsw i32 %i, 1 + %cmp = icmp slt i32 %inc, %n + br i1 %cmp, label %loop, label %ret + +ret: + ret void +} + +@le = external thread_local(localexec) global i32 + +define void @test_la_tls_le(i32 signext %n) { +; LA32-LABEL: test_la_tls_le: +; LA32: # %bb.0: # %entry +; LA32-NEXT: move $a1, $zero +; LA32-NEXT: lu12i.w $a2, %le_hi20(le) +; LA32-NEXT: ori $a2, $a2, %le_lo12(le) +; LA32-NEXT: add.w $a2, $a2, $tp +; LA32-NEXT: .p2align 4, , 16 +; LA32-NEXT: .LBB4_1: # %loop +; LA32-NEXT: # =>This Inner Loop Header: Depth=1 +; LA32-NEXT: ld.w $a3, $a2, 0 +; LA32-NEXT: addi.w $a1, $a1, 1 +; LA32-NEXT: blt $a1, $a0, .LBB4_1 +; LA32-NEXT: # %bb.2: # %ret +; LA32-NEXT: ret +; +; LA64-LABEL: test_la_tls_le: +; LA64: # %bb.0: # %entry +; LA64-NEXT: move $a1, $zero +; LA64-NEXT: lu12i.w $a2, %le_hi20(le) +; LA64-NEXT: ori $a2, $a2, %le_lo12(le) +; LA64-NEXT: .p2align 4, , 16 +; LA64-NEXT: .LBB4_1: # %loop +; LA64-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64-NEXT: ldx.w $a3, $a2, $tp +; LA64-NEXT: addi.w $a1, $a1, 1 +; LA64-NEXT: blt $a1, $a0, .LBB4_1 +; LA64-NEXT: # %bb.2: # %ret +; LA64-NEXT: ret +; +; LA64LARGE-LABEL: test_la_tls_le: +; LA64LARGE: # %bb.0: # %entry +; LA64LARGE-NEXT: move $a1, $zero +; LA64LARGE-NEXT: lu12i.w $a2, %le_hi20(le) +; LA64LARGE-NEXT: ori $a2, $a2, %le_lo12(le) +; LA64LARGE-NEXT: lu32i.d $a2, %le64_lo20(le) +; LA64LARGE-NEXT: lu52i.d $a2, $a2, %le64_hi12(le) +; LA64LARGE-NEXT: .p2align 4, , 16 +; LA64LARGE-NEXT: .LBB4_1: # %loop +; LA64LARGE-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64LARGE-NEXT: ldx.w $a3, $a2, $tp +; LA64LARGE-NEXT: addi.w $a1, $a1, 1 +; LA64LARGE-NEXT: blt $a1, $a0, .LBB4_1 +; LA64LARGE-NEXT: # %bb.2: # %ret +; LA64LARGE-NEXT: ret +entry: + br label %loop + +loop: + %i = phi i32 [ %inc, %loop ], [ 0, %entry ] + %0 = load volatile i32, ptr @le, align 4 + %inc = add nuw nsw i32 %i, 1 + %cmp = icmp slt i32 %inc, %n + br i1 %cmp, label %loop, label %ret + +ret: + ret void +} + +@gd = external thread_local global i32 + +define void @test_la_tls_gd(i32 signext %n) nounwind { +; LA32-LABEL: test_la_tls_gd: +; LA32: # %bb.0: # %entry +; LA32-NEXT: addi.w $sp, $sp, -16 +; LA32-NEXT: st.w $ra, $sp, 12 # 4-byte Folded Spill +; LA32-NEXT: st.w $fp, $sp, 8 # 4-byte Folded Spill +; LA32-NEXT: st.w $s0, $sp, 4 # 4-byte Folded Spill +; LA32-NEXT: move $fp, $a0 +; LA32-NEXT: move $s0, $zero +; LA32-NEXT: .p2align 4, , 16 +; LA32-NEXT: .LBB5_1: # %loop +; LA32-NEXT: # =>This Inner Loop Header: Depth=1 +; LA32-NEXT: pcalau12i $a0, %gd_pc_hi20(gd) +; LA32-NEXT: addi.w $a0, $a0, %got_pc_lo12(gd) +; LA32-NEXT: bl %plt(__tls_get_addr) +; LA32-NEXT: ld.w $a0, $a0, 0 +; LA32-NEXT: addi.w $s0, $s0, 1 +; LA32-NEXT: blt $s0, $fp, .LBB5_1 +; LA32-NEXT: # %bb.2: # %ret +; LA32-NEXT: ld.w $s0, $sp, 4 # 4-byte Folded Reload +; LA32-NEXT: ld.w $fp, $sp, 8 # 4-byte Folded Reload +; LA32-NEXT: ld.w $ra, $sp, 12 # 4-byte Folded Reload +; LA32-NEXT: addi.w $sp, $sp, 16 +; LA32-NEXT: ret +; +; LA64-LABEL: test_la_tls_gd: +; LA64: # %bb.0: # %entry +; LA64-NEXT: addi.d $sp, $sp, -32 +; LA64-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill +; LA64-NEXT: st.d $fp, $sp, 16 # 8-byte Folded Spill +; LA64-NEXT: st.d $s0, $sp, 8 # 8-byte Folded Spill +; LA64-NEXT: move $fp, $a0 +; LA64-NEXT: move $s0, $zero +; LA64-NEXT: .p2align 4, , 16 +; LA64-NEXT: .LBB5_1: # %loop +; LA64-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64-NEXT: pcalau12i $a0, %gd_pc_hi20(gd) +; LA64-NEXT: addi.d $a0, $a0, %got_pc_lo12(gd) +; LA64-NEXT: bl %plt(__tls_get_addr) +; LA64-NEXT: ld.w $a0, $a0, 0 +; LA64-NEXT: addi.w $s0, $s0, 1 +; LA64-NEXT: blt $s0, $fp, .LBB5_1 +; LA64-NEXT: # %bb.2: # %ret +; LA64-NEXT: ld.d $s0, $sp, 8 # 8-byte Folded Reload +; LA64-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload +; LA64-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload +; LA64-NEXT: addi.d $sp, $sp, 32 +; LA64-NEXT: ret +; +; LA64LARGE-LABEL: test_la_tls_gd: +; LA64LARGE: # %bb.0: # %entry +; LA64LARGE-NEXT: addi.d $sp, $sp, -32 +; LA64LARGE-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill +; LA64LARGE-NEXT: st.d $fp, $sp, 16 # 8-byte Folded Spill +; LA64LARGE-NEXT: st.d $s0, $sp, 8 # 8-byte Folded Spill +; LA64LARGE-NEXT: move $fp, $a0 +; LA64LARGE-NEXT: move $s0, $zero +; LA64LARGE-NEXT: .p2align 4, , 16 +; LA64LARGE-NEXT: .LBB5_1: # %loop +; LA64LARGE-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64LARGE-NEXT: pcalau12i $a0, %gd_pc_hi20(gd) +; LA64LARGE-NEXT: addi.d $t8, $zero, %got_pc_lo12(gd) +; LA64LARGE-NEXT: lu32i.d $t8, %got64_pc_lo20(gd) +; LA64LARGE-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(gd) +; LA64LARGE-NEXT: add.d $a0, $t8, $a0 +; LA64LARGE-NEXT: pcalau12i $ra, %pc_hi20(__tls_get_addr) +; LA64LARGE-NEXT: addi.d $t8, $zero, %pc_lo12(__tls_get_addr) +; LA64LARGE-NEXT: lu32i.d $t8, %pc64_lo20(__tls_get_addr) +; LA64LARGE-NEXT: lu52i.d $t8, $t8, %pc64_hi12(__tls_get_addr) +; LA64LARGE-NEXT: add.d $ra, $t8, $ra +; LA64LARGE-NEXT: jirl $ra, $ra, 0 +; LA64LARGE-NEXT: ld.w $a0, $a0, 0 +; LA64LARGE-NEXT: addi.w $s0, $s0, 1 +; LA64LARGE-NEXT: blt $s0, $fp, .LBB5_1 +; LA64LARGE-NEXT: # %bb.2: # %ret +; LA64LARGE-NEXT: ld.d $s0, $sp, 8 # 8-byte Folded Reload +; LA64LARGE-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload +; LA64LARGE-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload +; LA64LARGE-NEXT: addi.d $sp, $sp, 32 +; LA64LARGE-NEXT: ret +entry: + br label %loop + +loop: + %i = phi i32 [ %inc, %loop ], [ 0, %entry ] + %0 = load volatile i32, ptr @gd, align 4 + %inc = add nuw nsw i32 %i, 1 + %cmp = icmp slt i32 %inc, %n + br i1 %cmp, label %loop, label %ret + +ret: + ret void +} -- GitLab From a6350d63af63fb11d87d508965559bbb2833927d Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 7 Jun 2024 13:08:29 +0800 Subject: [PATCH 189/462] [test] Don't generate `regalloc-amdgpu.s` in #94426 (#94722) The test will generate an empty `regalloc-amdgpu.s` file in test, which causes an unresolved test. --- llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir b/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir index 872c35237c20..0603ffc4c5ea 100644 --- a/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir +++ b/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir @@ -1,6 +1,6 @@ # REQUIRES: amdgpu-registered-target # RUN: llc -mtriple=amdgcn --passes='regallocfast' --print-pipeline-passes %s | FileCheck %s --check-prefix=PASS -# RUN: not llc -mtriple=amdgcn --passes='regallocfast' --print-pipeline-passes %s 2>&1 | FileCheck %s --check-prefix=BAD-FILTER +# RUN: not llc -mtriple=amdgcn --passes='regallocfast' --print-pipeline-passes --filetype=null %s 2>&1 | FileCheck %s --check-prefix=BAD-FILTER # PASS: regallocfast # BAD-FILTER: invalid regallocfast register filter 'bad-filter' -- GitLab From c5ff983fe4a3180e13c7244a6ce9f5994b4379b4 Mon Sep 17 00:00:00 2001 From: Congcong Cai Date: Fri, 7 Jun 2024 13:32:18 +0800 Subject: [PATCH 190/462] [clang-tidy] refactor misc-header-include-cycle (#94697) 1. merge valid check 2. use range base loop --- .../clang-tidy/misc/HeaderIncludeCycleCheck.cpp | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/clang-tools-extra/clang-tidy/misc/HeaderIncludeCycleCheck.cpp b/clang-tools-extra/clang-tidy/misc/HeaderIncludeCycleCheck.cpp index 37bc577c646a..cdb5e6b16069 100644 --- a/clang-tools-extra/clang-tidy/misc/HeaderIncludeCycleCheck.cpp +++ b/clang-tools-extra/clang-tidy/misc/HeaderIncludeCycleCheck.cpp @@ -130,20 +130,15 @@ public: << FileName; const bool IsIncludePathValid = - std::all_of(Files.rbegin(), It, [](const Include &Elem) { + std::all_of(Files.rbegin(), It + 1, [](const Include &Elem) { return !Elem.Name.empty() && Elem.Loc.isValid(); }); - if (!IsIncludePathValid) return; - auto CurrentIt = Files.rbegin(); - do { - if (CurrentIt->Loc.isValid()) - Check.diag(CurrentIt->Loc, "'%0' included from here", - DiagnosticIDs::Note) - << CurrentIt->Name; - } while (CurrentIt++ != It); + for (const Include &I : llvm::make_range(Files.rbegin(), It + 1)) + Check.diag(I.Loc, "'%0' included from here", DiagnosticIDs::Note) + << I.Name; } bool isFileIgnored(StringRef FileName) const { -- GitLab From d231b50dba6cf1b3ab095d381ae622e9498bbb3c Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Fri, 7 Jun 2024 07:43:20 +0200 Subject: [PATCH 191/462] [libc++] Fix naming of value template parameters (#76888) --- libcxx/.clang-tidy | 8 ++++++-- libcxx/include/__functional/bind.h | 4 ++-- libcxx/include/__mdspan/extents.h | 6 +++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/libcxx/.clang-tidy b/libcxx/.clang-tidy index ec7cab9b878e..d0b4cced9ffc 100644 --- a/libcxx/.clang-tidy +++ b/libcxx/.clang-tidy @@ -57,8 +57,12 @@ CheckOptions: value: _ - key: readability-identifier-naming.TemplateParameterIgnoredRegexp value: (.*\:auto|expr-type) # This is https://llvm.org/PR56464 - - key: readability-identifier-naming.ValueTemplateParameterIgnoredRegexp # TODO: enforce naming of variable parameters - value: .* + - key: readability-identifier-naming.ValueTemplateParameterCase + value: CamelCase + - key: readability-identifier-naming.ValueTemplateParameterPrefix + value: _ + - key: readability-identifier-naming.ValueTemplateParameterIgnoredRegexp + value: (__[a-z_]|_[A-Z]).* # TODO: Converge on a single style for value template parameters # TODO: investigate these checks # bugprone-branch-clone, diff --git a/libcxx/include/__functional/bind.h b/libcxx/include/__functional/bind.h index 8a0e3b7ffa58..385dabaee424 100644 --- a/libcxx/include/__functional/bind.h +++ b/libcxx/include/__functional/bind.h @@ -95,7 +95,7 @@ __mu(_Ti& __ti, tuple<_Uj...>& __uj) { return std::__mu_expand(__ti, __uj, __indices()); } -template +template struct __mu_return2 {}; template @@ -120,7 +120,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Ti& __mu(_Ti& __ti, return __ti; } -template +template struct __mu_return_impl; template diff --git a/libcxx/include/__mdspan/extents.h b/libcxx/include/__mdspan/extents.h index f6bcd940ee60..16fac1354961 100644 --- a/libcxx/include/__mdspan/extents.h +++ b/libcxx/include/__mdspan/extents.h @@ -137,9 +137,9 @@ private: // static mapping of indices to the position in the dynamic values array using _DynamicIdxMap = __static_partial_sums(_Values == _DynTag)...>; - template - _LIBCPP_HIDE_FROM_ABI static constexpr _DynamicValues __zeros(index_sequence) noexcept { - return _DynamicValues{((void)Indices, 0)...}; + template + _LIBCPP_HIDE_FROM_ABI static constexpr _DynamicValues __zeros(index_sequence<_Indices...>) noexcept { + return _DynamicValues{((void)_Indices, 0)...}; } public: -- GitLab From 670fa2bd7a4d50c33bc24d4e3e1fa4079592b730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Horv=C3=A1th?= Date: Fri, 7 Jun 2024 06:47:57 +0100 Subject: [PATCH 192/462] Fix spurious non-strict availability warning (#94377) The availability attributes are stored on the function declarations. The code was looking for them in the function template declarations. This resulted in spuriously diagnosing (non-strict) availablity issues in contexts that are not available. Co-authored-by: Gabor Horvath --- clang/include/clang/Sema/Initialization.h | 2 +- clang/lib/Sema/SemaAvailability.cpp | 5 +++++ clang/test/Sema/attr-availability-macosx.cpp | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 clang/test/Sema/attr-availability-macosx.cpp diff --git a/clang/include/clang/Sema/Initialization.h b/clang/include/clang/Sema/Initialization.h index 2072cd8d1c3e..f443e327eaf3 100644 --- a/clang/include/clang/Sema/Initialization.h +++ b/clang/include/clang/Sema/Initialization.h @@ -212,7 +212,7 @@ private: struct C Capture; }; - InitializedEntity() {}; + InitializedEntity() {} /// Create the initialization entity for a variable. InitializedEntity(VarDecl *Var, EntityKind EK = EK_Variable) diff --git a/clang/lib/Sema/SemaAvailability.cpp b/clang/lib/Sema/SemaAvailability.cpp index 330cd602297d..3e5f90b45036 100644 --- a/clang/lib/Sema/SemaAvailability.cpp +++ b/clang/lib/Sema/SemaAvailability.cpp @@ -12,6 +12,7 @@ #include "clang/AST/Attr.h" #include "clang/AST/Decl.h" +#include "clang/AST/DeclTemplate.h" #include "clang/AST/RecursiveASTVisitor.h" #include "clang/Basic/DiagnosticSema.h" #include "clang/Basic/IdentifierTable.h" @@ -46,6 +47,10 @@ static const AvailabilityAttr *getAttrForPlatform(ASTContext &Context, // Check each AvailabilityAttr to find the one for this platform. // For multiple attributes with the same platform try to find one for this // environment. + // The attribute is always on the FunctionDecl, not on the + // FunctionTemplateDecl. + if (const auto *FTD = dyn_cast(D)) + D = FTD->getTemplatedDecl(); for (const auto *A : D->attrs()) { if (const auto *Avail = dyn_cast(A)) { // FIXME: this is copied from CheckAvailability. We should try to diff --git a/clang/test/Sema/attr-availability-macosx.cpp b/clang/test/Sema/attr-availability-macosx.cpp new file mode 100644 index 000000000000..52f320d40928 --- /dev/null +++ b/clang/test/Sema/attr-availability-macosx.cpp @@ -0,0 +1,17 @@ +// RUN: %clang_cc1 "-triple" "arm64-apple-macosx10.15" -fsyntax-only -verify %s + +__attribute__((availability(macos,introduced=11))) +inline bool try_acquire() { + return true; +} + +template +__attribute__((availability(macos,introduced=11))) +bool try_acquire_for(T duration) { // expected-note{{'try_acquire_for' has been marked as being introduced in macOS 11 here, but the deployment target is macOS 10.15}} + return try_acquire(); +} + +int main() { + try_acquire_for(1); // expected-warning{{'try_acquire_for' is only available on macOS 11 or newer}} + // expected-note@-1{{enclose 'try_acquire_for' in a __builtin_available check to silence this warning}} +} \ No newline at end of file -- GitLab From 1752740f4b4b752bbe2987a0de398c6f671ceb71 Mon Sep 17 00:00:00 2001 From: Prashant Kumar Date: Fri, 7 Jun 2024 11:22:36 +0530 Subject: [PATCH 193/462] [mlir][tensor] Fix FoldTensorCastProducerOp for multiple result operations (#93374) For patterns where there are multiple results apart from dpsInits, this fails. E.g.: ``` %13:2 = iree_codegen.ukernel.generic "iree_uk_unpack" ins(%extracted_slice : tensor) outs(%11 : tensor) ... -> tensor, i32 ``` The above op has results apart from dpsInit and hence fails. The PR assumes that the result has dpsInits followed by nonDpsInits. --- mlir/lib/Dialect/Tensor/IR/TensorOps.cpp | 7 ++++--- mlir/test/Dialect/Tensor/canonicalize.mlir | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp index 8545c7b9af8f..7fc29ec0139c 100644 --- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp +++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp @@ -4531,17 +4531,18 @@ struct FoldTensorCastProducerOp if (!hasTensorCastOperand) return failure(); - SmallVector newResultTypes; - newResultTypes.reserve(op->getNumResults()); + SmallVector newResultTypes(op->getResultTypes()); SmallVector newOperands; newOperands.reserve(op->getNumOperands()); + // Assumes that the result has dpsInits followed by nonDpsInits. + int64_t dpsInitIdx = 0; for (OpOperand &opOperand : op->getOpOperands()) { auto tensorCastOp = opOperand.get().getDefiningOp(); bool fold = canFoldIntoConsumerOp(tensorCastOp); newOperands.push_back(fold ? tensorCastOp.getOperand() : opOperand.get()); if (op.isDpsInit(&opOperand) && !llvm::isa(newOperands.back().getType())) - newResultTypes.push_back(newOperands.back().getType()); + newResultTypes[dpsInitIdx++] = newOperands.back().getType(); } // Clone op. diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir index f7fbd3834288..6b51d0b294bc 100644 --- a/mlir/test/Dialect/Tensor/canonicalize.mlir +++ b/mlir/test/Dialect/Tensor/canonicalize.mlir @@ -2523,3 +2523,18 @@ func.func @dim_out_of_bounds() -> vector<7xi32> { %16 = affine.vector_load %alloc_21[%c1, %c1, %dim] : memref, vector<7xi32> return %16 : vector<7xi32> } + +// ----- + +// CHECK-LABEL: func.func @test_destination_multiple_result( +// CHECK-SAME: %[[ARG1:.*]]: tensor<2x2xf32>, +// CHECK-SAME: %[[ARG2:.*]]: tensor<2x2xf32>) -> index { +// CHECK: %[[RES:.*]]:2 = test.destination_style_op ins(%[[ARG1]] : tensor<2x2xf32>) +// CHECK-SAME: outs(%[[ARG2]] : tensor<2x2xf32>) -> tensor<2x2xf32>, index +// CHECK: return %[[RES]]#1 : index +func.func @test_destination_multiple_result(%arg0: tensor<2x2xf32>, %arg1: tensor<2x2xf32>) -> index { + %cast = tensor.cast %arg0 : tensor<2x2xf32> to tensor + %cast_0 = tensor.cast %arg1 : tensor<2x2xf32> to tensor + %0:2 = test.destination_style_op ins(%cast : tensor) outs(%cast_0 : tensor) -> tensor, index + return %0#1 : index +} -- GitLab From ec94e7a93031549b1f4e21e560ff4d3db265973a Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 6 Jun 2024 23:00:52 -0700 Subject: [PATCH 194/462] [MC] MCSection::dump: print section name --- llvm/lib/MC/MCSection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/MC/MCSection.cpp b/llvm/lib/MC/MCSection.cpp index 0fb9e8e13910..ea3baf96b38e 100644 --- a/llvm/lib/MC/MCSection.cpp +++ b/llvm/lib/MC/MCSection.cpp @@ -128,7 +128,7 @@ void MCSection::flushPendingLabels() { LLVM_DUMP_METHOD void MCSection::dump() const { raw_ostream &OS = errs(); - OS << " Date: Fri, 7 Jun 2024 08:15:43 +0200 Subject: [PATCH 195/462] [clang][Interp] Improve APValue machinery Handle lvalues pointing to declarations, unions and member pointers. --- clang/lib/AST/Interp/ByteCodeExprGen.cpp | 50 ++++++++++++++++++++---- clang/lib/AST/Interp/ByteCodeExprGen.h | 2 + clang/test/AST/Interp/cxx20.cpp | 8 ++++ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index d124248a3605..907317526364 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -3218,9 +3218,18 @@ bool ByteCodeExprGen::visitAPValue(const APValue &Val, return this->emitConst(Val.getInt(), ValType, E); if (Val.isLValue()) { + if (Val.isNullPointer()) + return this->emitNull(ValType, nullptr, E); APValue::LValueBase Base = Val.getLValueBase(); if (const Expr *BaseExpr = Base.dyn_cast()) return this->visit(BaseExpr); + else if (const auto *VD = Base.dyn_cast()) { + return this->visitDeclRef(VD, E); + } + } else if (Val.isMemberPointer()) { + if (const ValueDecl *MemberDecl = Val.getMemberPointerDecl()) + return this->emitGetMemberPtr(MemberDecl, E); + return this->emitNullMemberPtr(nullptr, E); } return false; @@ -3229,15 +3238,15 @@ bool ByteCodeExprGen::visitAPValue(const APValue &Val, template bool ByteCodeExprGen::visitAPValueInitializer(const APValue &Val, const Expr *E) { + if (Val.isStruct()) { const Record *R = this->getRecord(E->getType()); assert(R); - for (unsigned I = 0, N = Val.getStructNumFields(); I != N; ++I) { const APValue &F = Val.getStructField(I); const Record::Field *RF = R->getField(I); - if (F.isInt() || F.isLValue()) { + if (F.isInt() || F.isLValue() || F.isMemberPointer()) { PrimType T = classifyPrim(RF->Decl->getType()); if (!this->visitAPValue(F, T, E)) return false; @@ -3263,11 +3272,30 @@ bool ByteCodeExprGen::visitAPValueInitializer(const APValue &Val, if (!this->emitPopPtr(E)) return false; + } else if (F.isStruct() || F.isUnion()) { + if (!this->emitDupPtr(E)) + return false; + if (!this->emitGetPtrField(RF->Offset, E)) + return false; + if (!this->visitAPValueInitializer(F, E)) + return false; + if (!this->emitPopPtr(E)) + return false; } else { assert(false && "I don't think this should be possible"); } } return true; + } else if (Val.isUnion()) { + const FieldDecl *UnionField = Val.getUnionField(); + const Record *R = this->getRecord(UnionField->getParent()); + assert(R); + const APValue &F = Val.getUnionValue(); + const Record::Field *RF = R->getField(UnionField); + PrimType T = classifyPrim(RF->Decl->getType()); + if (!this->visitAPValue(F, T, E)) + return false; + return this->emitInitElem(T, 0, E); } // TODO: Other types. @@ -3827,12 +3855,10 @@ bool ByteCodeExprGen::VisitComplexUnaryOperator( } template -bool ByteCodeExprGen::VisitDeclRefExpr(const DeclRefExpr *E) { +bool ByteCodeExprGen::visitDeclRef(const ValueDecl *D, const Expr *E) { if (DiscardResult) return true; - const auto *D = E->getDecl(); - if (const auto *ECD = dyn_cast(D)) { return this->emitConst(ECD->getInitVal(), E); } else if (const auto *BD = dyn_cast(D)) { @@ -3900,7 +3926,7 @@ bool ByteCodeExprGen::VisitDeclRefExpr(const DeclRefExpr *E) { if (!this->visitVarDecl(VD)) return false; // Retry. - return this->VisitDeclRefExpr(E); + return this->visitDeclRef(VD, E); } } } else { @@ -3910,7 +3936,7 @@ bool ByteCodeExprGen::VisitDeclRefExpr(const DeclRefExpr *E) { if (!this->visitVarDecl(VD)) return false; // Retry. - return this->VisitDeclRefExpr(E); + return this->visitDeclRef(VD, E); } } @@ -3927,7 +3953,15 @@ bool ByteCodeExprGen::VisitDeclRefExpr(const DeclRefExpr *E) { return true; } - return this->emitInvalidDeclRef(E, E); + if (const auto *DRE = dyn_cast(E)) + return this->emitInvalidDeclRef(DRE, E); + return false; +} + +template +bool ByteCodeExprGen::VisitDeclRefExpr(const DeclRefExpr *E) { + const auto *D = E->getDecl(); + return this->visitDeclRef(D, E); } template diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.h b/clang/lib/AST/Interp/ByteCodeExprGen.h index 44c495240289..7ab14b6ab383 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.h +++ b/clang/lib/AST/Interp/ByteCodeExprGen.h @@ -189,6 +189,8 @@ protected: /// Visit an APValue. bool visitAPValue(const APValue &Val, PrimType ValType, const Expr *E); bool visitAPValueInitializer(const APValue &Val, const Expr *E); + /// Visit the given decl as if we have a reference to it. + bool visitDeclRef(const ValueDecl *D, const Expr *E); /// Visits an expression and converts it to a boolean. bool visitBool(const Expr *E); diff --git a/clang/test/AST/Interp/cxx20.cpp b/clang/test/AST/Interp/cxx20.cpp index 000ffe39eb94..49ee040be392 100644 --- a/clang/test/AST/Interp/cxx20.cpp +++ b/clang/test/AST/Interp/cxx20.cpp @@ -774,3 +774,11 @@ void overflowInSwitchCase(int n) { break; } } + +namespace APValues { + int g; + struct A { union { int n, m; }; int *p; int A::*q; char buffer[32]; }; + template constexpr const A &get = a; + constexpr const A &v = get; + constexpr const A &w = get; +} -- GitLab From d843c02aba5d253891ea0e599c4b6e25247c1af7 Mon Sep 17 00:00:00 2001 From: Freddy Ye Date: Fri, 7 Jun 2024 14:17:21 +0800 Subject: [PATCH 196/462] [X86] Assign AVX10_1 feature priority to align with gcc. (#94557) gcc patch: https://gcc.gnu.org/git/?p=gcc.git;a=commit;h=1f2ca510065a2033bac408eb5a960ef0126f25cc --- llvm/include/llvm/TargetParser/X86TargetParser.def | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.def b/llvm/include/llvm/TargetParser/X86TargetParser.def index 8daa8a689c95..f10cede0734e 100644 --- a/llvm/include/llvm/TargetParser/X86TargetParser.def +++ b/llvm/include/llvm/TargetParser/X86TargetParser.def @@ -250,8 +250,8 @@ X86_FEATURE_COMPAT(SHA512, "sha512", 0) X86_FEATURE_COMPAT(SM4, "sm4", 0) X86_FEATURE (EGPR, "egpr") X86_FEATURE_COMPAT(USERMSR, "usermsr", 0) -X86_FEATURE_COMPAT(AVX10_1, "avx10.1-256", 0) -X86_FEATURE_COMPAT(AVX10_1_512, "avx10.1-512", 0) +X86_FEATURE_COMPAT(AVX10_1, "avx10.1-256", 36) +X86_FEATURE_COMPAT(AVX10_1_512, "avx10.1-512", 37) // These features aren't really CPU features, but the frontend can set them. X86_FEATURE (RETPOLINE_EXTERNAL_THUNK, "retpoline-external-thunk") X86_FEATURE (RETPOLINE_INDIRECT_BRANCHES, "retpoline-indirect-branches") -- GitLab From 90b9922df2d8fb65a33eb9abd002fa9262863b5b Mon Sep 17 00:00:00 2001 From: Pavel Labath Date: Fri, 7 Jun 2024 08:39:10 +0200 Subject: [PATCH 197/462] [lldb] Split ValueObject::CreateChildAtIndex into two functions (#94455) The the function is doing two fairly different things, depending on how it is called. While this allows for some code reuse, it also makes it hard to override it correctly. Possibly for this reason ValueObjectSynthetic overerides GetChildAtIndex instead, which forces it to reimplement some of its functionality, most notably caching of generated children. Splitting this up makes it easier to move the caching to a common place (and hopefully makes the code easier to follow in general). --- lldb/include/lldb/Core/ValueObject.h | 9 +- .../lldb/Core/ValueObjectConstResult.h | 10 +- .../lldb/Core/ValueObjectConstResultCast.h | 10 +- .../lldb/Core/ValueObjectConstResultChild.h | 10 +- .../lldb/Core/ValueObjectConstResultImpl.h | 4 +- lldb/include/lldb/Core/ValueObjectRegister.h | 8 +- lldb/include/lldb/Core/ValueObjectVTable.h | 8 +- lldb/source/Core/ValueObject.cpp | 91 ++++++++------ lldb/source/Core/ValueObjectConstResult.cpp | 6 - .../Core/ValueObjectConstResultCast.cpp | 6 - .../Core/ValueObjectConstResultChild.cpp | 6 - .../Core/ValueObjectConstResultImpl.cpp | 115 ++++++++++++------ lldb/source/Core/ValueObjectRegister.cpp | 14 +-- lldb/source/Core/ValueObjectVTable.cpp | 6 +- 14 files changed, 178 insertions(+), 125 deletions(-) diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h index e7e35e2b2bff..1e8c7c9c0053 100644 --- a/lldb/include/lldb/Core/ValueObject.h +++ b/lldb/include/lldb/Core/ValueObject.h @@ -959,9 +959,12 @@ protected: /// Should only be called by ValueObject::GetChildAtIndex(). /// /// \return A ValueObject managed by this ValueObject's manager. - virtual ValueObject *CreateChildAtIndex(size_t idx, - bool synthetic_array_member, - int32_t synthetic_index); + virtual ValueObject *CreateChildAtIndex(size_t idx); + + /// Should only be called by ValueObject::GetSyntheticArrayMember(). + /// + /// \return A ValueObject managed by this ValueObject's manager. + virtual ValueObject *CreateSyntheticArrayMember(size_t idx); /// Should only be called by ValueObject::GetNumChildren(). virtual llvm::Expected diff --git a/lldb/include/lldb/Core/ValueObjectConstResult.h b/lldb/include/lldb/Core/ValueObjectConstResult.h index 37dc0867f26c..d3b3362bd0e9 100644 --- a/lldb/include/lldb/Core/ValueObjectConstResult.h +++ b/lldb/include/lldb/Core/ValueObjectConstResult.h @@ -79,9 +79,6 @@ public: lldb::ValueObjectSP Dereference(Status &error) override; - ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member, - int32_t synthetic_index) override; - lldb::ValueObjectSP GetSyntheticChildAtOffset( uint32_t offset, const CompilerType &type, bool can_create, ConstString name_const_str = ConstString()) override; @@ -151,6 +148,13 @@ private: ValueObjectConstResult(ExecutionContextScope *exe_scope, ValueObjectManager &manager, const Status &error); + ValueObject *CreateChildAtIndex(size_t idx) override { + return m_impl.CreateChildAtIndex(idx); + } + ValueObject *CreateSyntheticArrayMember(size_t idx) override { + return m_impl.CreateSyntheticArrayMember(idx); + } + ValueObjectConstResult(const ValueObjectConstResult &) = delete; const ValueObjectConstResult & operator=(const ValueObjectConstResult &) = delete; diff --git a/lldb/include/lldb/Core/ValueObjectConstResultCast.h b/lldb/include/lldb/Core/ValueObjectConstResultCast.h index efcbe0dc6a0b..911a08363b39 100644 --- a/lldb/include/lldb/Core/ValueObjectConstResultCast.h +++ b/lldb/include/lldb/Core/ValueObjectConstResultCast.h @@ -35,9 +35,6 @@ public: lldb::ValueObjectSP Dereference(Status &error) override; - ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member, - int32_t synthetic_index) override; - virtual CompilerType GetCompilerType() { return ValueObjectCast::GetCompilerType(); } @@ -61,6 +58,13 @@ private: friend class ValueObjectConstResult; friend class ValueObjectConstResultImpl; + ValueObject *CreateChildAtIndex(size_t idx) override { + return m_impl.CreateChildAtIndex(idx); + } + ValueObject *CreateSyntheticArrayMember(size_t idx) override { + return m_impl.CreateSyntheticArrayMember(idx); + } + ValueObjectConstResultCast(const ValueObjectConstResultCast &) = delete; const ValueObjectConstResultCast & operator=(const ValueObjectConstResultCast &) = delete; diff --git a/lldb/include/lldb/Core/ValueObjectConstResultChild.h b/lldb/include/lldb/Core/ValueObjectConstResultChild.h index 7e9da14e8e97..71a3c53befe7 100644 --- a/lldb/include/lldb/Core/ValueObjectConstResultChild.h +++ b/lldb/include/lldb/Core/ValueObjectConstResultChild.h @@ -41,9 +41,6 @@ public: lldb::ValueObjectSP Dereference(Status &error) override; - ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member, - int32_t synthetic_index) override; - virtual CompilerType GetCompilerType() { return ValueObjectChild::GetCompilerType(); } @@ -70,6 +67,13 @@ private: friend class ValueObjectConstResult; friend class ValueObjectConstResultImpl; + ValueObject *CreateChildAtIndex(size_t idx) override { + return m_impl.CreateChildAtIndex(idx); + } + ValueObject *CreateSyntheticArrayMember(size_t idx) override { + return m_impl.CreateSyntheticArrayMember(idx); + } + ValueObjectConstResultChild(const ValueObjectConstResultChild &) = delete; const ValueObjectConstResultChild & operator=(const ValueObjectConstResultChild &) = delete; diff --git a/lldb/include/lldb/Core/ValueObjectConstResultImpl.h b/lldb/include/lldb/Core/ValueObjectConstResultImpl.h index 5a7a079d3095..68ba8ae7fba2 100644 --- a/lldb/include/lldb/Core/ValueObjectConstResultImpl.h +++ b/lldb/include/lldb/Core/ValueObjectConstResultImpl.h @@ -38,8 +38,8 @@ public: lldb::ValueObjectSP Dereference(Status &error); - ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member, - int32_t synthetic_index); + ValueObject *CreateChildAtIndex(size_t idx); + ValueObject *CreateSyntheticArrayMember(size_t idx); lldb::ValueObjectSP GetSyntheticChildAtOffset(uint32_t offset, const CompilerType &type, diff --git a/lldb/include/lldb/Core/ValueObjectRegister.h b/lldb/include/lldb/Core/ValueObjectRegister.h index fec8566ba33d..d948c663a4f8 100644 --- a/lldb/include/lldb/Core/ValueObjectRegister.h +++ b/lldb/include/lldb/Core/ValueObjectRegister.h @@ -49,9 +49,6 @@ public: llvm::Expected CalculateNumChildren(uint32_t max) override; - ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member, - int32_t synthetic_index) override; - lldb::ValueObjectSP GetChildMemberWithName(llvm::StringRef name, bool can_create = true) override; @@ -73,6 +70,11 @@ private: ValueObjectManager &manager, lldb::RegisterContextSP ®_ctx_sp, uint32_t set_idx); + ValueObject *CreateChildAtIndex(size_t idx) override; + ValueObject *CreateSyntheticArrayMember(size_t idx) override { + return nullptr; + } + // For ValueObject only ValueObjectRegisterSet(const ValueObjectRegisterSet &) = delete; const ValueObjectRegisterSet & diff --git a/lldb/include/lldb/Core/ValueObjectVTable.h b/lldb/include/lldb/Core/ValueObjectVTable.h index 4662f395a4dd..7087dcc1d1be 100644 --- a/lldb/include/lldb/Core/ValueObjectVTable.h +++ b/lldb/include/lldb/Core/ValueObjectVTable.h @@ -66,9 +66,6 @@ public: llvm::Expected CalculateNumChildren(uint32_t max) override; - ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member, - int32_t synthetic_index) override; - lldb::ValueType GetValueType() const override; ConstString GetTypeName() override; @@ -95,6 +92,11 @@ protected: private: ValueObjectVTable(ValueObject &parent); + ValueObject *CreateChildAtIndex(size_t idx) override; + ValueObject *CreateSyntheticArrayMember(size_t idx) override { + return nullptr; + } + // For ValueObject only ValueObjectVTable(const ValueObjectVTable &) = delete; const ValueObjectVTable &operator=(const ValueObjectVTable &) = delete; diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp index c5c434a941b3..f32cb82faa00 100644 --- a/lldb/source/Core/ValueObject.cpp +++ b/lldb/source/Core/ValueObject.cpp @@ -382,7 +382,7 @@ ValueObjectSP ValueObject::GetChildAtIndex(uint32_t idx, bool can_create) { if (can_create && !m_children.HasChildAtIndex(idx)) { // No we haven't created the child at this index, so lets have our // subclass do it and cache the result for quick future access. - m_children.SetChildAtIndex(idx, CreateChildAtIndex(idx, false, 0)); + m_children.SetChildAtIndex(idx, CreateChildAtIndex(idx)); } ValueObject *child = m_children.GetChildAtIndex(idx); @@ -488,14 +488,10 @@ void ValueObject::SetNumChildren(uint32_t num_children) { m_children.SetChildrenCount(num_children); } -ValueObject *ValueObject::CreateChildAtIndex(size_t idx, - bool synthetic_array_member, - int32_t synthetic_index) { - ValueObject *valobj = nullptr; - +ValueObject *ValueObject::CreateChildAtIndex(size_t idx) { bool omit_empty_base_classes = true; - bool ignore_array_bounds = synthetic_array_member; - std::string child_name_str; + bool ignore_array_bounds = false; + std::string child_name; uint32_t child_byte_size = 0; int32_t child_byte_offset = 0; uint32_t child_bitfield_bit_size = 0; @@ -503,51 +499,74 @@ ValueObject *ValueObject::CreateChildAtIndex(size_t idx, bool child_is_base_class = false; bool child_is_deref_of_parent = false; uint64_t language_flags = 0; - - const bool transparent_pointers = !synthetic_array_member; + const bool transparent_pointers = true; ExecutionContext exe_ctx(GetExecutionContextRef()); auto child_compiler_type_or_err = GetCompilerType().GetChildCompilerTypeAtIndex( &exe_ctx, idx, transparent_pointers, omit_empty_base_classes, - ignore_array_bounds, child_name_str, child_byte_size, - child_byte_offset, child_bitfield_bit_size, child_bitfield_bit_offset, + ignore_array_bounds, child_name, child_byte_size, child_byte_offset, + child_bitfield_bit_size, child_bitfield_bit_offset, child_is_base_class, child_is_deref_of_parent, this, language_flags); - CompilerType child_compiler_type; - if (!child_compiler_type_or_err) + if (!child_compiler_type_or_err || !child_compiler_type_or_err->IsValid()) { LLDB_LOG_ERROR(GetLog(LLDBLog::Types), child_compiler_type_or_err.takeError(), "could not find child: {0}"); - else - child_compiler_type = *child_compiler_type_or_err; + return nullptr; + } + + return new ValueObjectChild( + *this, *child_compiler_type_or_err, ConstString(child_name), + child_byte_size, child_byte_offset, child_bitfield_bit_size, + child_bitfield_bit_offset, child_is_base_class, child_is_deref_of_parent, + eAddressTypeInvalid, language_flags); +} + +ValueObject *ValueObject::CreateSyntheticArrayMember(size_t idx) { + bool omit_empty_base_classes = true; + bool ignore_array_bounds = true; + std::string child_name; + uint32_t child_byte_size = 0; + int32_t child_byte_offset = 0; + uint32_t child_bitfield_bit_size = 0; + uint32_t child_bitfield_bit_offset = 0; + bool child_is_base_class = false; + bool child_is_deref_of_parent = false; + uint64_t language_flags = 0; + const bool transparent_pointers = false; - if (child_compiler_type) { - if (synthetic_index) - child_byte_offset += child_byte_size * synthetic_index; + ExecutionContext exe_ctx(GetExecutionContextRef()); - ConstString child_name; - if (!child_name_str.empty()) - child_name.SetCString(child_name_str.c_str()); + auto child_compiler_type_or_err = + GetCompilerType().GetChildCompilerTypeAtIndex( + &exe_ctx, 0, transparent_pointers, omit_empty_base_classes, + ignore_array_bounds, child_name, child_byte_size, child_byte_offset, + child_bitfield_bit_size, child_bitfield_bit_offset, + child_is_base_class, child_is_deref_of_parent, this, language_flags); + if (!child_compiler_type_or_err) { + LLDB_LOG_ERROR(GetLog(LLDBLog::Types), + child_compiler_type_or_err.takeError(), + "could not find child: {0}"); + return nullptr; + } + + if (child_compiler_type_or_err->IsValid()) { + child_byte_offset += child_byte_size * idx; - valobj = new ValueObjectChild( - *this, child_compiler_type, child_name, child_byte_size, - child_byte_offset, child_bitfield_bit_size, child_bitfield_bit_offset, - child_is_base_class, child_is_deref_of_parent, eAddressTypeInvalid, - language_flags); + return new ValueObjectChild( + *this, *child_compiler_type_or_err, ConstString(child_name), + child_byte_size, child_byte_offset, child_bitfield_bit_size, + child_bitfield_bit_offset, child_is_base_class, + child_is_deref_of_parent, eAddressTypeInvalid, language_flags); } // In case of an incomplete type, try to use the ValueObject's // synthetic value to create the child ValueObject. - if (!valobj && synthetic_array_member) { - if (ValueObjectSP synth_valobj_sp = GetSyntheticValue()) { - valobj = synth_valobj_sp - ->GetChildAtIndex(synthetic_index, synthetic_array_member) - .get(); - } - } + if (ValueObjectSP synth_valobj_sp = GetSyntheticValue()) + return synth_valobj_sp->GetChildAtIndex(idx, /*can_create=*/true).get(); - return valobj; + return nullptr; } bool ValueObject::GetSummaryAsCString(TypeSummaryImpl *summary_ptr, @@ -1616,7 +1635,7 @@ ValueObjectSP ValueObject::GetSyntheticArrayMember(size_t index, ValueObject *synthetic_child; // We haven't made a synthetic array member for INDEX yet, so lets make // one and cache it for any future reference. - synthetic_child = CreateChildAtIndex(0, true, index); + synthetic_child = CreateSyntheticArrayMember(index); // Cache the value if we got one back... if (synthetic_child) { diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/Core/ValueObjectConstResult.cpp index 8ac2c1cac2f6..879d3c3f6b03 100644 --- a/lldb/source/Core/ValueObjectConstResult.cpp +++ b/lldb/source/Core/ValueObjectConstResult.cpp @@ -267,12 +267,6 @@ lldb::addr_t ValueObjectConstResult::GetAddressOf(bool scalar_is_load_address, return m_impl.GetAddressOf(scalar_is_load_address, address_type); } -ValueObject *ValueObjectConstResult::CreateChildAtIndex( - size_t idx, bool synthetic_array_member, int32_t synthetic_index) { - return m_impl.CreateChildAtIndex(idx, synthetic_array_member, - synthetic_index); -} - size_t ValueObjectConstResult::GetPointeeData(DataExtractor &data, uint32_t item_idx, uint32_t item_count) { diff --git a/lldb/source/Core/ValueObjectConstResultCast.cpp b/lldb/source/Core/ValueObjectConstResultCast.cpp index fceb2635f876..bf7a12dc6823 100644 --- a/lldb/source/Core/ValueObjectConstResultCast.cpp +++ b/lldb/source/Core/ValueObjectConstResultCast.cpp @@ -44,12 +44,6 @@ lldb::ValueObjectSP ValueObjectConstResultCast::AddressOf(Status &error) { return m_impl.AddressOf(error); } -ValueObject *ValueObjectConstResultCast::CreateChildAtIndex( - size_t idx, bool synthetic_array_member, int32_t synthetic_index) { - return m_impl.CreateChildAtIndex(idx, synthetic_array_member, - synthetic_index); -} - size_t ValueObjectConstResultCast::GetPointeeData(DataExtractor &data, uint32_t item_idx, uint32_t item_count) { diff --git a/lldb/source/Core/ValueObjectConstResultChild.cpp b/lldb/source/Core/ValueObjectConstResultChild.cpp index 36bf11a0b73a..39fc0c9fbb35 100644 --- a/lldb/source/Core/ValueObjectConstResultChild.cpp +++ b/lldb/source/Core/ValueObjectConstResultChild.cpp @@ -56,12 +56,6 @@ lldb::addr_t ValueObjectConstResultChild::GetAddressOf( return m_impl.GetAddressOf(scalar_is_load_address, address_type); } -ValueObject *ValueObjectConstResultChild::CreateChildAtIndex( - size_t idx, bool synthetic_array_member, int32_t synthetic_index) { - return m_impl.CreateChildAtIndex(idx, synthetic_array_member, - synthetic_index); -} - size_t ValueObjectConstResultChild::GetPointeeData(DataExtractor &data, uint32_t item_idx, uint32_t item_count) { diff --git a/lldb/source/Core/ValueObjectConstResultImpl.cpp b/lldb/source/Core/ValueObjectConstResultImpl.cpp index 493980d7ea96..2a7c90770076 100644 --- a/lldb/source/Core/ValueObjectConstResultImpl.cpp +++ b/lldb/source/Core/ValueObjectConstResultImpl.cpp @@ -46,18 +46,15 @@ lldb::ValueObjectSP ValueObjectConstResultImpl::Dereference(Status &error) { return m_impl_backend->ValueObject::Dereference(error); } -ValueObject *ValueObjectConstResultImpl::CreateChildAtIndex( - size_t idx, bool synthetic_array_member, int32_t synthetic_index) { +ValueObject *ValueObjectConstResultImpl::CreateChildAtIndex(size_t idx) { if (m_impl_backend == nullptr) return nullptr; m_impl_backend->UpdateValueIfNeeded(false); - ValueObjectConstResultChild *valobj = nullptr; - bool omit_empty_base_classes = true; - bool ignore_array_bounds = synthetic_array_member; - std::string child_name_str; + bool ignore_array_bounds = false; + std::string child_name; uint32_t child_byte_size = 0; int32_t child_byte_offset = 0; uint32_t child_bitfield_bit_size = 0; @@ -65,56 +62,100 @@ ValueObject *ValueObjectConstResultImpl::CreateChildAtIndex( bool child_is_base_class = false; bool child_is_deref_of_parent = false; uint64_t language_flags; - - const bool transparent_pointers = !synthetic_array_member; + const bool transparent_pointers = true; CompilerType compiler_type = m_impl_backend->GetCompilerType(); ExecutionContext exe_ctx(m_impl_backend->GetExecutionContextRef()); auto child_compiler_type_or_err = compiler_type.GetChildCompilerTypeAtIndex( &exe_ctx, idx, transparent_pointers, omit_empty_base_classes, - ignore_array_bounds, child_name_str, child_byte_size, child_byte_offset, + ignore_array_bounds, child_name, child_byte_size, child_byte_offset, child_bitfield_bit_size, child_bitfield_bit_offset, child_is_base_class, child_is_deref_of_parent, m_impl_backend, language_flags); - CompilerType child_compiler_type; - if (!child_compiler_type_or_err) + + // One might think we should check that the size of the children + // is always strictly positive, hence we could avoid creating a + // ValueObject if that's not the case, but it turns out there + // are languages out there which allow zero-size types with + // children (e.g. Swift). + if (!child_compiler_type_or_err || !child_compiler_type_or_err->IsValid()) { LLDB_LOG_ERROR(GetLog(LLDBLog::Types), child_compiler_type_or_err.takeError(), "could not find child: {0}"); - else - child_compiler_type = *child_compiler_type_or_err; + return nullptr; + } + lldb::addr_t child_live_addr = LLDB_INVALID_ADDRESS; + // Transfer the live address (with offset) to the child. But if + // the parent is a pointer, the live address is where that pointer + // value lives in memory, so the children live addresses aren't + // offsets from that value, they are just other load addresses that + // are recorded in the Value of the child ValueObjects. + if (m_live_address != LLDB_INVALID_ADDRESS && !compiler_type.IsPointerType()) + child_live_addr = m_live_address + child_byte_offset; + + return new ValueObjectConstResultChild( + *m_impl_backend, *child_compiler_type_or_err, ConstString(child_name), + child_byte_size, child_byte_offset, child_bitfield_bit_size, + child_bitfield_bit_offset, child_is_base_class, child_is_deref_of_parent, + child_live_addr, language_flags); +} + +ValueObject * +ValueObjectConstResultImpl::CreateSyntheticArrayMember(size_t idx) { + if (m_impl_backend == nullptr) + return nullptr; + + m_impl_backend->UpdateValueIfNeeded(false); + + bool omit_empty_base_classes = true; + bool ignore_array_bounds = true; + std::string child_name; + uint32_t child_byte_size = 0; + int32_t child_byte_offset = 0; + uint32_t child_bitfield_bit_size = 0; + uint32_t child_bitfield_bit_offset = 0; + bool child_is_base_class = false; + bool child_is_deref_of_parent = false; + uint64_t language_flags; + + const bool transparent_pointers = false; + CompilerType compiler_type = m_impl_backend->GetCompilerType(); + + ExecutionContext exe_ctx(m_impl_backend->GetExecutionContextRef()); + + auto child_compiler_type_or_err = compiler_type.GetChildCompilerTypeAtIndex( + &exe_ctx, 0, transparent_pointers, omit_empty_base_classes, + ignore_array_bounds, child_name, child_byte_size, child_byte_offset, + child_bitfield_bit_size, child_bitfield_bit_offset, child_is_base_class, + child_is_deref_of_parent, m_impl_backend, language_flags); // One might think we should check that the size of the children // is always strictly positive, hence we could avoid creating a // ValueObject if that's not the case, but it turns out there // are languages out there which allow zero-size types with // children (e.g. Swift). - if (child_compiler_type) { - if (synthetic_index) - child_byte_offset += child_byte_size * synthetic_index; - - ConstString child_name; - if (!child_name_str.empty()) - child_name.SetCString(child_name_str.c_str()); - - lldb::addr_t child_live_addr = LLDB_INVALID_ADDRESS; - // Transfer the live address (with offset) to the child. But if - // the parent is a pointer, the live address is where that pointer - // value lives in memory, so the children live addresses aren't - // offsets from that value, they are just other load addresses that - // are recorded in the Value of the child ValueObjects. - if (m_live_address != LLDB_INVALID_ADDRESS) { - if (!compiler_type.IsPointerType()) - child_live_addr = m_live_address + child_byte_offset; - } - valobj = new ValueObjectConstResultChild( - *m_impl_backend, child_compiler_type, child_name, child_byte_size, - child_byte_offset, child_bitfield_bit_size, child_bitfield_bit_offset, - child_is_base_class, child_is_deref_of_parent, child_live_addr, - language_flags); + if (!child_compiler_type_or_err || !child_compiler_type_or_err->IsValid()) { + LLDB_LOG_ERROR(GetLog(LLDBLog::Types), + child_compiler_type_or_err.takeError(), + "could not find child: {0}"); + return nullptr; } - return valobj; + child_byte_offset += child_byte_size * idx; + + lldb::addr_t child_live_addr = LLDB_INVALID_ADDRESS; + // Transfer the live address (with offset) to the child. But if + // the parent is a pointer, the live address is where that pointer + // value lives in memory, so the children live addresses aren't + // offsets from that value, they are just other load addresses that + // are recorded in the Value of the child ValueObjects. + if (m_live_address != LLDB_INVALID_ADDRESS && !compiler_type.IsPointerType()) + child_live_addr = m_live_address + child_byte_offset; + return new ValueObjectConstResultChild( + *m_impl_backend, *child_compiler_type_or_err, ConstString(child_name), + child_byte_size, child_byte_offset, child_bitfield_bit_size, + child_bitfield_bit_offset, child_is_base_class, child_is_deref_of_parent, + child_live_addr, language_flags); } lldb::ValueObjectSP ValueObjectConstResultImpl::GetSyntheticChildAtOffset( diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/Core/ValueObjectRegister.cpp index 9d9da8bb119d..8b5557a0178b 100644 --- a/lldb/source/Core/ValueObjectRegister.cpp +++ b/lldb/source/Core/ValueObjectRegister.cpp @@ -115,17 +115,13 @@ bool ValueObjectRegisterSet::UpdateValue() { return m_error.Success(); } -ValueObject *ValueObjectRegisterSet::CreateChildAtIndex( - size_t idx, bool synthetic_array_member, int32_t synthetic_index) { - ValueObject *valobj = nullptr; +ValueObject *ValueObjectRegisterSet::CreateChildAtIndex(size_t idx) { if (m_reg_ctx_sp && m_reg_set) { - uint32_t num_children = GetNumChildrenIgnoringErrors(); - if (idx < num_children) - valobj = new ValueObjectRegister( - *this, m_reg_ctx_sp, - m_reg_ctx_sp->GetRegisterInfoAtIndex(m_reg_set->registers[idx])); + return new ValueObjectRegister( + *this, m_reg_ctx_sp, + m_reg_ctx_sp->GetRegisterInfoAtIndex(m_reg_set->registers[idx])); } - return valobj; + return nullptr; } lldb::ValueObjectSP diff --git a/lldb/source/Core/ValueObjectVTable.cpp b/lldb/source/Core/ValueObjectVTable.cpp index 9bfe8dc9700d..72e198d08c9c 100644 --- a/lldb/source/Core/ValueObjectVTable.cpp +++ b/lldb/source/Core/ValueObjectVTable.cpp @@ -185,11 +185,7 @@ ConstString ValueObjectVTable::GetDisplayTypeName() { bool ValueObjectVTable::IsInScope() { return GetParent()->IsInScope(); } -ValueObject *ValueObjectVTable::CreateChildAtIndex(size_t idx, - bool synthetic_array_member, - int32_t synthetic_index) { - if (synthetic_array_member) - return nullptr; +ValueObject *ValueObjectVTable::CreateChildAtIndex(size_t idx) { return new ValueObjectVTableChild(*this, idx, m_addr_size); } -- GitLab From c007883f0286a314eb69976ad14da2bce988fb55 Mon Sep 17 00:00:00 2001 From: Freddy Ye Date: Fri, 7 Jun 2024 14:44:46 +0800 Subject: [PATCH 198/462] Revert "[X86] Assign AVX10_1 feature priority to align with gcc. (#94557)" (#94730) This reverts commit d843c02aba5d253891ea0e599c4b6e25247c1af7. --- llvm/include/llvm/TargetParser/X86TargetParser.def | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.def b/llvm/include/llvm/TargetParser/X86TargetParser.def index f10cede0734e..8daa8a689c95 100644 --- a/llvm/include/llvm/TargetParser/X86TargetParser.def +++ b/llvm/include/llvm/TargetParser/X86TargetParser.def @@ -250,8 +250,8 @@ X86_FEATURE_COMPAT(SHA512, "sha512", 0) X86_FEATURE_COMPAT(SM4, "sm4", 0) X86_FEATURE (EGPR, "egpr") X86_FEATURE_COMPAT(USERMSR, "usermsr", 0) -X86_FEATURE_COMPAT(AVX10_1, "avx10.1-256", 36) -X86_FEATURE_COMPAT(AVX10_1_512, "avx10.1-512", 37) +X86_FEATURE_COMPAT(AVX10_1, "avx10.1-256", 0) +X86_FEATURE_COMPAT(AVX10_1_512, "avx10.1-512", 0) // These features aren't really CPU features, but the frontend can set them. X86_FEATURE (RETPOLINE_EXTERNAL_THUNK, "retpoline-external-thunk") X86_FEATURE (RETPOLINE_INDIRECT_BRANCHES, "retpoline-indirect-branches") -- GitLab From b7d976d4e520a7acba71e006e1518d147a862138 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 6 Jun 2024 23:49:05 -0700 Subject: [PATCH 199/462] [memprof] Use std::move in ContextEdge::ContextEdge (NFC) (#94687) Since the constructor of ContextEdge takes ContextIds by value, we should move it to the corresponding member variable as suggested by clang-tidy's performance-unnecessary-value-param. While we are at it, this patch updates a couple of callers. To avoid the ambiguity in the evaluation order among the constructor arguments, I'm calling computeAllocType before calling the constructor. --- .../Transforms/IPO/MemProfContextDisambiguation.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp index f033d2b0d6d0..b58b906465e5 100644 --- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp +++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp @@ -402,7 +402,7 @@ public: ContextEdge(ContextNode *Callee, ContextNode *Caller, uint8_t AllocType, DenseSet ContextIds) : Callee(Callee), Caller(Caller), AllocTypes(AllocType), - ContextIds(ContextIds) {} + ContextIds(std::move(ContextIds)) {} DenseSet &getContextIds() { return ContextIds; } @@ -1127,15 +1127,15 @@ void CallsiteContextGraph::connectNewNode( continue; } if (TowardsCallee) { + uint8_t NewAllocType = computeAllocType(NewEdgeContextIds); auto NewEdge = std::make_shared( - Edge->Callee, NewNode, computeAllocType(NewEdgeContextIds), - NewEdgeContextIds); + Edge->Callee, NewNode, NewAllocType, std::move(NewEdgeContextIds)); NewNode->CalleeEdges.push_back(NewEdge); NewEdge->Callee->CallerEdges.push_back(NewEdge); } else { + uint8_t NewAllocType = computeAllocType(NewEdgeContextIds); auto NewEdge = std::make_shared( - NewNode, Edge->Caller, computeAllocType(NewEdgeContextIds), - NewEdgeContextIds); + NewNode, Edge->Caller, NewAllocType, std::move(NewEdgeContextIds)); NewNode->CallerEdges.push_back(NewEdge); NewEdge->Caller->CalleeEdges.push_back(NewEdge); } -- GitLab From 4a7b8003012985028bdf637be0a9c7a71dff65d9 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Fri, 7 Jun 2024 16:22:47 +1000 Subject: [PATCH 200/462] [ORC] Switch ExecutionSession::ErrorReporter to use unique_function. This allows the ReportError functor to hold move-only types. --- llvm/include/llvm/ExecutionEngine/Orc/Core.h | 2 +- .../ExecutionEngine/Orc/CoreAPIsTest.cpp | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h index bac923aba02a..6ce6affe38e0 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h @@ -1438,7 +1438,7 @@ class ExecutionSession { public: /// For reporting errors. - using ErrorReporter = std::function; + using ErrorReporter = unique_function; /// Send a result to the remote. using SendResultFunction = unique_function; diff --git a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp index 53a74c833eb3..39a49eb17999 100644 --- a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp @@ -24,6 +24,32 @@ class CoreAPIsStandardTest : public CoreAPIsBasedStandardTest {}; namespace { +class CustomError : public ErrorInfo { +public: + static char ID; + void log(raw_ostream &OS) const override { OS << "CustomError"; } + std::error_code convertToErrorCode() const override { return {}; } +}; +char CustomError::ID = 0; + +TEST_F(CoreAPIsStandardTest, ErrorReporter) { + // Check that errors reported via ExecutionSession::reportError are sent to + // the registered error reporter, and that the error reporter can hold + // uniquely owned state. + + Error ReportedError = Error::success(); + + ES.setErrorReporter( + // Make sure error reporter can capture uniquely-owned state. + [&, State = std::make_unique(42)](Error Err) { + ReportedError = joinErrors(std::move(Err), std::move(ReportedError)); + }); + + ES.reportError(make_error()); + + EXPECT_THAT_ERROR(std::move(ReportedError), Failed()); +} + TEST_F(CoreAPIsStandardTest, JITDylibAddToLinkOrder) { // Check that the JITDylib::addToLinkOrder methods behave as expected. auto &JD2 = ES.createBareJITDylib("JD2"); -- GitLab From f21c2fae95f76300c4230f75b6e97445932f6bc6 Mon Sep 17 00:00:00 2001 From: hev Date: Fri, 7 Jun 2024 15:21:44 +0800 Subject: [PATCH 201/462] [LoongArch] Set isReMaterializable on LU{12,32,52}I.D/ADDI.D and {X}ORI instructions (#94552) --- .../Target/LoongArch/LoongArchInstrInfo.td | 13 +++- llvm/test/CodeGen/LoongArch/vector-fp-imm.ll | 73 +++++++++---------- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td index 66bd74e068b9..f72f46e39e2a 100644 --- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td +++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td @@ -734,7 +734,9 @@ def ADD_W : ALU_3R<0x00100000>; def SUB_W : ALU_3R<0x00110000>; def ADDI_W : ALU_2RI12<0x02800000, simm12_addlike>; def ALSL_W : ALU_3RI2<0x00040000, uimm2_plus1>; +let isReMaterializable = 1 in { def LU12I_W : ALU_1RI20<0x14000000, simm20_lu12iw>; +} def SLT : ALU_3R<0x00120000>; def SLTU : ALU_3R<0x00128000>; def SLTI : ALU_2RI12<0x02000000, simm12>; @@ -749,8 +751,10 @@ def XOR : ALU_3R<0x00158000>; def ANDN : ALU_3R<0x00168000>; def ORN : ALU_3R<0x00160000>; def ANDI : ALU_2RI12<0x03400000, uimm12>; +let isReMaterializable = 1 in { def ORI : ALU_2RI12<0x03800000, uimm12_ori>; def XORI : ALU_2RI12<0x03c00000, uimm12>; +} def MUL_W : ALU_3R<0x001c0000>; def MULH_W : ALU_3R<0x001c8000>; def MULH_WU : ALU_3R<0x001d0000>; @@ -852,17 +856,24 @@ let Predicates = [IsLA64] in { // Arithmetic Operation Instructions for 64-bits def ADD_D : ALU_3R<0x00108000>; def SUB_D : ALU_3R<0x00118000>; +// ADDI_D isn't always rematerializable, but isReMaterializable will be used as +// a hint which is verified in isReallyTriviallyReMaterializable. +let isReMaterializable = 1 in { def ADDI_D : ALU_2RI12<0x02c00000, simm12_addlike>; +} def ADDU16I_D : ALU_2RI16<0x10000000, simm16>; def ALSL_WU : ALU_3RI2<0x00060000, uimm2_plus1>; def ALSL_D : ALU_3RI2<0x002c0000, uimm2_plus1>; let Constraints = "$rd = $dst" in { -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +let hasSideEffects = 0, mayLoad = 0, mayStore = 0, + isReMaterializable = 1 in def LU32I_D : Fmt1RI20<0x16000000, (outs GPR:$dst), (ins GPR:$rd, simm20_lu32id:$imm20), "$rd, $imm20">; } +let isReMaterializable = 1 in { def LU52I_D : ALU_2RI12<0x03000000, simm12_lu52id>; +} def PCADDU18I : ALU_1RI20<0x1e000000, simm20_pcaddu18i>; def MUL_D : ALU_3R<0x001d8000>; def MULH_D : ALU_3R<0x001e0000>; diff --git a/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll b/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll index 18d17751a771..0a401ebe5f6b 100644 --- a/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll +++ b/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll @@ -742,12 +742,10 @@ define void @test_d8(ptr %P, ptr %S) nounwind { ; LA32F-NEXT: st.w $a2, $sp, 32 # 4-byte Folded Spill ; LA32F-NEXT: ld.w $a2, $a0, 52 ; LA32F-NEXT: st.w $a2, $sp, 28 # 4-byte Folded Spill -; LA32F-NEXT: ld.w $a2, $a0, 40 -; LA32F-NEXT: st.w $a2, $sp, 16 # 4-byte Folded Spill +; LA32F-NEXT: ld.w $s8, $a0, 40 ; LA32F-NEXT: ld.w $a2, $a0, 44 -; LA32F-NEXT: st.w $a2, $sp, 12 # 4-byte Folded Spill -; LA32F-NEXT: ld.w $a2, $a0, 32 -; LA32F-NEXT: st.w $a2, $sp, 0 # 4-byte Folded Spill +; LA32F-NEXT: st.w $a2, $sp, 16 # 4-byte Folded Spill +; LA32F-NEXT: ld.w $s3, $a0, 32 ; LA32F-NEXT: ld.w $s4, $a0, 36 ; LA32F-NEXT: ld.w $s5, $a0, 24 ; LA32F-NEXT: ld.w $s6, $a0, 28 @@ -756,80 +754,77 @@ define void @test_d8(ptr %P, ptr %S) nounwind { ; LA32F-NEXT: ld.w $s7, $a0, 8 ; LA32F-NEXT: ld.w $s0, $a0, 12 ; LA32F-NEXT: ld.w $a2, $a0, 0 -; LA32F-NEXT: ld.w $a3, $a0, 4 +; LA32F-NEXT: ld.w $a4, $a0, 4 ; LA32F-NEXT: move $fp, $a1 -; LA32F-NEXT: lu12i.w $s8, 261888 +; LA32F-NEXT: lu12i.w $a3, 261888 ; LA32F-NEXT: move $a0, $a2 -; LA32F-NEXT: move $a1, $a3 +; LA32F-NEXT: move $a1, $a4 ; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s8 ; LA32F-NEXT: bl %plt(__adddf3) ; LA32F-NEXT: st.w $a0, $sp, 40 # 4-byte Folded Spill ; LA32F-NEXT: st.w $a1, $sp, 36 # 4-byte Folded Spill -; LA32F-NEXT: lu12i.w $s3, 262144 +; LA32F-NEXT: lu12i.w $a3, 262144 ; LA32F-NEXT: move $a0, $s7 ; LA32F-NEXT: move $a1, $s0 ; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s3 +; LA32F-NEXT: move $s0, $a3 ; LA32F-NEXT: bl %plt(__adddf3) ; LA32F-NEXT: st.w $a0, $sp, 24 # 4-byte Folded Spill ; LA32F-NEXT: st.w $a1, $sp, 20 # 4-byte Folded Spill -; LA32F-NEXT: lu12i.w $s0, 262272 +; LA32F-NEXT: lu12i.w $s7, 262272 ; LA32F-NEXT: move $a0, $s1 ; LA32F-NEXT: move $a1, $s2 ; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s0 +; LA32F-NEXT: move $a3, $s7 ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $sp, 8 # 4-byte Folded Spill -; LA32F-NEXT: st.w $a1, $sp, 4 # 4-byte Folded Spill -; LA32F-NEXT: lu12i.w $s7, 262400 +; LA32F-NEXT: st.w $a0, $sp, 12 # 4-byte Folded Spill +; LA32F-NEXT: move $s2, $a1 +; LA32F-NEXT: lu12i.w $a3, 262400 ; LA32F-NEXT: move $a0, $s5 ; LA32F-NEXT: move $a1, $s6 ; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s7 ; LA32F-NEXT: bl %plt(__adddf3) ; LA32F-NEXT: move $s5, $a0 ; LA32F-NEXT: move $s6, $a1 -; LA32F-NEXT: ld.w $a0, $sp, 0 # 4-byte Folded Reload +; LA32F-NEXT: move $a0, $s3 ; LA32F-NEXT: move $a1, $s4 ; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s8 +; LA32F-NEXT: lu12i.w $a3, 261888 ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: move $s4, $a0 -; LA32F-NEXT: move $s8, $a1 -; LA32F-NEXT: ld.w $a0, $sp, 16 # 4-byte Folded Reload -; LA32F-NEXT: ld.w $a1, $sp, 12 # 4-byte Folded Reload +; LA32F-NEXT: move $s3, $a0 +; LA32F-NEXT: move $s4, $a1 +; LA32F-NEXT: move $a0, $s8 +; LA32F-NEXT: ld.w $a1, $sp, 16 # 4-byte Folded Reload ; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s3 +; LA32F-NEXT: move $a3, $s0 ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: move $s3, $a0 -; LA32F-NEXT: move $s1, $a1 +; LA32F-NEXT: move $s8, $a0 +; LA32F-NEXT: move $s0, $a1 ; LA32F-NEXT: ld.w $a0, $sp, 32 # 4-byte Folded Reload ; LA32F-NEXT: ld.w $a1, $sp, 28 # 4-byte Folded Reload ; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s0 +; LA32F-NEXT: move $a3, $s7 ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: move $s0, $a0 -; LA32F-NEXT: move $s2, $a1 +; LA32F-NEXT: move $s7, $a0 +; LA32F-NEXT: move $s1, $a1 ; LA32F-NEXT: ld.w $a0, $sp, 48 # 4-byte Folded Reload ; LA32F-NEXT: ld.w $a1, $sp, 44 # 4-byte Folded Reload ; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s7 +; LA32F-NEXT: lu12i.w $a3, 262400 ; LA32F-NEXT: bl %plt(__adddf3) ; LA32F-NEXT: st.w $a0, $fp, 56 ; LA32F-NEXT: st.w $a1, $fp, 60 -; LA32F-NEXT: st.w $s0, $fp, 48 -; LA32F-NEXT: st.w $s2, $fp, 52 -; LA32F-NEXT: st.w $s3, $fp, 40 -; LA32F-NEXT: st.w $s1, $fp, 44 -; LA32F-NEXT: st.w $s4, $fp, 32 -; LA32F-NEXT: st.w $s8, $fp, 36 +; LA32F-NEXT: st.w $s7, $fp, 48 +; LA32F-NEXT: st.w $s1, $fp, 52 +; LA32F-NEXT: st.w $s8, $fp, 40 +; LA32F-NEXT: st.w $s0, $fp, 44 +; LA32F-NEXT: st.w $s3, $fp, 32 +; LA32F-NEXT: st.w $s4, $fp, 36 ; LA32F-NEXT: st.w $s5, $fp, 24 ; LA32F-NEXT: st.w $s6, $fp, 28 -; LA32F-NEXT: ld.w $a0, $sp, 8 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $a0, $sp, 12 # 4-byte Folded Reload ; LA32F-NEXT: st.w $a0, $fp, 16 -; LA32F-NEXT: ld.w $a0, $sp, 4 # 4-byte Folded Reload -; LA32F-NEXT: st.w $a0, $fp, 20 +; LA32F-NEXT: st.w $s2, $fp, 20 ; LA32F-NEXT: ld.w $a0, $sp, 24 # 4-byte Folded Reload ; LA32F-NEXT: st.w $a0, $fp, 8 ; LA32F-NEXT: ld.w $a0, $sp, 20 # 4-byte Folded Reload -- GitLab From d224a034a20016272a3257f7081d02f90eb9cc78 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 7 Jun 2024 09:24:58 +0200 Subject: [PATCH 202/462] [SCEV] Use insert_or_assign() (NFC) --- llvm/include/llvm/Analysis/ScalarEvolution.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h index 72f3d9454249..b273d118c4c7 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -1656,9 +1656,7 @@ private: DenseMap &Cache = Hint == HINT_RANGE_UNSIGNED ? UnsignedRanges : SignedRanges; - auto Pair = Cache.try_emplace(S, std::move(CR)); - if (!Pair.second) - Pair.first->second = std::move(CR); + auto Pair = Cache.insert_or_assign(S, std::move(CR)); return Pair.first->second; } -- GitLab From 240512c43234f58b3924cd90fe0781445d97e98d Mon Sep 17 00:00:00 2001 From: hev Date: Fri, 7 Jun 2024 15:39:05 +0800 Subject: [PATCH 203/462] [LoongArch] Add a pass to rewrite rd to r0 for non-computational instrs whose return values are unused (#94590) This patch adds a peephole pass `LoongArchDeadRegisterDefinitions`. It rewrites `rd` to `r0` when `rd` is marked as dead. It may improve the register allocation and reduce pipeline hazards on CPUs without register renaming and OOO. --- llvm/lib/Target/LoongArch/CMakeLists.txt | 1 + llvm/lib/Target/LoongArch/LoongArch.h | 2 + .../LoongArchDeadRegisterDefinitions.cpp | 108 ++++++++++++++++++ .../LoongArch/LoongArchTargetMachine.cpp | 24 ++++ llvm/test/CodeGen/LoongArch/global-address.ll | 24 ++-- llvm/test/CodeGen/LoongArch/intrinsic-la64.ll | 6 +- llvm/test/CodeGen/LoongArch/intrinsic.ll | 10 +- .../CodeGen/LoongArch/ir-instruction/br.ll | 44 +++---- .../LoongArch/ir-instruction/load-store.ll | 96 ++++++++-------- .../LoongArch/psabi-restricted-scheduling.ll | 40 +++---- .../llvm/lib/Target/LoongArch/BUILD.gn | 1 + 11 files changed, 246 insertions(+), 110 deletions(-) create mode 100644 llvm/lib/Target/LoongArch/LoongArchDeadRegisterDefinitions.cpp diff --git a/llvm/lib/Target/LoongArch/CMakeLists.txt b/llvm/lib/Target/LoongArch/CMakeLists.txt index 5085e23f82a7..cadc335a621f 100644 --- a/llvm/lib/Target/LoongArch/CMakeLists.txt +++ b/llvm/lib/Target/LoongArch/CMakeLists.txt @@ -16,6 +16,7 @@ add_public_tablegen_target(LoongArchCommonTableGen) add_llvm_target(LoongArchCodeGen LoongArchAsmPrinter.cpp + LoongArchDeadRegisterDefinitions.cpp LoongArchExpandAtomicPseudoInsts.cpp LoongArchExpandPseudoInsts.cpp LoongArchFrameLowering.cpp diff --git a/llvm/lib/Target/LoongArch/LoongArch.h b/llvm/lib/Target/LoongArch/LoongArch.h index 0928ea31054a..adfb844ee31b 100644 --- a/llvm/lib/Target/LoongArch/LoongArch.h +++ b/llvm/lib/Target/LoongArch/LoongArch.h @@ -33,12 +33,14 @@ bool lowerLoongArchMachineOperandToMCOperand(const MachineOperand &MO, MCOperand &MCOp, const AsmPrinter &AP); +FunctionPass *createLoongArchDeadRegisterDefinitionsPass(); FunctionPass *createLoongArchExpandAtomicPseudoPass(); FunctionPass *createLoongArchISelDag(LoongArchTargetMachine &TM); FunctionPass *createLoongArchOptWInstrsPass(); FunctionPass *createLoongArchPreRAExpandPseudoPass(); FunctionPass *createLoongArchExpandPseudoPass(); void initializeLoongArchDAGToDAGISelLegacyPass(PassRegistry &); +void initializeLoongArchDeadRegisterDefinitionsPass(PassRegistry &); void initializeLoongArchExpandAtomicPseudoPass(PassRegistry &); void initializeLoongArchOptWInstrsPass(PassRegistry &); void initializeLoongArchPreRAExpandPseudoPass(PassRegistry &); diff --git a/llvm/lib/Target/LoongArch/LoongArchDeadRegisterDefinitions.cpp b/llvm/lib/Target/LoongArch/LoongArchDeadRegisterDefinitions.cpp new file mode 100644 index 000000000000..6ff9b6ac25b7 --- /dev/null +++ b/llvm/lib/Target/LoongArch/LoongArchDeadRegisterDefinitions.cpp @@ -0,0 +1,108 @@ +//=== LoongArchDeadRegisterDefinitions.cpp - Replace dead defs w/ zero reg ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// +// +// This pass rewrites Rd to r0 for instrs whose return values are unused. +// +//===---------------------------------------------------------------------===// + +#include "LoongArch.h" +#include "LoongArchInstrInfo.h" +#include "LoongArchSubtarget.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/LiveDebugVariables.h" +#include "llvm/CodeGen/LiveIntervals.h" +#include "llvm/CodeGen/LiveStacks.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" + +using namespace llvm; +#define DEBUG_TYPE "loongarch-dead-defs" +#define LoongArch_DEAD_REG_DEF_NAME "LoongArch Dead register definitions" + +STATISTIC(NumDeadDefsReplaced, "Number of dead definitions replaced"); + +namespace { +class LoongArchDeadRegisterDefinitions : public MachineFunctionPass { +public: + static char ID; + + LoongArchDeadRegisterDefinitions() : MachineFunctionPass(ID) {} + bool runOnMachineFunction(MachineFunction &MF) override; + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + AU.addRequired(); + AU.addPreserved(); + AU.addRequired(); + AU.addPreserved(); + AU.addPreserved(); + AU.addPreserved(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + StringRef getPassName() const override { return LoongArch_DEAD_REG_DEF_NAME; } +}; +} // end anonymous namespace + +char LoongArchDeadRegisterDefinitions::ID = 0; +INITIALIZE_PASS(LoongArchDeadRegisterDefinitions, DEBUG_TYPE, + LoongArch_DEAD_REG_DEF_NAME, false, false) + +FunctionPass *llvm::createLoongArchDeadRegisterDefinitionsPass() { + return new LoongArchDeadRegisterDefinitions(); +} + +bool LoongArchDeadRegisterDefinitions::runOnMachineFunction( + MachineFunction &MF) { + if (skipFunction(MF.getFunction())) + return false; + + const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); + const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); + LiveIntervals &LIS = getAnalysis(); + LLVM_DEBUG(dbgs() << "***** LoongArchDeadRegisterDefinitions *****\n"); + + bool MadeChange = false; + for (MachineBasicBlock &MBB : MF) { + for (MachineInstr &MI : MBB) { + // We only handle non-computational instructions. + const MCInstrDesc &Desc = MI.getDesc(); + if (!Desc.mayLoad() && !Desc.mayStore() && + !Desc.hasUnmodeledSideEffects()) + continue; + for (int I = 0, E = Desc.getNumDefs(); I != E; ++I) { + MachineOperand &MO = MI.getOperand(I); + if (!MO.isReg() || !MO.isDef() || MO.isEarlyClobber()) + continue; + // Be careful not to change the register if it's a tied operand. + if (MI.isRegTiedToUseOperand(I)) { + LLVM_DEBUG(dbgs() << " Ignoring, def is tied operand.\n"); + continue; + } + Register Reg = MO.getReg(); + if (!Reg.isVirtual() || !MO.isDead()) + continue; + LLVM_DEBUG(dbgs() << " Dead def operand #" << I << " in:\n "; + MI.print(dbgs())); + const TargetRegisterClass *RC = TII->getRegClass(Desc, I, TRI, MF); + if (!(RC && RC->contains(LoongArch::R0))) { + LLVM_DEBUG(dbgs() << " Ignoring, register is not a GPR.\n"); + continue; + } + assert(LIS.hasInterval(Reg)); + LIS.removeInterval(Reg); + MO.setReg(LoongArch::R0); + LLVM_DEBUG(dbgs() << " Replacing with zero register. New:\n "; + MI.print(dbgs())); + ++NumDeadDefsReplaced; + MadeChange = true; + } + } + } + + return MadeChange; +} diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp index c29c1b593321..137fe1d04f45 100644 --- a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp @@ -34,11 +34,19 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeLoongArchTarget() { RegisterTargetMachine X(getTheLoongArch32Target()); RegisterTargetMachine Y(getTheLoongArch64Target()); auto *PR = PassRegistry::getPassRegistry(); + initializeLoongArchDeadRegisterDefinitionsPass(*PR); initializeLoongArchOptWInstrsPass(*PR); initializeLoongArchPreRAExpandPseudoPass(*PR); initializeLoongArchDAGToDAGISelLegacyPass(*PR); } +static cl::opt EnableLoongArchDeadRegisterElimination( + "loongarch-enable-dead-defs", cl::Hidden, + cl::desc("Enable the pass that removes dead" + " definitons and replaces stores to" + " them with stores to r0"), + cl::init(true)); + static cl::opt EnableLoopDataPrefetch("loongarch-enable-loop-data-prefetch", cl::Hidden, cl::desc("Enable the loop data prefetch pass"), @@ -148,6 +156,8 @@ public: void addPreEmitPass2() override; void addMachineSSAOptimization() override; void addPreRegAlloc() override; + bool addRegAssignAndRewriteFast() override; + bool addRegAssignAndRewriteOptimized() override; }; } // end namespace @@ -200,3 +210,17 @@ void LoongArchPassConfig::addMachineSSAOptimization() { void LoongArchPassConfig::addPreRegAlloc() { addPass(createLoongArchPreRAExpandPseudoPass()); } + +bool LoongArchPassConfig::addRegAssignAndRewriteFast() { + if (TM->getOptLevel() != CodeGenOptLevel::None && + EnableLoongArchDeadRegisterElimination) + addPass(createLoongArchDeadRegisterDefinitionsPass()); + return TargetPassConfig::addRegAssignAndRewriteFast(); +} + +bool LoongArchPassConfig::addRegAssignAndRewriteOptimized() { + if (TM->getOptLevel() != CodeGenOptLevel::None && + EnableLoongArchDeadRegisterElimination) + addPass(createLoongArchDeadRegisterDefinitionsPass()); + return TargetPassConfig::addRegAssignAndRewriteOptimized(); +} diff --git a/llvm/test/CodeGen/LoongArch/global-address.ll b/llvm/test/CodeGen/LoongArch/global-address.ll index 0c8958b6ab33..fb2929572f31 100644 --- a/llvm/test/CodeGen/LoongArch/global-address.ll +++ b/llvm/test/CodeGen/LoongArch/global-address.ll @@ -14,40 +14,40 @@ define void @foo() nounwind { ; LA32NOPIC: # %bb.0: ; LA32NOPIC-NEXT: pcalau12i $a0, %got_pc_hi20(G) ; LA32NOPIC-NEXT: ld.w $a0, $a0, %got_pc_lo12(G) -; LA32NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA32NOPIC-NEXT: ld.w $zero, $a0, 0 ; LA32NOPIC-NEXT: pcalau12i $a0, %pc_hi20(g) ; LA32NOPIC-NEXT: addi.w $a0, $a0, %pc_lo12(g) -; LA32NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA32NOPIC-NEXT: ld.w $zero, $a0, 0 ; LA32NOPIC-NEXT: ret ; ; LA32PIC-LABEL: foo: ; LA32PIC: # %bb.0: ; LA32PIC-NEXT: pcalau12i $a0, %got_pc_hi20(G) ; LA32PIC-NEXT: ld.w $a0, $a0, %got_pc_lo12(G) -; LA32PIC-NEXT: ld.w $a0, $a0, 0 +; LA32PIC-NEXT: ld.w $zero, $a0, 0 ; LA32PIC-NEXT: pcalau12i $a0, %pc_hi20(.Lg$local) ; LA32PIC-NEXT: addi.w $a0, $a0, %pc_lo12(.Lg$local) -; LA32PIC-NEXT: ld.w $a0, $a0, 0 +; LA32PIC-NEXT: ld.w $zero, $a0, 0 ; LA32PIC-NEXT: ret ; ; LA64NOPIC-LABEL: foo: ; LA64NOPIC: # %bb.0: ; LA64NOPIC-NEXT: pcalau12i $a0, %got_pc_hi20(G) ; LA64NOPIC-NEXT: ld.d $a0, $a0, %got_pc_lo12(G) -; LA64NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA64NOPIC-NEXT: ld.w $zero, $a0, 0 ; LA64NOPIC-NEXT: pcalau12i $a0, %pc_hi20(g) ; LA64NOPIC-NEXT: addi.d $a0, $a0, %pc_lo12(g) -; LA64NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA64NOPIC-NEXT: ld.w $zero, $a0, 0 ; LA64NOPIC-NEXT: ret ; ; LA64PIC-LABEL: foo: ; LA64PIC: # %bb.0: ; LA64PIC-NEXT: pcalau12i $a0, %got_pc_hi20(G) ; LA64PIC-NEXT: ld.d $a0, $a0, %got_pc_lo12(G) -; LA64PIC-NEXT: ld.w $a0, $a0, 0 +; LA64PIC-NEXT: ld.w $zero, $a0, 0 ; LA64PIC-NEXT: pcalau12i $a0, %pc_hi20(.Lg$local) ; LA64PIC-NEXT: addi.d $a0, $a0, %pc_lo12(.Lg$local) -; LA64PIC-NEXT: ld.w $a0, $a0, 0 +; LA64PIC-NEXT: ld.w $zero, $a0, 0 ; LA64PIC-NEXT: ret ; ; LA64LARGENOPIC-LABEL: foo: @@ -57,13 +57,13 @@ define void @foo() nounwind { ; LA64LARGENOPIC-NEXT: lu32i.d $t8, %got64_pc_lo20(G) ; LA64LARGENOPIC-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(G) ; LA64LARGENOPIC-NEXT: ldx.d $a0, $t8, $a0 -; LA64LARGENOPIC-NEXT: ld.w $a0, $a0, 0 +; LA64LARGENOPIC-NEXT: ld.w $zero, $a0, 0 ; LA64LARGENOPIC-NEXT: pcalau12i $a0, %pc_hi20(g) ; LA64LARGENOPIC-NEXT: addi.d $t8, $zero, %pc_lo12(g) ; LA64LARGENOPIC-NEXT: lu32i.d $t8, %pc64_lo20(g) ; LA64LARGENOPIC-NEXT: lu52i.d $t8, $t8, %pc64_hi12(g) ; LA64LARGENOPIC-NEXT: add.d $a0, $t8, $a0 -; LA64LARGENOPIC-NEXT: ld.w $a0, $a0, 0 +; LA64LARGENOPIC-NEXT: ld.w $zero, $a0, 0 ; LA64LARGENOPIC-NEXT: ret ; ; LA64LARGEPIC-LABEL: foo: @@ -73,13 +73,13 @@ define void @foo() nounwind { ; LA64LARGEPIC-NEXT: lu32i.d $t8, %got64_pc_lo20(G) ; LA64LARGEPIC-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(G) ; LA64LARGEPIC-NEXT: ldx.d $a0, $t8, $a0 -; LA64LARGEPIC-NEXT: ld.w $a0, $a0, 0 +; LA64LARGEPIC-NEXT: ld.w $zero, $a0, 0 ; LA64LARGEPIC-NEXT: pcalau12i $a0, %pc_hi20(.Lg$local) ; LA64LARGEPIC-NEXT: addi.d $t8, $zero, %pc_lo12(.Lg$local) ; LA64LARGEPIC-NEXT: lu32i.d $t8, %pc64_lo20(.Lg$local) ; LA64LARGEPIC-NEXT: lu52i.d $t8, $t8, %pc64_hi12(.Lg$local) ; LA64LARGEPIC-NEXT: add.d $a0, $t8, $a0 -; LA64LARGEPIC-NEXT: ld.w $a0, $a0, 0 +; LA64LARGEPIC-NEXT: ld.w $zero, $a0, 0 ; LA64LARGEPIC-NEXT: ret %V = load volatile i32, ptr @G %v = load volatile i32, ptr @g diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll b/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll index 4a59e2af533e..d94da34e14c4 100644 --- a/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll +++ b/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll @@ -178,7 +178,7 @@ entry: define void @csrrd_d_noret() { ; CHECK-LABEL: csrrd_d_noret: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: csrrd $a0, 1 +; CHECK-NEXT: csrrd $zero, 1 ; CHECK-NEXT: ret entry: %0 = tail call i64 @llvm.loongarch.csrrd.d(i32 1) @@ -240,7 +240,7 @@ entry: define void @iocsrrd_d_noret(i32 %a) { ; CHECK-LABEL: iocsrrd_d_noret: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: iocsrrd.d $a0, $a0 +; CHECK-NEXT: iocsrrd.d $zero, $a0 ; CHECK-NEXT: ret entry: %0 = tail call i64 @llvm.loongarch.iocsrrd.d(i32 %a) @@ -290,7 +290,7 @@ entry: define void @lddir_d_noret(i64 %a) { ; CHECK-LABEL: lddir_d_noret: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: lddir $a0, $a0, 1 +; CHECK-NEXT: lddir $zero, $a0, 1 ; CHECK-NEXT: ret entry: %0 = tail call i64 @llvm.loongarch.lddir.d(i64 %a, i64 1) diff --git a/llvm/test/CodeGen/LoongArch/intrinsic.ll b/llvm/test/CodeGen/LoongArch/intrinsic.ll index f49a2500ad3c..6d8adb5da926 100644 --- a/llvm/test/CodeGen/LoongArch/intrinsic.ll +++ b/llvm/test/CodeGen/LoongArch/intrinsic.ll @@ -73,7 +73,7 @@ entry: define void @movfcsr2gr_noret() nounwind { ; CHECK-LABEL: movfcsr2gr_noret: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: movfcsr2gr $a0, $fcsr1 +; CHECK-NEXT: movfcsr2gr $zero, $fcsr1 ; CHECK-NEXT: ret entry: %res = call i32 @llvm.loongarch.movfcsr2gr(i32 1) @@ -103,7 +103,7 @@ entry: define void @csrrd_w_noret() { ; CHECK-LABEL: csrrd_w_noret: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: csrrd $a0, 1 +; CHECK-NEXT: csrrd $zero, 1 ; CHECK-NEXT: ret entry: %0 = tail call i32 @llvm.loongarch.csrrd.w(i32 1) @@ -185,7 +185,7 @@ entry: define void @iocsrrd_b_noret(i32 %a) { ; CHECK-LABEL: iocsrrd_b_noret: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: iocsrrd.b $a0, $a0 +; CHECK-NEXT: iocsrrd.b $zero, $a0 ; CHECK-NEXT: ret entry: %0 = tail call i32 @llvm.loongarch.iocsrrd.b(i32 %a) @@ -195,7 +195,7 @@ entry: define void @iocsrrd_h_noret(i32 %a) { ; CHECK-LABEL: iocsrrd_h_noret: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: iocsrrd.h $a0, $a0 +; CHECK-NEXT: iocsrrd.h $zero, $a0 ; CHECK-NEXT: ret entry: %0 = tail call i32 @llvm.loongarch.iocsrrd.h(i32 %a) @@ -205,7 +205,7 @@ entry: define void @iocsrrd_w_noret(i32 %a) { ; CHECK-LABEL: iocsrrd_w_noret: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: iocsrrd.w $a0, $a0 +; CHECK-NEXT: iocsrrd.w $zero, $a0 ; CHECK-NEXT: ret entry: %0 = tail call i32 @llvm.loongarch.iocsrrd.w(i32 %a) diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/br.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/br.ll index 02375a925723..8b909e3314d6 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/br.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/br.ll @@ -21,7 +21,7 @@ define void @foo_br_eq(i32 %a, ptr %b) nounwind { ; LA32-NEXT: ld.w $a2, $a1, 0 ; LA32-NEXT: beq $a2, $a0, .LBB1_2 ; LA32-NEXT: # %bb.1: # %test -; LA32-NEXT: ld.w $a0, $a1, 0 +; LA32-NEXT: ld.w $zero, $a1, 0 ; LA32-NEXT: .LBB1_2: # %end ; LA32-NEXT: ret ; @@ -31,7 +31,7 @@ define void @foo_br_eq(i32 %a, ptr %b) nounwind { ; LA64-NEXT: addi.w $a0, $a0, 0 ; LA64-NEXT: beq $a2, $a0, .LBB1_2 ; LA64-NEXT: # %bb.1: # %test -; LA64-NEXT: ld.w $a0, $a1, 0 +; LA64-NEXT: ld.w $zero, $a1, 0 ; LA64-NEXT: .LBB1_2: # %end ; LA64-NEXT: ret %val = load volatile i32, ptr %b @@ -51,7 +51,7 @@ define void @foo_br_ne(i32 %a, ptr %b) nounwind { ; LA32-NEXT: ld.w $a2, $a1, 0 ; LA32-NEXT: bne $a2, $a0, .LBB2_2 ; LA32-NEXT: # %bb.1: # %test -; LA32-NEXT: ld.w $a0, $a1, 0 +; LA32-NEXT: ld.w $zero, $a1, 0 ; LA32-NEXT: .LBB2_2: # %end ; LA32-NEXT: ret ; @@ -61,7 +61,7 @@ define void @foo_br_ne(i32 %a, ptr %b) nounwind { ; LA64-NEXT: addi.w $a0, $a0, 0 ; LA64-NEXT: bne $a2, $a0, .LBB2_2 ; LA64-NEXT: # %bb.1: # %test -; LA64-NEXT: ld.w $a0, $a1, 0 +; LA64-NEXT: ld.w $zero, $a1, 0 ; LA64-NEXT: .LBB2_2: # %end ; LA64-NEXT: ret %val = load volatile i32, ptr %b @@ -81,7 +81,7 @@ define void @foo_br_slt(i32 %a, ptr %b) nounwind { ; LA32-NEXT: ld.w $a2, $a1, 0 ; LA32-NEXT: blt $a2, $a0, .LBB3_2 ; LA32-NEXT: # %bb.1: # %test -; LA32-NEXT: ld.w $a0, $a1, 0 +; LA32-NEXT: ld.w $zero, $a1, 0 ; LA32-NEXT: .LBB3_2: # %end ; LA32-NEXT: ret ; @@ -91,7 +91,7 @@ define void @foo_br_slt(i32 %a, ptr %b) nounwind { ; LA64-NEXT: addi.w $a0, $a0, 0 ; LA64-NEXT: blt $a2, $a0, .LBB3_2 ; LA64-NEXT: # %bb.1: # %test -; LA64-NEXT: ld.w $a0, $a1, 0 +; LA64-NEXT: ld.w $zero, $a1, 0 ; LA64-NEXT: .LBB3_2: # %end ; LA64-NEXT: ret %val = load volatile i32, ptr %b @@ -111,7 +111,7 @@ define void @foo_br_sge(i32 %a, ptr %b) nounwind { ; LA32-NEXT: ld.w $a2, $a1, 0 ; LA32-NEXT: bge $a2, $a0, .LBB4_2 ; LA32-NEXT: # %bb.1: # %test -; LA32-NEXT: ld.w $a0, $a1, 0 +; LA32-NEXT: ld.w $zero, $a1, 0 ; LA32-NEXT: .LBB4_2: # %end ; LA32-NEXT: ret ; @@ -121,7 +121,7 @@ define void @foo_br_sge(i32 %a, ptr %b) nounwind { ; LA64-NEXT: addi.w $a0, $a0, 0 ; LA64-NEXT: bge $a2, $a0, .LBB4_2 ; LA64-NEXT: # %bb.1: # %test -; LA64-NEXT: ld.w $a0, $a1, 0 +; LA64-NEXT: ld.w $zero, $a1, 0 ; LA64-NEXT: .LBB4_2: # %end ; LA64-NEXT: ret %val = load volatile i32, ptr %b @@ -141,7 +141,7 @@ define void @foo_br_ult(i32 %a, ptr %b) nounwind { ; LA32-NEXT: ld.w $a2, $a1, 0 ; LA32-NEXT: bltu $a2, $a0, .LBB5_2 ; LA32-NEXT: # %bb.1: # %test -; LA32-NEXT: ld.w $a0, $a1, 0 +; LA32-NEXT: ld.w $zero, $a1, 0 ; LA32-NEXT: .LBB5_2: # %end ; LA32-NEXT: ret ; @@ -151,7 +151,7 @@ define void @foo_br_ult(i32 %a, ptr %b) nounwind { ; LA64-NEXT: addi.w $a0, $a0, 0 ; LA64-NEXT: bltu $a2, $a0, .LBB5_2 ; LA64-NEXT: # %bb.1: # %test -; LA64-NEXT: ld.w $a0, $a1, 0 +; LA64-NEXT: ld.w $zero, $a1, 0 ; LA64-NEXT: .LBB5_2: # %end ; LA64-NEXT: ret %val = load volatile i32, ptr %b @@ -171,7 +171,7 @@ define void @foo_br_uge(i32 %a, ptr %b) nounwind { ; LA32-NEXT: ld.w $a2, $a1, 0 ; LA32-NEXT: bgeu $a2, $a0, .LBB6_2 ; LA32-NEXT: # %bb.1: # %test -; LA32-NEXT: ld.w $a0, $a1, 0 +; LA32-NEXT: ld.w $zero, $a1, 0 ; LA32-NEXT: .LBB6_2: # %end ; LA32-NEXT: ret ; @@ -181,7 +181,7 @@ define void @foo_br_uge(i32 %a, ptr %b) nounwind { ; LA64-NEXT: addi.w $a0, $a0, 0 ; LA64-NEXT: bgeu $a2, $a0, .LBB6_2 ; LA64-NEXT: # %bb.1: # %test -; LA64-NEXT: ld.w $a0, $a1, 0 +; LA64-NEXT: ld.w $zero, $a1, 0 ; LA64-NEXT: .LBB6_2: # %end ; LA64-NEXT: ret %val = load volatile i32, ptr %b @@ -202,7 +202,7 @@ define void @foo_br_sgt(i32 %a, ptr %b) nounwind { ; LA32-NEXT: ld.w $a2, $a1, 0 ; LA32-NEXT: blt $a0, $a2, .LBB7_2 ; LA32-NEXT: # %bb.1: # %test -; LA32-NEXT: ld.w $a0, $a1, 0 +; LA32-NEXT: ld.w $zero, $a1, 0 ; LA32-NEXT: .LBB7_2: # %end ; LA32-NEXT: ret ; @@ -212,7 +212,7 @@ define void @foo_br_sgt(i32 %a, ptr %b) nounwind { ; LA64-NEXT: addi.w $a0, $a0, 0 ; LA64-NEXT: blt $a0, $a2, .LBB7_2 ; LA64-NEXT: # %bb.1: # %test -; LA64-NEXT: ld.w $a0, $a1, 0 +; LA64-NEXT: ld.w $zero, $a1, 0 ; LA64-NEXT: .LBB7_2: # %end ; LA64-NEXT: ret %val = load volatile i32, ptr %b @@ -232,7 +232,7 @@ define void @foo_br_sle(i32 %a, ptr %b) nounwind { ; LA32-NEXT: ld.w $a2, $a1, 0 ; LA32-NEXT: bge $a0, $a2, .LBB8_2 ; LA32-NEXT: # %bb.1: # %test -; LA32-NEXT: ld.w $a0, $a1, 0 +; LA32-NEXT: ld.w $zero, $a1, 0 ; LA32-NEXT: .LBB8_2: # %end ; LA32-NEXT: ret ; @@ -242,7 +242,7 @@ define void @foo_br_sle(i32 %a, ptr %b) nounwind { ; LA64-NEXT: addi.w $a0, $a0, 0 ; LA64-NEXT: bge $a0, $a2, .LBB8_2 ; LA64-NEXT: # %bb.1: # %test -; LA64-NEXT: ld.w $a0, $a1, 0 +; LA64-NEXT: ld.w $zero, $a1, 0 ; LA64-NEXT: .LBB8_2: # %end ; LA64-NEXT: ret %val = load volatile i32, ptr %b @@ -262,7 +262,7 @@ define void @foo_br_ugt(i32 %a, ptr %b) nounwind { ; LA32-NEXT: ld.w $a2, $a1, 0 ; LA32-NEXT: bltu $a0, $a2, .LBB9_2 ; LA32-NEXT: # %bb.1: # %test -; LA32-NEXT: ld.w $a0, $a1, 0 +; LA32-NEXT: ld.w $zero, $a1, 0 ; LA32-NEXT: .LBB9_2: # %end ; LA32-NEXT: ret ; @@ -272,7 +272,7 @@ define void @foo_br_ugt(i32 %a, ptr %b) nounwind { ; LA64-NEXT: addi.w $a0, $a0, 0 ; LA64-NEXT: bltu $a0, $a2, .LBB9_2 ; LA64-NEXT: # %bb.1: # %test -; LA64-NEXT: ld.w $a0, $a1, 0 +; LA64-NEXT: ld.w $zero, $a1, 0 ; LA64-NEXT: .LBB9_2: # %end ; LA64-NEXT: ret %val = load volatile i32, ptr %b @@ -292,7 +292,7 @@ define void @foo_br_ule(i32 %a, ptr %b) nounwind { ; LA32-NEXT: ld.w $a2, $a1, 0 ; LA32-NEXT: bgeu $a0, $a2, .LBB10_2 ; LA32-NEXT: # %bb.1: # %test -; LA32-NEXT: ld.w $a0, $a1, 0 +; LA32-NEXT: ld.w $zero, $a1, 0 ; LA32-NEXT: .LBB10_2: # %end ; LA32-NEXT: ret ; @@ -302,7 +302,7 @@ define void @foo_br_ule(i32 %a, ptr %b) nounwind { ; LA64-NEXT: addi.w $a0, $a0, 0 ; LA64-NEXT: bgeu $a0, $a2, .LBB10_2 ; LA64-NEXT: # %bb.1: # %test -; LA64-NEXT: ld.w $a0, $a1, 0 +; LA64-NEXT: ld.w $zero, $a1, 0 ; LA64-NEXT: .LBB10_2: # %end ; LA64-NEXT: ret %val = load volatile i32, ptr %b @@ -321,11 +321,11 @@ end: define void @foo_br_cc(ptr %a, i1 %cc) nounwind { ; ALL-LABEL: foo_br_cc: ; ALL: # %bb.0: -; ALL-NEXT: ld.w $a2, $a0, 0 +; ALL-NEXT: ld.w $zero, $a0, 0 ; ALL-NEXT: andi $a1, $a1, 1 ; ALL-NEXT: bnez $a1, .LBB11_2 ; ALL-NEXT: # %bb.1: # %test -; ALL-NEXT: ld.w $a0, $a0, 0 +; ALL-NEXT: ld.w $zero, $a0, 0 ; ALL-NEXT: .LBB11_2: # %end ; ALL-NEXT: ret %val = load volatile i32, ptr %a diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store.ll index 387d9333b90f..26f44adc6135 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store.ll @@ -57,7 +57,7 @@ define i32 @load_store_global_array(i32 %a) nounwind { ; LA32NOPIC-NEXT: addi.w $a2, $a1, %pc_lo12(arr) ; LA32NOPIC-NEXT: ld.w $a1, $a2, 0 ; LA32NOPIC-NEXT: st.w $a0, $a2, 0 -; LA32NOPIC-NEXT: ld.w $a3, $a2, 36 +; LA32NOPIC-NEXT: ld.w $zero, $a2, 36 ; LA32NOPIC-NEXT: st.w $a0, $a2, 36 ; LA32NOPIC-NEXT: move $a0, $a1 ; LA32NOPIC-NEXT: ret @@ -68,7 +68,7 @@ define i32 @load_store_global_array(i32 %a) nounwind { ; LA32PIC-NEXT: addi.w $a2, $a1, %pc_lo12(.Larr$local) ; LA32PIC-NEXT: ld.w $a1, $a2, 0 ; LA32PIC-NEXT: st.w $a0, $a2, 0 -; LA32PIC-NEXT: ld.w $a3, $a2, 36 +; LA32PIC-NEXT: ld.w $zero, $a2, 36 ; LA32PIC-NEXT: st.w $a0, $a2, 36 ; LA32PIC-NEXT: move $a0, $a1 ; LA32PIC-NEXT: ret @@ -79,7 +79,7 @@ define i32 @load_store_global_array(i32 %a) nounwind { ; LA64NOPIC-NEXT: addi.d $a2, $a1, %pc_lo12(arr) ; LA64NOPIC-NEXT: ld.w $a1, $a2, 0 ; LA64NOPIC-NEXT: st.w $a0, $a2, 0 -; LA64NOPIC-NEXT: ld.w $a3, $a2, 36 +; LA64NOPIC-NEXT: ld.w $zero, $a2, 36 ; LA64NOPIC-NEXT: st.w $a0, $a2, 36 ; LA64NOPIC-NEXT: move $a0, $a1 ; LA64NOPIC-NEXT: ret @@ -90,7 +90,7 @@ define i32 @load_store_global_array(i32 %a) nounwind { ; LA64PIC-NEXT: addi.d $a2, $a1, %pc_lo12(.Larr$local) ; LA64PIC-NEXT: ld.w $a1, $a2, 0 ; LA64PIC-NEXT: st.w $a0, $a2, 0 -; LA64PIC-NEXT: ld.w $a3, $a2, 36 +; LA64PIC-NEXT: ld.w $zero, $a2, 36 ; LA64PIC-NEXT: st.w $a0, $a2, 36 ; LA64PIC-NEXT: move $a0, $a1 ; LA64PIC-NEXT: ret @@ -108,7 +108,7 @@ define i64 @ld_b(ptr %a) nounwind { ; LA32NOPIC-LABEL: ld_b: ; LA32NOPIC: # %bb.0: ; LA32NOPIC-NEXT: ld.b $a2, $a0, 1 -; LA32NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA32NOPIC-NEXT: ld.b $zero, $a0, 0 ; LA32NOPIC-NEXT: srai.w $a1, $a2, 31 ; LA32NOPIC-NEXT: move $a0, $a2 ; LA32NOPIC-NEXT: ret @@ -116,7 +116,7 @@ define i64 @ld_b(ptr %a) nounwind { ; LA32PIC-LABEL: ld_b: ; LA32PIC: # %bb.0: ; LA32PIC-NEXT: ld.b $a2, $a0, 1 -; LA32PIC-NEXT: ld.b $a0, $a0, 0 +; LA32PIC-NEXT: ld.b $zero, $a0, 0 ; LA32PIC-NEXT: srai.w $a1, $a2, 31 ; LA32PIC-NEXT: move $a0, $a2 ; LA32PIC-NEXT: ret @@ -124,14 +124,14 @@ define i64 @ld_b(ptr %a) nounwind { ; LA64NOPIC-LABEL: ld_b: ; LA64NOPIC: # %bb.0: ; LA64NOPIC-NEXT: ld.b $a1, $a0, 1 -; LA64NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA64NOPIC-NEXT: ld.b $zero, $a0, 0 ; LA64NOPIC-NEXT: move $a0, $a1 ; LA64NOPIC-NEXT: ret ; ; LA64PIC-LABEL: ld_b: ; LA64PIC: # %bb.0: ; LA64PIC-NEXT: ld.b $a1, $a0, 1 -; LA64PIC-NEXT: ld.b $a0, $a0, 0 +; LA64PIC-NEXT: ld.b $zero, $a0, 0 ; LA64PIC-NEXT: move $a0, $a1 ; LA64PIC-NEXT: ret %1 = getelementptr i8, ptr %a, i64 1 @@ -145,7 +145,7 @@ define i64 @ld_h(ptr %a) nounwind { ; LA32NOPIC-LABEL: ld_h: ; LA32NOPIC: # %bb.0: ; LA32NOPIC-NEXT: ld.h $a2, $a0, 4 -; LA32NOPIC-NEXT: ld.h $a0, $a0, 0 +; LA32NOPIC-NEXT: ld.h $zero, $a0, 0 ; LA32NOPIC-NEXT: srai.w $a1, $a2, 31 ; LA32NOPIC-NEXT: move $a0, $a2 ; LA32NOPIC-NEXT: ret @@ -153,7 +153,7 @@ define i64 @ld_h(ptr %a) nounwind { ; LA32PIC-LABEL: ld_h: ; LA32PIC: # %bb.0: ; LA32PIC-NEXT: ld.h $a2, $a0, 4 -; LA32PIC-NEXT: ld.h $a0, $a0, 0 +; LA32PIC-NEXT: ld.h $zero, $a0, 0 ; LA32PIC-NEXT: srai.w $a1, $a2, 31 ; LA32PIC-NEXT: move $a0, $a2 ; LA32PIC-NEXT: ret @@ -161,14 +161,14 @@ define i64 @ld_h(ptr %a) nounwind { ; LA64NOPIC-LABEL: ld_h: ; LA64NOPIC: # %bb.0: ; LA64NOPIC-NEXT: ld.h $a1, $a0, 4 -; LA64NOPIC-NEXT: ld.h $a0, $a0, 0 +; LA64NOPIC-NEXT: ld.h $zero, $a0, 0 ; LA64NOPIC-NEXT: move $a0, $a1 ; LA64NOPIC-NEXT: ret ; ; LA64PIC-LABEL: ld_h: ; LA64PIC: # %bb.0: ; LA64PIC-NEXT: ld.h $a1, $a0, 4 -; LA64PIC-NEXT: ld.h $a0, $a0, 0 +; LA64PIC-NEXT: ld.h $zero, $a0, 0 ; LA64PIC-NEXT: move $a0, $a1 ; LA64PIC-NEXT: ret %1 = getelementptr i16, ptr %a, i64 2 @@ -182,7 +182,7 @@ define i64 @ld_w(ptr %a) nounwind { ; LA32NOPIC-LABEL: ld_w: ; LA32NOPIC: # %bb.0: ; LA32NOPIC-NEXT: ld.w $a2, $a0, 12 -; LA32NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA32NOPIC-NEXT: ld.w $zero, $a0, 0 ; LA32NOPIC-NEXT: srai.w $a1, $a2, 31 ; LA32NOPIC-NEXT: move $a0, $a2 ; LA32NOPIC-NEXT: ret @@ -190,7 +190,7 @@ define i64 @ld_w(ptr %a) nounwind { ; LA32PIC-LABEL: ld_w: ; LA32PIC: # %bb.0: ; LA32PIC-NEXT: ld.w $a2, $a0, 12 -; LA32PIC-NEXT: ld.w $a0, $a0, 0 +; LA32PIC-NEXT: ld.w $zero, $a0, 0 ; LA32PIC-NEXT: srai.w $a1, $a2, 31 ; LA32PIC-NEXT: move $a0, $a2 ; LA32PIC-NEXT: ret @@ -198,14 +198,14 @@ define i64 @ld_w(ptr %a) nounwind { ; LA64NOPIC-LABEL: ld_w: ; LA64NOPIC: # %bb.0: ; LA64NOPIC-NEXT: ld.w $a1, $a0, 12 -; LA64NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA64NOPIC-NEXT: ld.w $zero, $a0, 0 ; LA64NOPIC-NEXT: move $a0, $a1 ; LA64NOPIC-NEXT: ret ; ; LA64PIC-LABEL: ld_w: ; LA64PIC: # %bb.0: ; LA64PIC-NEXT: ld.w $a1, $a0, 12 -; LA64PIC-NEXT: ld.w $a0, $a0, 0 +; LA64PIC-NEXT: ld.w $zero, $a0, 0 ; LA64PIC-NEXT: move $a0, $a1 ; LA64PIC-NEXT: ret %1 = getelementptr i32, ptr %a, i64 3 @@ -220,8 +220,8 @@ define i64 @ld_d(ptr %a) nounwind { ; LA32NOPIC: # %bb.0: ; LA32NOPIC-NEXT: ld.w $a1, $a0, 28 ; LA32NOPIC-NEXT: ld.w $a2, $a0, 24 -; LA32NOPIC-NEXT: ld.w $a3, $a0, 4 -; LA32NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA32NOPIC-NEXT: ld.w $zero, $a0, 4 +; LA32NOPIC-NEXT: ld.w $zero, $a0, 0 ; LA32NOPIC-NEXT: move $a0, $a2 ; LA32NOPIC-NEXT: ret ; @@ -229,22 +229,22 @@ define i64 @ld_d(ptr %a) nounwind { ; LA32PIC: # %bb.0: ; LA32PIC-NEXT: ld.w $a1, $a0, 28 ; LA32PIC-NEXT: ld.w $a2, $a0, 24 -; LA32PIC-NEXT: ld.w $a3, $a0, 4 -; LA32PIC-NEXT: ld.w $a0, $a0, 0 +; LA32PIC-NEXT: ld.w $zero, $a0, 4 +; LA32PIC-NEXT: ld.w $zero, $a0, 0 ; LA32PIC-NEXT: move $a0, $a2 ; LA32PIC-NEXT: ret ; ; LA64NOPIC-LABEL: ld_d: ; LA64NOPIC: # %bb.0: ; LA64NOPIC-NEXT: ld.d $a1, $a0, 24 -; LA64NOPIC-NEXT: ld.d $a0, $a0, 0 +; LA64NOPIC-NEXT: ld.d $zero, $a0, 0 ; LA64NOPIC-NEXT: move $a0, $a1 ; LA64NOPIC-NEXT: ret ; ; LA64PIC-LABEL: ld_d: ; LA64PIC: # %bb.0: ; LA64PIC-NEXT: ld.d $a1, $a0, 24 -; LA64PIC-NEXT: ld.d $a0, $a0, 0 +; LA64PIC-NEXT: ld.d $zero, $a0, 0 ; LA64PIC-NEXT: move $a0, $a1 ; LA64PIC-NEXT: ret %1 = getelementptr i64, ptr %a, i64 3 @@ -375,7 +375,7 @@ define i64 @ldx_b(ptr %a, i64 %idx) nounwind { ; LA32NOPIC: # %bb.0: ; LA32NOPIC-NEXT: add.w $a1, $a0, $a1 ; LA32NOPIC-NEXT: ld.b $a2, $a1, 0 -; LA32NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA32NOPIC-NEXT: ld.b $zero, $a0, 0 ; LA32NOPIC-NEXT: srai.w $a1, $a2, 31 ; LA32NOPIC-NEXT: move $a0, $a2 ; LA32NOPIC-NEXT: ret @@ -384,7 +384,7 @@ define i64 @ldx_b(ptr %a, i64 %idx) nounwind { ; LA32PIC: # %bb.0: ; LA32PIC-NEXT: add.w $a1, $a0, $a1 ; LA32PIC-NEXT: ld.b $a2, $a1, 0 -; LA32PIC-NEXT: ld.b $a0, $a0, 0 +; LA32PIC-NEXT: ld.b $zero, $a0, 0 ; LA32PIC-NEXT: srai.w $a1, $a2, 31 ; LA32PIC-NEXT: move $a0, $a2 ; LA32PIC-NEXT: ret @@ -392,14 +392,14 @@ define i64 @ldx_b(ptr %a, i64 %idx) nounwind { ; LA64NOPIC-LABEL: ldx_b: ; LA64NOPIC: # %bb.0: ; LA64NOPIC-NEXT: ldx.b $a1, $a0, $a1 -; LA64NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA64NOPIC-NEXT: ld.b $zero, $a0, 0 ; LA64NOPIC-NEXT: move $a0, $a1 ; LA64NOPIC-NEXT: ret ; ; LA64PIC-LABEL: ldx_b: ; LA64PIC: # %bb.0: ; LA64PIC-NEXT: ldx.b $a1, $a0, $a1 -; LA64PIC-NEXT: ld.b $a0, $a0, 0 +; LA64PIC-NEXT: ld.b $zero, $a0, 0 ; LA64PIC-NEXT: move $a0, $a1 ; LA64PIC-NEXT: ret %1 = getelementptr i8, ptr %a, i64 %idx @@ -414,7 +414,7 @@ define i64 @ldx_h(ptr %a, i64 %idx) nounwind { ; LA32NOPIC: # %bb.0: ; LA32NOPIC-NEXT: alsl.w $a1, $a1, $a0, 1 ; LA32NOPIC-NEXT: ld.h $a2, $a1, 0 -; LA32NOPIC-NEXT: ld.h $a0, $a0, 0 +; LA32NOPIC-NEXT: ld.h $zero, $a0, 0 ; LA32NOPIC-NEXT: srai.w $a1, $a2, 31 ; LA32NOPIC-NEXT: move $a0, $a2 ; LA32NOPIC-NEXT: ret @@ -423,7 +423,7 @@ define i64 @ldx_h(ptr %a, i64 %idx) nounwind { ; LA32PIC: # %bb.0: ; LA32PIC-NEXT: alsl.w $a1, $a1, $a0, 1 ; LA32PIC-NEXT: ld.h $a2, $a1, 0 -; LA32PIC-NEXT: ld.h $a0, $a0, 0 +; LA32PIC-NEXT: ld.h $zero, $a0, 0 ; LA32PIC-NEXT: srai.w $a1, $a2, 31 ; LA32PIC-NEXT: move $a0, $a2 ; LA32PIC-NEXT: ret @@ -432,7 +432,7 @@ define i64 @ldx_h(ptr %a, i64 %idx) nounwind { ; LA64NOPIC: # %bb.0: ; LA64NOPIC-NEXT: slli.d $a1, $a1, 1 ; LA64NOPIC-NEXT: ldx.h $a1, $a0, $a1 -; LA64NOPIC-NEXT: ld.h $a0, $a0, 0 +; LA64NOPIC-NEXT: ld.h $zero, $a0, 0 ; LA64NOPIC-NEXT: move $a0, $a1 ; LA64NOPIC-NEXT: ret ; @@ -440,7 +440,7 @@ define i64 @ldx_h(ptr %a, i64 %idx) nounwind { ; LA64PIC: # %bb.0: ; LA64PIC-NEXT: slli.d $a1, $a1, 1 ; LA64PIC-NEXT: ldx.h $a1, $a0, $a1 -; LA64PIC-NEXT: ld.h $a0, $a0, 0 +; LA64PIC-NEXT: ld.h $zero, $a0, 0 ; LA64PIC-NEXT: move $a0, $a1 ; LA64PIC-NEXT: ret %1 = getelementptr i16, ptr %a, i64 %idx @@ -455,7 +455,7 @@ define i64 @ldx_w(ptr %a, i64 %idx) nounwind { ; LA32NOPIC: # %bb.0: ; LA32NOPIC-NEXT: alsl.w $a1, $a1, $a0, 2 ; LA32NOPIC-NEXT: ld.w $a2, $a1, 0 -; LA32NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA32NOPIC-NEXT: ld.w $zero, $a0, 0 ; LA32NOPIC-NEXT: srai.w $a1, $a2, 31 ; LA32NOPIC-NEXT: move $a0, $a2 ; LA32NOPIC-NEXT: ret @@ -464,7 +464,7 @@ define i64 @ldx_w(ptr %a, i64 %idx) nounwind { ; LA32PIC: # %bb.0: ; LA32PIC-NEXT: alsl.w $a1, $a1, $a0, 2 ; LA32PIC-NEXT: ld.w $a2, $a1, 0 -; LA32PIC-NEXT: ld.w $a0, $a0, 0 +; LA32PIC-NEXT: ld.w $zero, $a0, 0 ; LA32PIC-NEXT: srai.w $a1, $a2, 31 ; LA32PIC-NEXT: move $a0, $a2 ; LA32PIC-NEXT: ret @@ -473,7 +473,7 @@ define i64 @ldx_w(ptr %a, i64 %idx) nounwind { ; LA64NOPIC: # %bb.0: ; LA64NOPIC-NEXT: slli.d $a1, $a1, 2 ; LA64NOPIC-NEXT: ldx.w $a1, $a0, $a1 -; LA64NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA64NOPIC-NEXT: ld.w $zero, $a0, 0 ; LA64NOPIC-NEXT: move $a0, $a1 ; LA64NOPIC-NEXT: ret ; @@ -481,7 +481,7 @@ define i64 @ldx_w(ptr %a, i64 %idx) nounwind { ; LA64PIC: # %bb.0: ; LA64PIC-NEXT: slli.d $a1, $a1, 2 ; LA64PIC-NEXT: ldx.w $a1, $a0, $a1 -; LA64PIC-NEXT: ld.w $a0, $a0, 0 +; LA64PIC-NEXT: ld.w $zero, $a0, 0 ; LA64PIC-NEXT: move $a0, $a1 ; LA64PIC-NEXT: ret %1 = getelementptr i32, ptr %a, i64 %idx @@ -497,8 +497,8 @@ define i64 @ldx_d(ptr %a, i64 %idx) nounwind { ; LA32NOPIC-NEXT: alsl.w $a1, $a1, $a0, 3 ; LA32NOPIC-NEXT: ld.w $a2, $a1, 0 ; LA32NOPIC-NEXT: ld.w $a1, $a1, 4 -; LA32NOPIC-NEXT: ld.w $a3, $a0, 0 -; LA32NOPIC-NEXT: ld.w $a0, $a0, 4 +; LA32NOPIC-NEXT: ld.w $zero, $a0, 0 +; LA32NOPIC-NEXT: ld.w $zero, $a0, 4 ; LA32NOPIC-NEXT: move $a0, $a2 ; LA32NOPIC-NEXT: ret ; @@ -507,8 +507,8 @@ define i64 @ldx_d(ptr %a, i64 %idx) nounwind { ; LA32PIC-NEXT: alsl.w $a1, $a1, $a0, 3 ; LA32PIC-NEXT: ld.w $a2, $a1, 0 ; LA32PIC-NEXT: ld.w $a1, $a1, 4 -; LA32PIC-NEXT: ld.w $a3, $a0, 0 -; LA32PIC-NEXT: ld.w $a0, $a0, 4 +; LA32PIC-NEXT: ld.w $zero, $a0, 0 +; LA32PIC-NEXT: ld.w $zero, $a0, 4 ; LA32PIC-NEXT: move $a0, $a2 ; LA32PIC-NEXT: ret ; @@ -516,7 +516,7 @@ define i64 @ldx_d(ptr %a, i64 %idx) nounwind { ; LA64NOPIC: # %bb.0: ; LA64NOPIC-NEXT: slli.d $a1, $a1, 3 ; LA64NOPIC-NEXT: ldx.d $a1, $a0, $a1 -; LA64NOPIC-NEXT: ld.d $a0, $a0, 0 +; LA64NOPIC-NEXT: ld.d $zero, $a0, 0 ; LA64NOPIC-NEXT: move $a0, $a1 ; LA64NOPIC-NEXT: ret ; @@ -524,7 +524,7 @@ define i64 @ldx_d(ptr %a, i64 %idx) nounwind { ; LA64PIC: # %bb.0: ; LA64PIC-NEXT: slli.d $a1, $a1, 3 ; LA64PIC-NEXT: ldx.d $a1, $a0, $a1 -; LA64PIC-NEXT: ld.d $a0, $a0, 0 +; LA64PIC-NEXT: ld.d $zero, $a0, 0 ; LA64PIC-NEXT: move $a0, $a1 ; LA64PIC-NEXT: ret %1 = getelementptr i64, ptr %a, i64 %idx @@ -855,7 +855,7 @@ define i64 @load_sext_zext_anyext_i1(ptr %a) nounwind { ; LA32NOPIC-NEXT: ld.bu $a1, $a0, 1 ; LA32NOPIC-NEXT: ld.bu $a3, $a0, 2 ; LA32NOPIC-NEXT: sub.w $a2, $a3, $a1 -; LA32NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA32NOPIC-NEXT: ld.b $zero, $a0, 0 ; LA32NOPIC-NEXT: sltu $a0, $a3, $a1 ; LA32NOPIC-NEXT: sub.w $a1, $zero, $a0 ; LA32NOPIC-NEXT: move $a0, $a2 @@ -866,7 +866,7 @@ define i64 @load_sext_zext_anyext_i1(ptr %a) nounwind { ; LA32PIC-NEXT: ld.bu $a1, $a0, 1 ; LA32PIC-NEXT: ld.bu $a3, $a0, 2 ; LA32PIC-NEXT: sub.w $a2, $a3, $a1 -; LA32PIC-NEXT: ld.b $a0, $a0, 0 +; LA32PIC-NEXT: ld.b $zero, $a0, 0 ; LA32PIC-NEXT: sltu $a0, $a3, $a1 ; LA32PIC-NEXT: sub.w $a1, $zero, $a0 ; LA32PIC-NEXT: move $a0, $a2 @@ -876,7 +876,7 @@ define i64 @load_sext_zext_anyext_i1(ptr %a) nounwind { ; LA64NOPIC: # %bb.0: ; LA64NOPIC-NEXT: ld.bu $a1, $a0, 1 ; LA64NOPIC-NEXT: ld.bu $a2, $a0, 2 -; LA64NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA64NOPIC-NEXT: ld.b $zero, $a0, 0 ; LA64NOPIC-NEXT: sub.d $a0, $a2, $a1 ; LA64NOPIC-NEXT: ret ; @@ -884,7 +884,7 @@ define i64 @load_sext_zext_anyext_i1(ptr %a) nounwind { ; LA64PIC: # %bb.0: ; LA64PIC-NEXT: ld.bu $a1, $a0, 1 ; LA64PIC-NEXT: ld.bu $a2, $a0, 2 -; LA64PIC-NEXT: ld.b $a0, $a0, 0 +; LA64PIC-NEXT: ld.b $zero, $a0, 0 ; LA64PIC-NEXT: sub.d $a0, $a2, $a1 ; LA64PIC-NEXT: ret ;; sextload i1 @@ -906,7 +906,7 @@ define i16 @load_sext_zext_anyext_i1_i16(ptr %a) nounwind { ; LA32NOPIC: # %bb.0: ; LA32NOPIC-NEXT: ld.bu $a1, $a0, 1 ; LA32NOPIC-NEXT: ld.bu $a2, $a0, 2 -; LA32NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA32NOPIC-NEXT: ld.b $zero, $a0, 0 ; LA32NOPIC-NEXT: sub.w $a0, $a2, $a1 ; LA32NOPIC-NEXT: ret ; @@ -914,7 +914,7 @@ define i16 @load_sext_zext_anyext_i1_i16(ptr %a) nounwind { ; LA32PIC: # %bb.0: ; LA32PIC-NEXT: ld.bu $a1, $a0, 1 ; LA32PIC-NEXT: ld.bu $a2, $a0, 2 -; LA32PIC-NEXT: ld.b $a0, $a0, 0 +; LA32PIC-NEXT: ld.b $zero, $a0, 0 ; LA32PIC-NEXT: sub.w $a0, $a2, $a1 ; LA32PIC-NEXT: ret ; @@ -922,7 +922,7 @@ define i16 @load_sext_zext_anyext_i1_i16(ptr %a) nounwind { ; LA64NOPIC: # %bb.0: ; LA64NOPIC-NEXT: ld.bu $a1, $a0, 1 ; LA64NOPIC-NEXT: ld.bu $a2, $a0, 2 -; LA64NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA64NOPIC-NEXT: ld.b $zero, $a0, 0 ; LA64NOPIC-NEXT: sub.d $a0, $a2, $a1 ; LA64NOPIC-NEXT: ret ; @@ -930,7 +930,7 @@ define i16 @load_sext_zext_anyext_i1_i16(ptr %a) nounwind { ; LA64PIC: # %bb.0: ; LA64PIC-NEXT: ld.bu $a1, $a0, 1 ; LA64PIC-NEXT: ld.bu $a2, $a0, 2 -; LA64PIC-NEXT: ld.b $a0, $a0, 0 +; LA64PIC-NEXT: ld.b $zero, $a0, 0 ; LA64PIC-NEXT: sub.d $a0, $a2, $a1 ; LA64PIC-NEXT: ret ;; sextload i1 diff --git a/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll b/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll index 0effd469e3fb..678546f1ef53 100644 --- a/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll +++ b/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll @@ -27,22 +27,22 @@ define void @foo() nounwind { ; MEDIUM_NO_SCH-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill ; MEDIUM_NO_SCH-NEXT: pcalau12i $a0, %got_pc_hi20(G) ; MEDIUM_NO_SCH-NEXT: ld.d $a0, $a0, %got_pc_lo12(G) -; MEDIUM_NO_SCH-NEXT: ld.d $a0, $a0, 0 +; MEDIUM_NO_SCH-NEXT: ld.d $zero, $a0, 0 ; MEDIUM_NO_SCH-NEXT: pcalau12i $a0, %pc_hi20(g) ; MEDIUM_NO_SCH-NEXT: addi.d $a0, $a0, %pc_lo12(g) -; MEDIUM_NO_SCH-NEXT: ld.d $a0, $a0, 0 +; MEDIUM_NO_SCH-NEXT: ld.d $zero, $a0, 0 ; MEDIUM_NO_SCH-NEXT: ori $a0, $zero, 1 ; MEDIUM_NO_SCH-NEXT: pcaddu18i $ra, %call36(bar) ; MEDIUM_NO_SCH-NEXT: jirl $ra, $ra, 0 ; MEDIUM_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(gd) ; MEDIUM_NO_SCH-NEXT: ld.d $a0, $a0, %ie_pc_lo12(gd) -; MEDIUM_NO_SCH-NEXT: ldx.d $a0, $a0, $tp +; MEDIUM_NO_SCH-NEXT: ldx.d $zero, $a0, $tp ; MEDIUM_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ld) ; MEDIUM_NO_SCH-NEXT: ld.d $a0, $a0, %ie_pc_lo12(ld) -; MEDIUM_NO_SCH-NEXT: ldx.d $a0, $a0, $tp +; MEDIUM_NO_SCH-NEXT: ldx.d $zero, $a0, $tp ; MEDIUM_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ie) ; MEDIUM_NO_SCH-NEXT: ld.d $a0, $a0, %ie_pc_lo12(ie) -; MEDIUM_NO_SCH-NEXT: ldx.d $a0, $a0, $tp +; MEDIUM_NO_SCH-NEXT: ldx.d $zero, $a0, $tp ; MEDIUM_NO_SCH-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload ; MEDIUM_NO_SCH-NEXT: addi.d $sp, $sp, 16 ; MEDIUM_NO_SCH-NEXT: ret @@ -53,22 +53,22 @@ define void @foo() nounwind { ; MEDIUM_SCH-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill ; MEDIUM_SCH-NEXT: pcalau12i $a0, %got_pc_hi20(G) ; MEDIUM_SCH-NEXT: ld.d $a0, $a0, %got_pc_lo12(G) -; MEDIUM_SCH-NEXT: ld.d $a0, $a0, 0 +; MEDIUM_SCH-NEXT: ld.d $zero, $a0, 0 ; MEDIUM_SCH-NEXT: pcalau12i $a0, %pc_hi20(g) ; MEDIUM_SCH-NEXT: addi.d $a0, $a0, %pc_lo12(g) -; MEDIUM_SCH-NEXT: ld.d $a0, $a0, 0 +; MEDIUM_SCH-NEXT: ld.d $zero, $a0, 0 ; MEDIUM_SCH-NEXT: ori $a0, $zero, 1 ; MEDIUM_SCH-NEXT: pcaddu18i $ra, %call36(bar) ; MEDIUM_SCH-NEXT: jirl $ra, $ra, 0 ; MEDIUM_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(gd) ; MEDIUM_SCH-NEXT: ld.d $a0, $a0, %ie_pc_lo12(gd) -; MEDIUM_SCH-NEXT: ldx.d $a0, $a0, $tp +; MEDIUM_SCH-NEXT: ldx.d $zero, $a0, $tp ; MEDIUM_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ld) ; MEDIUM_SCH-NEXT: ld.d $a0, $a0, %ie_pc_lo12(ld) -; MEDIUM_SCH-NEXT: ldx.d $a0, $a0, $tp +; MEDIUM_SCH-NEXT: ldx.d $zero, $a0, $tp ; MEDIUM_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ie) ; MEDIUM_SCH-NEXT: ld.d $a0, $a0, %ie_pc_lo12(ie) -; MEDIUM_SCH-NEXT: ldx.d $a0, $a0, $tp +; MEDIUM_SCH-NEXT: ldx.d $zero, $a0, $tp ; MEDIUM_SCH-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload ; MEDIUM_SCH-NEXT: addi.d $sp, $sp, 16 ; MEDIUM_SCH-NEXT: ret @@ -82,13 +82,13 @@ define void @foo() nounwind { ; LARGE_NO_SCH-NEXT: lu32i.d $t8, %got64_pc_lo20(G) ; LARGE_NO_SCH-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(G) ; LARGE_NO_SCH-NEXT: ldx.d $a0, $t8, $a0 -; LARGE_NO_SCH-NEXT: ld.d $a0, $a0, 0 +; LARGE_NO_SCH-NEXT: ld.d $zero, $a0, 0 ; LARGE_NO_SCH-NEXT: pcalau12i $a0, %pc_hi20(g) ; LARGE_NO_SCH-NEXT: addi.d $t8, $zero, %pc_lo12(g) ; LARGE_NO_SCH-NEXT: lu32i.d $t8, %pc64_lo20(g) ; LARGE_NO_SCH-NEXT: lu52i.d $t8, $t8, %pc64_hi12(g) ; LARGE_NO_SCH-NEXT: add.d $a0, $t8, $a0 -; LARGE_NO_SCH-NEXT: ld.d $a0, $a0, 0 +; LARGE_NO_SCH-NEXT: ld.d $zero, $a0, 0 ; LARGE_NO_SCH-NEXT: ori $a0, $zero, 1 ; LARGE_NO_SCH-NEXT: pcalau12i $ra, %got_pc_hi20(bar) ; LARGE_NO_SCH-NEXT: addi.d $t8, $zero, %got_pc_lo12(bar) @@ -101,19 +101,19 @@ define void @foo() nounwind { ; LARGE_NO_SCH-NEXT: lu32i.d $t8, %ie64_pc_lo20(gd) ; LARGE_NO_SCH-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(gd) ; LARGE_NO_SCH-NEXT: ldx.d $a0, $t8, $a0 -; LARGE_NO_SCH-NEXT: ldx.d $a0, $a0, $tp +; LARGE_NO_SCH-NEXT: ldx.d $zero, $a0, $tp ; LARGE_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ld) ; LARGE_NO_SCH-NEXT: addi.d $t8, $zero, %ie_pc_lo12(ld) ; LARGE_NO_SCH-NEXT: lu32i.d $t8, %ie64_pc_lo20(ld) ; LARGE_NO_SCH-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(ld) ; LARGE_NO_SCH-NEXT: ldx.d $a0, $t8, $a0 -; LARGE_NO_SCH-NEXT: ldx.d $a0, $a0, $tp +; LARGE_NO_SCH-NEXT: ldx.d $zero, $a0, $tp ; LARGE_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ie) ; LARGE_NO_SCH-NEXT: addi.d $t8, $zero, %ie_pc_lo12(ie) ; LARGE_NO_SCH-NEXT: lu32i.d $t8, %ie64_pc_lo20(ie) ; LARGE_NO_SCH-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(ie) ; LARGE_NO_SCH-NEXT: ldx.d $a0, $t8, $a0 -; LARGE_NO_SCH-NEXT: ldx.d $a0, $a0, $tp +; LARGE_NO_SCH-NEXT: ldx.d $zero, $a0, $tp ; LARGE_NO_SCH-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload ; LARGE_NO_SCH-NEXT: addi.d $sp, $sp, 16 ; LARGE_NO_SCH-NEXT: ret @@ -127,13 +127,13 @@ define void @foo() nounwind { ; LARGE_SCH-NEXT: lu32i.d $t8, %got64_pc_lo20(G) ; LARGE_SCH-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(G) ; LARGE_SCH-NEXT: ldx.d $a0, $t8, $a0 -; LARGE_SCH-NEXT: ld.d $a0, $a0, 0 +; LARGE_SCH-NEXT: ld.d $zero, $a0, 0 ; LARGE_SCH-NEXT: pcalau12i $a0, %pc_hi20(g) ; LARGE_SCH-NEXT: addi.d $t8, $zero, %pc_lo12(g) ; LARGE_SCH-NEXT: lu32i.d $t8, %pc64_lo20(g) ; LARGE_SCH-NEXT: lu52i.d $t8, $t8, %pc64_hi12(g) ; LARGE_SCH-NEXT: add.d $a0, $t8, $a0 -; LARGE_SCH-NEXT: ld.d $a0, $a0, 0 +; LARGE_SCH-NEXT: ld.d $zero, $a0, 0 ; LARGE_SCH-NEXT: ori $a0, $zero, 1 ; LARGE_SCH-NEXT: pcalau12i $ra, %got_pc_hi20(bar) ; LARGE_SCH-NEXT: addi.d $t8, $zero, %got_pc_lo12(bar) @@ -146,19 +146,19 @@ define void @foo() nounwind { ; LARGE_SCH-NEXT: lu32i.d $t8, %ie64_pc_lo20(gd) ; LARGE_SCH-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(gd) ; LARGE_SCH-NEXT: ldx.d $a0, $t8, $a0 -; LARGE_SCH-NEXT: ldx.d $a0, $a0, $tp +; LARGE_SCH-NEXT: ldx.d $zero, $a0, $tp ; LARGE_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ld) ; LARGE_SCH-NEXT: addi.d $t8, $zero, %ie_pc_lo12(ld) ; LARGE_SCH-NEXT: lu32i.d $t8, %ie64_pc_lo20(ld) ; LARGE_SCH-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(ld) ; LARGE_SCH-NEXT: ldx.d $a0, $t8, $a0 -; LARGE_SCH-NEXT: ldx.d $a0, $a0, $tp +; LARGE_SCH-NEXT: ldx.d $zero, $a0, $tp ; LARGE_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ie) ; LARGE_SCH-NEXT: addi.d $t8, $zero, %ie_pc_lo12(ie) ; LARGE_SCH-NEXT: lu32i.d $t8, %ie64_pc_lo20(ie) ; LARGE_SCH-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(ie) ; LARGE_SCH-NEXT: ldx.d $a0, $t8, $a0 -; LARGE_SCH-NEXT: ldx.d $a0, $a0, $tp +; LARGE_SCH-NEXT: ldx.d $zero, $a0, $tp ; LARGE_SCH-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload ; LARGE_SCH-NEXT: addi.d $sp, $sp, 16 ; LARGE_SCH-NEXT: ret diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn index 822c2ec8afc6..248e47a9584c 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn @@ -34,6 +34,7 @@ static_library("LLVMLoongArchCodeGen") { include_dirs = [ "." ] sources = [ "LoongArchAsmPrinter.cpp", + "LoongArchDeadRegisterDefinitions.cpp", "LoongArchExpandAtomicPseudoInsts.cpp", "LoongArchExpandPseudoInsts.cpp", "LoongArchFrameLowering.cpp", -- GitLab From c15b86731b78de88fadbc16ea1c2df2f60c991e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Fri, 7 Jun 2024 09:24:33 +0200 Subject: [PATCH 204/462] [clang][Interp][NFC] Add GetPtrFieldPop opcode And change the previous GetPtrField to only peek() the base pointer. We can get rid of a whole bunch of DupPtr ops this way. --- clang/lib/AST/Interp/ByteCodeExprGen.cpp | 32 +++------------- clang/lib/AST/Interp/Interp.h | 25 +++++++++++- clang/lib/AST/Interp/Opcodes.td | 48 +++++++----------------- 3 files changed, 42 insertions(+), 63 deletions(-) diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index 907317526364..1d0c36d0bf09 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -1104,14 +1104,9 @@ bool ByteCodeExprGen::visitInitList(ArrayRef Inits, if (!this->visit(Init)) return false; - if (FieldToInit->isBitField()) { - if (!this->emitInitBitField(T, FieldToInit, E)) - return false; - } else { - if (!this->emitInitField(T, FieldToInit->Offset, E)) - return false; - } - return this->emitPopPtr(E); + if (FieldToInit->isBitField()) + return this->emitInitBitField(T, FieldToInit, E); + return this->emitInitField(T, FieldToInit->Offset, E); }; auto initCompositeField = [=](const Record::Field *FieldToInit, @@ -1147,9 +1142,6 @@ bool ByteCodeExprGen::visitInitList(ArrayRef Inits, else FToInit = cast(E)->getInitializedFieldInUnion(); - if (!this->emitDupPtr(E)) - return false; - const Record::Field *FieldToInit = R->getField(FToInit); if (std::optional T = classify(Init)) { if (!initPrimitiveField(FieldToInit, Init, *T)) @@ -1169,8 +1161,6 @@ bool ByteCodeExprGen::visitInitList(ArrayRef Inits, while (InitIndex < R->getNumFields() && R->getField(InitIndex)->Decl->isUnnamedBitField()) ++InitIndex; - if (!this->emitDupPtr(E)) - return false; if (std::optional T = classify(Init)) { const Record::Field *FieldToInit = R->getField(InitIndex); @@ -1180,7 +1170,7 @@ bool ByteCodeExprGen::visitInitList(ArrayRef Inits, } else { // Initializer for a direct base class. if (const Record::Base *B = R->getBase(Init->getType())) { - if (!this->emitGetPtrBasePop(B->Offset, Init)) + if (!this->emitGetPtrBase(B->Offset, Init)) return false; if (!this->visitInitializer(Init)) @@ -1513,7 +1503,7 @@ bool ByteCodeExprGen::VisitMemberExpr(const MemberExpr *E) { // Leave a pointer to the field on the stack. if (F->Decl->getType()->isReferenceType()) return this->emitGetFieldPop(PT_Ptr, F->Offset, E) && maybeLoadValue(); - return this->emitGetPtrField(F->Offset, E) && maybeLoadValue(); + return this->emitGetPtrFieldPop(F->Offset, E) && maybeLoadValue(); } return false; @@ -2147,9 +2137,6 @@ bool ByteCodeExprGen::VisitLambdaExpr(const LambdaExpr *E) { if (!this->emitInitField(*T, F.Offset, E)) return false; } else { - if (!this->emitDupPtr(E)) - return false; - if (!this->emitGetPtrField(F.Offset, E)) return false; @@ -2846,9 +2833,6 @@ bool ByteCodeExprGen::visitZeroRecordInitializer(const Record *R, continue; } - // TODO: Add GetPtrFieldPop and get rid of this dup. - if (!this->emitDupPtr(E)) - return false; if (!this->emitGetPtrField(Field.Offset, E)) return false; @@ -3258,8 +3242,6 @@ bool ByteCodeExprGen::visitAPValueInitializer(const APValue &Val, PrimType ElemT = classifyPrim(ArrType->getElementType()); assert(ArrType); - if (!this->emitDupPtr(E)) - return false; if (!this->emitGetPtrField(RF->Offset, E)) return false; @@ -3273,8 +3255,6 @@ bool ByteCodeExprGen::visitAPValueInitializer(const APValue &Val, if (!this->emitPopPtr(E)) return false; } else if (F.isStruct() || F.isUnion()) { - if (!this->emitDupPtr(E)) - return false; if (!this->emitGetPtrField(RF->Offset, E)) return false; if (!this->visitAPValueInitializer(F, E)) @@ -4201,8 +4181,6 @@ bool ByteCodeExprGen::emitRecordDestruction(const Record *R) { for (const Record::Field &Field : llvm::reverse(R->fields())) { const Descriptor *D = Field.Desc; if (!D->isPrimitive() && !D->isPrimitiveArray()) { - if (!this->emitDupPtr(SourceInfo{})) - return false; if (!this->emitGetPtrField(Field.Offset, SourceInfo{})) return false; if (!this->emitDestruction(D)) diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h index 98caea5c7014..f63711da90c7 100644 --- a/clang/lib/AST/Interp/Interp.h +++ b/clang/lib/AST/Interp/Interp.h @@ -1283,9 +1283,32 @@ inline bool GetPtrGlobal(InterpState &S, CodePtr OpPC, uint32_t I) { return true; } -/// 1) Pops a Pointer from the stack +/// 1) Peeks a Pointer /// 2) Pushes Pointer.atField(Off) on the stack inline bool GetPtrField(InterpState &S, CodePtr OpPC, uint32_t Off) { + const Pointer &Ptr = S.Stk.peek(); + + if (S.getLangOpts().CPlusPlus && S.inConstantContext() && + !CheckNull(S, OpPC, Ptr, CSK_Field)) + return false; + + if (!CheckExtern(S, OpPC, Ptr)) + return false; + if (!CheckRange(S, OpPC, Ptr, CSK_Field)) + return false; + if (!CheckArray(S, OpPC, Ptr)) + return false; + if (!CheckSubobject(S, OpPC, Ptr, CSK_Field)) + return false; + + if (Ptr.isBlockPointer() && Off > Ptr.block()->getSize()) + return false; + + S.Stk.push(Ptr.atField(Off)); + return true; +} + +inline bool GetPtrFieldPop(InterpState &S, CodePtr OpPC, uint32_t Off) { const Pointer &Ptr = S.Stk.pop(); if (S.getLangOpts().CPlusPlus && S.inConstantContext() && diff --git a/clang/lib/AST/Interp/Opcodes.td b/clang/lib/AST/Interp/Opcodes.td index cb4f299c8d51..a5ac8206104c 100644 --- a/clang/lib/AST/Interp/Opcodes.td +++ b/clang/lib/AST/Interp/Opcodes.td @@ -281,53 +281,31 @@ def Null : Opcode { //===----------------------------------------------------------------------===// // Pointer generation //===----------------------------------------------------------------------===// +class OffsetOpcode : Opcode { + let Args = [ArgUint32]; +} // [] -> [Pointer] -def GetPtrLocal : Opcode { - // Offset of local. - let Args = [ArgUint32]; +def GetPtrLocal : OffsetOpcode { bit HasCustomEval = 1; } // [] -> [Pointer] -def GetPtrParam : Opcode { - // Offset of parameter. - let Args = [ArgUint32]; -} +def GetPtrParam : OffsetOpcode; // [] -> [Pointer] -def GetPtrGlobal : Opcode { - // Index of global. - let Args = [ArgUint32]; -} +def GetPtrGlobal : OffsetOpcode; // [Pointer] -> [Pointer] -def GetPtrField : Opcode { - // Offset of field. - let Args = [ArgUint32]; -} +def GetPtrField : OffsetOpcode; +def GetPtrFieldPop : OffsetOpcode; // [Pointer] -> [Pointer] -def GetPtrActiveField : Opcode { - // Offset of field. - let Args = [ArgUint32]; -} +def GetPtrActiveField : OffsetOpcode; // [] -> [Pointer] -def GetPtrActiveThisField : Opcode { - // Offset of field. - let Args = [ArgUint32]; -} +def GetPtrActiveThisField : OffsetOpcode; // [] -> [Pointer] -def GetPtrThisField : Opcode { - // Offset of field. - let Args = [ArgUint32]; -} +def GetPtrThisField : OffsetOpcode; // [Pointer] -> [Pointer] -def GetPtrBase : Opcode { - // Offset of field, which is a base. - let Args = [ArgUint32]; -} +def GetPtrBase : OffsetOpcode; // [Pointer] -> [Pointer] -def GetPtrBasePop : Opcode { - // Offset of field, which is a base. - let Args = [ArgUint32]; -} +def GetPtrBasePop : OffsetOpcode; def GetMemberPtrBasePop : Opcode { // Offset of field, which is a base. let Args = [ArgSint32]; -- GitLab From e622996eddfb2826d258b3a3760eed195f97aabe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Umann?= Date: Fri, 7 Jun 2024 09:49:16 +0200 Subject: [PATCH 205/462] [analyzer][NFC] Factor out NoOwnershipChangeVisitor (#94357) In preparation for adding essentially the same visitor to StreamChecker, this patch factors this visitor out to a common header. I'll be the first to admit that the interface of these classes are not terrific, but it rather tightly held back by its main technical debt, which is NoStoreFuncVisitor, the main descendant of NoStateChangeVisitor. Change-Id: I99d73ccd93a18dd145bbbc83afadbb432dd42b90 --- .../StaticAnalyzer/Checkers/CMakeLists.txt | 1 + .../StaticAnalyzer/Checkers/MallocChecker.cpp | 147 +++--------------- .../Checkers/NoOwnershipChangeVisitor.cpp | 116 ++++++++++++++ .../Checkers/NoOwnershipChangeVisitor.h | 77 +++++++++ 4 files changed, 212 insertions(+), 129 deletions(-) create mode 100644 clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.cpp create mode 100644 clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.h diff --git a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt index cd5a3bdd02e4..68e829cace49 100644 --- a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt +++ b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt @@ -78,6 +78,7 @@ add_clang_library(clangStaticAnalyzerCheckers NoReturnFunctionChecker.cpp NonNullParamChecker.cpp NonnullGlobalConstantsChecker.cpp + NoOwnershipChangeVisitor.cpp NullabilityChecker.cpp NumberObjectConversionChecker.cpp ObjCAtSyncChecker.cpp diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp index 467cf8889b6d..fe202c79ed62 100644 --- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp @@ -46,6 +46,7 @@ #include "AllocationState.h" #include "InterCheckerAPI.h" +#include "NoOwnershipChangeVisitor.h" #include "clang/AST/Attr.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/DeclTemplate.h" @@ -79,13 +80,11 @@ #include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetOperations.h" -#include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include #include #include #include @@ -414,7 +413,7 @@ private: bool isFreeingCall(const CallEvent &Call) const; static bool isFreeingOwnershipAttrCall(const FunctionDecl *Func); - friend class NoOwnershipChangeVisitor; + friend class NoMemOwnershipChangeVisitor; CallDescriptionMap AllocatingMemFnMap{ {{CDM::CLibrary, {"alloca"}, 1}, &MallocChecker::checkAlloca}, @@ -765,61 +764,8 @@ private: //===----------------------------------------------------------------------===// namespace { -class NoOwnershipChangeVisitor final : public NoStateChangeFuncVisitor { - // The symbol whose (lack of) ownership change we are interested in. - SymbolRef Sym; - const MallocChecker &Checker; - using OwnerSet = llvm::SmallPtrSet; - - // Collect which entities point to the allocated memory, and could be - // responsible for deallocating it. - class OwnershipBindingsHandler : public StoreManager::BindingsHandler { - SymbolRef Sym; - OwnerSet &Owners; - - public: - OwnershipBindingsHandler(SymbolRef Sym, OwnerSet &Owners) - : Sym(Sym), Owners(Owners) {} - - bool HandleBinding(StoreManager &SMgr, Store Store, const MemRegion *Region, - SVal Val) override { - if (Val.getAsSymbol() == Sym) - Owners.insert(Region); - return true; - } - - LLVM_DUMP_METHOD void dump() const { dumpToStream(llvm::errs()); } - LLVM_DUMP_METHOD void dumpToStream(llvm::raw_ostream &out) const { - out << "Owners: {\n"; - for (const MemRegion *Owner : Owners) { - out << " "; - Owner->dumpToStream(out); - out << ",\n"; - } - out << "}\n"; - } - }; - +class NoMemOwnershipChangeVisitor final : public NoOwnershipChangeVisitor { protected: - OwnerSet getOwnersAtNode(const ExplodedNode *N) { - OwnerSet Ret; - - ProgramStateRef State = N->getState(); - OwnershipBindingsHandler Handler{Sym, Ret}; - State->getStateManager().getStoreManager().iterBindings(State->getStore(), - Handler); - return Ret; - } - - LLVM_DUMP_METHOD static std::string - getFunctionName(const ExplodedNode *CallEnterN) { - if (const CallExpr *CE = llvm::dyn_cast_or_null( - CallEnterN->getLocationAs()->getCallExpr())) - if (const FunctionDecl *FD = CE->getDirectCallee()) - return FD->getQualifiedNameAsString(); - return ""; - } - /// Syntactically checks whether the callee is a deallocating function. Since /// we have no path-sensitive information on this call (we would need a /// CallEvent instead of a CallExpr for that), its possible that a @@ -828,8 +774,9 @@ protected: /// See namespace `memory_passed_to_fn_call_free_through_fn_ptr` in /// clang/test/Analysis/NewDeleteLeaks.cpp. bool isFreeingCallAsWritten(const CallExpr &Call) const { - if (Checker.FreeingMemFnMap.lookupAsWritten(Call) || - Checker.ReallocatingMemFnMap.lookupAsWritten(Call)) + const auto *MallocChk = static_cast(&Checker); + if (MallocChk->FreeingMemFnMap.lookupAsWritten(Call) || + MallocChk->ReallocatingMemFnMap.lookupAsWritten(Call)) return true; if (const auto *Func = @@ -839,23 +786,21 @@ protected: return false; } + bool hasResourceStateChanged(ProgramStateRef CallEnterState, + ProgramStateRef CallExitEndState) final { + return CallEnterState->get(Sym) != + CallExitEndState->get(Sym); + } + /// Heuristically guess whether the callee intended to free memory. This is /// done syntactically, because we are trying to argue about alternative /// paths of execution, and as a consequence we don't have path-sensitive /// information. - bool doesFnIntendToHandleOwnership(const Decl *Callee, ASTContext &ACtx) { + bool doesFnIntendToHandleOwnership(const Decl *Callee, + ASTContext &ACtx) final { using namespace clang::ast_matchers; const FunctionDecl *FD = dyn_cast(Callee); - // Given that the stack frame was entered, the body should always be - // theoretically obtainable. In case of body farms, the synthesized body - // is not attached to declaration, thus triggering the '!FD->hasBody()' - // branch. That said, would a synthesized body ever intend to handle - // ownership? As of today they don't. And if they did, how would we - // put notes inside it, given that it doesn't match any source locations? - if (!FD || !FD->hasBody()) - return false; - auto Matches = match(findAll(stmt(anyOf(cxxDeleteExpr().bind("delete"), callExpr().bind("call")))), *FD->getBody(), ACtx); @@ -873,30 +818,7 @@ protected: return false; } - bool wasModifiedInFunction(const ExplodedNode *CallEnterN, - const ExplodedNode *CallExitEndN) override { - if (!doesFnIntendToHandleOwnership( - CallExitEndN->getFirstPred()->getLocationContext()->getDecl(), - CallExitEndN->getState()->getAnalysisManager().getASTContext())) - return true; - - if (CallEnterN->getState()->get(Sym) != - CallExitEndN->getState()->get(Sym)) - return true; - - OwnerSet CurrOwners = getOwnersAtNode(CallEnterN); - OwnerSet ExitOwners = getOwnersAtNode(CallExitEndN); - - // Owners in the current set may be purged from the analyzer later on. - // If a variable is dead (is not referenced directly or indirectly after - // some point), it will be removed from the Store before the end of its - // actual lifetime. - // This means that if the ownership status didn't change, CurrOwners - // must be a superset of, but not necessarily equal to ExitOwners. - return !llvm::set_is_subset(ExitOwners, CurrOwners); - } - - static PathDiagnosticPieceRef emitNote(const ExplodedNode *N) { + PathDiagnosticPieceRef emitNote(const ExplodedNode *N) final { PathDiagnosticLocation L = PathDiagnosticLocation::create( N->getLocation(), N->getState()->getStateManager().getContext().getSourceManager()); @@ -905,42 +827,9 @@ protected: "later deallocation"); } - PathDiagnosticPieceRef - maybeEmitNoteForObjCSelf(PathSensitiveBugReport &R, - const ObjCMethodCall &Call, - const ExplodedNode *N) override { - // TODO: Implement. - return nullptr; - } - - PathDiagnosticPieceRef - maybeEmitNoteForCXXThis(PathSensitiveBugReport &R, - const CXXConstructorCall &Call, - const ExplodedNode *N) override { - // TODO: Implement. - return nullptr; - } - - PathDiagnosticPieceRef - maybeEmitNoteForParameters(PathSensitiveBugReport &R, const CallEvent &Call, - const ExplodedNode *N) override { - // TODO: Factor the logic of "what constitutes as an entity being passed - // into a function call" out by reusing the code in - // NoStoreFuncVisitor::maybeEmitNoteForParameters, maybe by incorporating - // the printing technology in UninitializedObject's FieldChainInfo. - ArrayRef Parameters = Call.parameters(); - for (unsigned I = 0; I < Call.getNumArgs() && I < Parameters.size(); ++I) { - SVal V = Call.getArgSVal(I); - if (V.getAsSymbol() == Sym) - return emitNote(N); - } - return nullptr; - } - public: - NoOwnershipChangeVisitor(SymbolRef Sym, const MallocChecker *Checker) - : NoStateChangeFuncVisitor(bugreporter::TrackingKind::Thorough), Sym(Sym), - Checker(*Checker) {} + NoMemOwnershipChangeVisitor(SymbolRef Sym, const MallocChecker *Checker) + : NoOwnershipChangeVisitor(Sym, Checker) {} void Profile(llvm::FoldingSetNodeID &ID) const override { static int Tag = 0; @@ -2949,7 +2838,7 @@ void MallocChecker::HandleLeak(SymbolRef Sym, ExplodedNode *N, R->markInteresting(Sym); R->addVisitor(Sym, true); if (ShouldRegisterNoOwnershipChangeVisitor) - R->addVisitor(Sym, this); + R->addVisitor(Sym, this); C.emitReport(std::move(R)); } diff --git a/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.cpp b/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.cpp new file mode 100644 index 000000000000..2ff76679b5eb --- /dev/null +++ b/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.cpp @@ -0,0 +1,116 @@ +//===--------------------------------------------------------------*- C++ -*--// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "NoOwnershipChangeVisitor.h" +#include "clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState_Fwd.h" +#include "llvm/ADT/SetOperations.h" + +using namespace clang; +using namespace ento; +using OwnerSet = NoOwnershipChangeVisitor::OwnerSet; + +// Collect which entities point to the allocated memory, and could be +// responsible for deallocating it. +class OwnershipBindingsHandler : public StoreManager::BindingsHandler { + SymbolRef Sym; + OwnerSet &Owners; + +public: + OwnershipBindingsHandler(SymbolRef Sym, OwnerSet &Owners) + : Sym(Sym), Owners(Owners) {} + + bool HandleBinding(StoreManager &SMgr, Store Store, const MemRegion *Region, + SVal Val) override { + if (Val.getAsSymbol() == Sym) + Owners.insert(Region); + return true; + } + + LLVM_DUMP_METHOD void dump() const { dumpToStream(llvm::errs()); } + LLVM_DUMP_METHOD void dumpToStream(llvm::raw_ostream &out) const { + out << "Owners: {\n"; + for (const MemRegion *Owner : Owners) { + out << " "; + Owner->dumpToStream(out); + out << ",\n"; + } + out << "}\n"; + } +}; + +OwnerSet NoOwnershipChangeVisitor::getOwnersAtNode(const ExplodedNode *N) { + OwnerSet Ret; + + ProgramStateRef State = N->getState(); + OwnershipBindingsHandler Handler{Sym, Ret}; + State->getStateManager().getStoreManager().iterBindings(State->getStore(), + Handler); + return Ret; +} + +LLVM_DUMP_METHOD std::string +NoOwnershipChangeVisitor::getFunctionName(const ExplodedNode *CallEnterN) { + if (const CallExpr *CE = llvm::dyn_cast_or_null( + CallEnterN->getLocationAs()->getCallExpr())) + if (const FunctionDecl *FD = CE->getDirectCallee()) + return FD->getQualifiedNameAsString(); + return ""; +} + +bool NoOwnershipChangeVisitor::wasModifiedInFunction( + const ExplodedNode *CallEnterN, const ExplodedNode *CallExitEndN) { + const Decl *Callee = + CallExitEndN->getFirstPred()->getLocationContext()->getDecl(); + const FunctionDecl *FD = dyn_cast(Callee); + + // Given that the stack frame was entered, the body should always be + // theoretically obtainable. In case of body farms, the synthesized body + // is not attached to declaration, thus triggering the '!FD->hasBody()' + // branch. That said, would a synthesized body ever intend to handle + // ownership? As of today they don't. And if they did, how would we + // put notes inside it, given that it doesn't match any source locations? + if (!FD || !FD->hasBody()) + return false; + if (!doesFnIntendToHandleOwnership( + Callee, + CallExitEndN->getState()->getAnalysisManager().getASTContext())) + return true; + + if (hasResourceStateChanged(CallEnterN->getState(), CallExitEndN->getState())) + return true; + + OwnerSet CurrOwners = getOwnersAtNode(CallEnterN); + OwnerSet ExitOwners = getOwnersAtNode(CallExitEndN); + + // Owners in the current set may be purged from the analyzer later on. + // If a variable is dead (is not referenced directly or indirectly after + // some point), it will be removed from the Store before the end of its + // actual lifetime. + // This means that if the ownership status didn't change, CurrOwners + // must be a superset of, but not necessarily equal to ExitOwners. + return !llvm::set_is_subset(ExitOwners, CurrOwners); +} + +PathDiagnosticPieceRef NoOwnershipChangeVisitor::maybeEmitNoteForParameters( + PathSensitiveBugReport &R, const CallEvent &Call, const ExplodedNode *N) { + // TODO: Factor the logic of "what constitutes as an entity being passed + // into a function call" out by reusing the code in + // NoStoreFuncVisitor::maybeEmitNoteForParameters, maybe by incorporating + // the printing technology in UninitializedObject's FieldChainInfo. + ArrayRef Parameters = Call.parameters(); + for (unsigned I = 0; I < Call.getNumArgs() && I < Parameters.size(); ++I) { + SVal V = Call.getArgSVal(I); + if (V.getAsSymbol() == Sym) + return emitNote(N); + } + return nullptr; +} diff --git a/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.h b/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.h new file mode 100644 index 000000000000..027f1a156a7c --- /dev/null +++ b/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.h @@ -0,0 +1,77 @@ +//===--------------------------------------------------------------*- C++ -*--// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h" +#include "clang/StaticAnalyzer/Core/Checker.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState_Fwd.h" + +namespace clang { +namespace ento { + +class NoOwnershipChangeVisitor : public NoStateChangeFuncVisitor { +protected: + // The symbol whose (lack of) ownership change we are interested in. + SymbolRef Sym; + const CheckerBase &Checker; + + LLVM_DUMP_METHOD static std::string + getFunctionName(const ExplodedNode *CallEnterN); + + /// Heuristically guess whether the callee intended to free the resource. This + /// is done syntactically, because we are trying to argue about alternative + /// paths of execution, and as a consequence we don't have path-sensitive + /// information. + virtual bool doesFnIntendToHandleOwnership(const Decl *Callee, + ASTContext &ACtx) = 0; + + virtual bool hasResourceStateChanged(ProgramStateRef CallEnterState, + ProgramStateRef CallExitEndState) = 0; + + bool wasModifiedInFunction(const ExplodedNode *CallEnterN, + const ExplodedNode *CallExitEndN) final; + + virtual PathDiagnosticPieceRef emitNote(const ExplodedNode *N) = 0; + + PathDiagnosticPieceRef maybeEmitNoteForObjCSelf(PathSensitiveBugReport &R, + const ObjCMethodCall &Call, + const ExplodedNode *N) final { + // TODO: Implement. + return nullptr; + } + + PathDiagnosticPieceRef maybeEmitNoteForCXXThis(PathSensitiveBugReport &R, + const CXXConstructorCall &Call, + const ExplodedNode *N) final { + // TODO: Implement. + return nullptr; + } + + // Set this to final, effectively dispatch to emitNote. + PathDiagnosticPieceRef + maybeEmitNoteForParameters(PathSensitiveBugReport &R, const CallEvent &Call, + const ExplodedNode *N) final; + +public: + using OwnerSet = llvm::SmallPtrSet; + +private: + OwnerSet getOwnersAtNode(const ExplodedNode *N); + +public: + NoOwnershipChangeVisitor(SymbolRef Sym, const CheckerBase *Checker) + : NoStateChangeFuncVisitor(bugreporter::TrackingKind::Thorough), Sym(Sym), + Checker(*Checker) {} + + void Profile(llvm::FoldingSetNodeID &ID) const override { + static int Tag = 0; + ID.AddPointer(&Tag); + ID.AddPointer(Sym); + } +}; +} // namespace ento +} // namespace clang -- GitLab From be18daad06a9b431d0b4b787bba81de579e0a583 Mon Sep 17 00:00:00 2001 From: Jianjian GUAN Date: Thu, 6 Jun 2024 14:39:01 +0800 Subject: [PATCH 206/462] Reland "[RISCV] Support select/merge like ops for bf16 vectors when have Zvfbfmin" (#94565)" --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 32 +- .../Target/RISCV/RISCVInstrInfoVPseudos.td | 15 +- .../Target/RISCV/RISCVInstrInfoVSDPatterns.td | 8 +- .../Target/RISCV/RISCVInstrInfoVVLPatterns.td | 5 +- .../RISCV/rvv/fixed-vectors-select-fp.ll | 128 +++++++- .../RISCV/rvv/fixed-vectors-vpmerge.ll | 144 +++++++- .../RISCV/rvv/fixed-vectors-vselect-vp.ll | 56 +++- llvm/test/CodeGen/RISCV/rvv/select-fp.ll | 188 ++++++++++- llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll | 308 +++++++++++++++++- llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll | 224 ++++++++++++- llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll | 80 ++++- 11 files changed, 1150 insertions(+), 38 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 8ace5d79af07..4051279fdbf8 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -1102,6 +1102,15 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::EXTRACT_SUBVECTOR}, VT, Custom); setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); + if (Subtarget.hasStdExtZfbfmin()) { + if (Subtarget.hasVInstructionsF16()) + setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); + else if (Subtarget.hasVInstructionsF16Minimal()) + setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); + } + setOperationAction({ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT, + Custom); + setOperationAction(ISD::SELECT_CC, VT, Expand); // TODO: Promote to fp32. } } @@ -1331,6 +1340,15 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::EXTRACT_SUBVECTOR}, VT, Custom); setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); + if (Subtarget.hasStdExtZfbfmin()) { + if (Subtarget.hasVInstructionsF16()) + setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); + else if (Subtarget.hasVInstructionsF16Minimal()) + setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); + } + setOperationAction( + {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT, + Custom); // TODO: Promote to fp32. continue; } @@ -6704,10 +6722,16 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, case ISD::BUILD_VECTOR: return lowerBUILD_VECTOR(Op, DAG, Subtarget); case ISD::SPLAT_VECTOR: - if (Op.getValueType().getScalarType() == MVT::f16 && - (Subtarget.hasVInstructionsF16Minimal() && - !Subtarget.hasVInstructionsF16())) { - if (Op.getValueType() == MVT::nxv32f16) + if ((Op.getValueType().getScalarType() == MVT::f16 && + (Subtarget.hasVInstructionsF16Minimal() && + Subtarget.hasStdExtZfhminOrZhinxmin() && + !Subtarget.hasVInstructionsF16())) || + (Op.getValueType().getScalarType() == MVT::bf16 && + (Subtarget.hasVInstructionsBF16() && Subtarget.hasStdExtZfbfmin() && + Subtarget.hasVInstructionsF16Minimal() && + !Subtarget.hasVInstructionsF16()))) { + if (Op.getValueType() == MVT::nxv32f16 || + Op.getValueType() == MVT::nxv32bf16) return SplitVectorOp(Op, DAG); SDLoc DL(Op); SDValue NewScalar = diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index b0949f5fc1d7..fe4d839e4fdc 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -382,7 +382,20 @@ class GetIntVTypeInfo { // Equivalent integer vector type. Eg. // VI8M1 → VI8M1 (identity) // VF64M4 → VI64M4 - VTypeInfo Vti = !cast(!subst("VF", "VI", !cast(vti))); + VTypeInfo Vti = !cast(!subst("VBF", "VI", + !subst("VF", "VI", + !cast(vti)))); +} + +// This functor is used to obtain the fp vector type that has the same SEW and +// multiplier as the input parameter type. +class GetFpVTypeInfo { + // Equivalent integer vector type. Eg. + // VF16M1 → VF16M1 (identity) + // VBF16M1 → VF16M1 + VTypeInfo Vti = !cast(!subst("VBF", "VF", + !subst("VI", "VF", + !cast(vti)))); } class MTypeInfo { diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td index 3163e4bafd4b..497c4aadf753 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td @@ -1394,7 +1394,7 @@ defm : VPatFPSetCCSDNode_VV_VF_FV; // Floating-point vselects: // 11.15. Vector Integer Merge Instructions // 13.15. Vector Floating-Point Merge Instruction -foreach fvti = AllFloatVectors in { +foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in { defvar ivti = GetIntVTypeInfo.Vti; let Predicates = GetVTypePredicates.Predicates in { def : Pat<(fvti.Vector (vselect (fvti.Mask V0), fvti.RegClass:$rs1, @@ -1412,7 +1412,9 @@ foreach fvti = AllFloatVectors in { fvti.RegClass:$rs2, 0, (fvti.Mask V0), fvti.AVL, fvti.Log2SEW)>; } - let Predicates = GetVTypePredicates.Predicates in + + let Predicates = !listconcat(GetVTypePredicates.Vti>.Predicates, + GetVTypeScalarPredicates.Predicates) in def : Pat<(fvti.Vector (vselect (fvti.Mask V0), (SplatFPOp fvti.ScalarRegClass:$rs1), fvti.RegClass:$rs2)), @@ -1475,7 +1477,7 @@ foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in { //===----------------------------------------------------------------------===// foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in { - let Predicates = !listconcat(GetVTypePredicates.Predicates, + let Predicates = !listconcat(GetVTypePredicates.Vti>.Predicates, GetVTypeScalarPredicates.Predicates) in def : Pat<(fvti.Vector (riscv_vfmv_v_f_vl undef, fvti.ScalarRegClass:$rs1, srcvalue)), (!cast("PseudoVFMV_V_"#fvti.ScalarSuffix#"_"#fvti.LMul.MX) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td index ce8133a5a297..70d8265e7be4 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td @@ -2604,7 +2604,7 @@ foreach vti = AllFloatVectors in { } } -foreach fvti = AllFloatVectors in { +foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in { // Floating-point vselects: // 11.15. Vector Integer Merge Instructions // 13.15. Vector Floating-Point Merge Instruction @@ -2639,7 +2639,8 @@ foreach fvti = AllFloatVectors in { GPR:$vl, fvti.Log2SEW)>; } - let Predicates = GetVTypePredicates.Predicates in { + let Predicates = !listconcat(GetVTypePredicates.Vti>.Predicates, + GetVTypeScalarPredicates.Predicates) in { def : Pat<(fvti.Vector (riscv_vmerge_vl (fvti.Mask V0), (SplatFPOp fvti.ScalarRegClass:$rs1), fvti.RegClass:$rs2, diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll index d945cf561698..7a96aad31f08 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-select-fp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d -riscv-v-vector-bits-min=128 \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d -riscv-v-vector-bits-min=128 \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d -riscv-v-vector-bits-min=128 \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d -riscv-v-vector-bits-min=128 \ ; RUN: -verify-machineinstrs < %s | FileCheck %s define <2 x half> @select_v2f16(i1 zeroext %c, <2 x half> %a, <2 x half> %b) { @@ -343,3 +343,123 @@ define <16 x double> @selectcc_v16f64(double %a, double %b, <16 x double> %c, <1 %v = select i1 %cmp, <16 x double> %c, <16 x double> %d ret <16 x double> %v } + +define <2 x bfloat> @select_v2bf16(i1 zeroext %c, <2 x bfloat> %a, <2 x bfloat> %b) { +; CHECK-LABEL: select_v2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, <2 x bfloat> %a, <2 x bfloat> %b + ret <2 x bfloat> %v +} + +define <2 x bfloat> @selectcc_v2bf16(bfloat %a, bfloat %b, <2 x bfloat> %c, <2 x bfloat> %d) { +; CHECK-LABEL: selectcc_v2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, <2 x bfloat> %c, <2 x bfloat> %d + ret <2 x bfloat> %v +} + +define <4 x bfloat> @select_v4bf16(i1 zeroext %c, <4 x bfloat> %a, <4 x bfloat> %b) { +; CHECK-LABEL: select_v4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, <4 x bfloat> %a, <4 x bfloat> %b + ret <4 x bfloat> %v +} + +define <4 x bfloat> @selectcc_v4bf16(bfloat %a, bfloat %b, <4 x bfloat> %c, <4 x bfloat> %d) { +; CHECK-LABEL: selectcc_v4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, <4 x bfloat> %c, <4 x bfloat> %d + ret <4 x bfloat> %v +} + +define <8 x bfloat> @select_v8bf16(i1 zeroext %c, <8 x bfloat> %a, <8 x bfloat> %b) { +; CHECK-LABEL: select_v8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, <8 x bfloat> %a, <8 x bfloat> %b + ret <8 x bfloat> %v +} + +define <8 x bfloat> @selectcc_v8bf16(bfloat %a, bfloat %b, <8 x bfloat> %c, <8 x bfloat> %d) { +; CHECK-LABEL: selectcc_v8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, <8 x bfloat> %c, <8 x bfloat> %d + ret <8 x bfloat> %v +} + +define <16 x bfloat> @select_v16bf16(i1 zeroext %c, <16 x bfloat> %a, <16 x bfloat> %b) { +; CHECK-LABEL: select_v16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vmsne.vi v0, v12, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, <16 x bfloat> %a, <16 x bfloat> %b + ret <16 x bfloat> %v +} + +define <16 x bfloat> @selectcc_v16bf16(bfloat %a, bfloat %b, <16 x bfloat> %c, <16 x bfloat> %d) { +; CHECK-LABEL: selectcc_v16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vmsne.vi v0, v12, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, <16 x bfloat> %c, <16 x bfloat> %d + ret <16 x bfloat> %v +} diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll index 9f0561b394b8..d360c3f635b5 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,RV32 -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH,RV64 -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,RV32ZVFHMIN -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN,RV64ZVFHMIN declare <4 x i1> @llvm.vp.merge.v4i1(<4 x i1>, <4 x i1>, <4 x i1>, i32) @@ -1240,3 +1240,139 @@ define <32 x double> @vpmerge_vf_v32f64(double %a, <32 x double> %vb, <32 x i1> %v = call <32 x double> @llvm.vp.merge.v32f64(<32 x i1> %m, <32 x double> %va, <32 x double> %vb, i32 %evl) ret <32 x double> %v } + +declare <2 x bfloat> @llvm.vp.merge.v2bf16(<2 x i1>, <2 x bfloat>, <2 x bfloat>, i32) + +define <2 x bfloat> @vpmerge_vv_v2bf16(<2 x bfloat> %va, <2 x bfloat> %vb, <2 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_v2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf4, tu, ma +; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call <2 x bfloat> @llvm.vp.merge.v2bf16(<2 x i1> %m, <2 x bfloat> %va, <2 x bfloat> %vb, i32 %evl) + ret <2 x bfloat> %v +} + +define <2 x bfloat> @vpmerge_vf_v2bf16(bfloat %a, <2 x bfloat> %vb, <2 x i1> %m, i32 zeroext %evl) { +; ZVFH-LABEL: vpmerge_vf_v2bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, mf4, tu, ma +; ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vpmerge_vf_v2bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a1, zero, e32, mf2, ta, ma +; ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, tu, mu +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFHMIN-NEXT: ret + %elt.head = insertelement <2 x bfloat> poison, bfloat %a, i32 0 + %va = shufflevector <2 x bfloat> %elt.head, <2 x bfloat> poison, <2 x i32> zeroinitializer + %v = call <2 x bfloat> @llvm.vp.merge.v2bf16(<2 x i1> %m, <2 x bfloat> %va, <2 x bfloat> %vb, i32 %evl) + ret <2 x bfloat> %v +} + +declare <4 x bfloat> @llvm.vp.merge.v4bf16(<4 x i1>, <4 x bfloat>, <4 x bfloat>, i32) + +define <4 x bfloat> @vpmerge_vv_v4bf16(<4 x bfloat> %va, <4 x bfloat> %vb, <4 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_v4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, tu, ma +; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call <4 x bfloat> @llvm.vp.merge.v4bf16(<4 x i1> %m, <4 x bfloat> %va, <4 x bfloat> %vb, i32 %evl) + ret <4 x bfloat> %v +} + +define <4 x bfloat> @vpmerge_vf_v4bf16(bfloat %a, <4 x bfloat> %vb, <4 x i1> %m, i32 zeroext %evl) { +; ZVFH-LABEL: vpmerge_vf_v4bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, mf2, tu, ma +; ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vpmerge_vf_v4bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a1, zero, e32, m1, ta, ma +; ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf2, tu, mu +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; ZVFHMIN-NEXT: ret + %elt.head = insertelement <4 x bfloat> poison, bfloat %a, i32 0 + %va = shufflevector <4 x bfloat> %elt.head, <4 x bfloat> poison, <4 x i32> zeroinitializer + %v = call <4 x bfloat> @llvm.vp.merge.v4bf16(<4 x i1> %m, <4 x bfloat> %va, <4 x bfloat> %vb, i32 %evl) + ret <4 x bfloat> %v +} + +declare <8 x bfloat> @llvm.vp.merge.v8bf16(<8 x i1>, <8 x bfloat>, <8 x bfloat>, i32) + +define <8 x bfloat> @vpmerge_vv_v8bf16(<8 x bfloat> %va, <8 x bfloat> %vb, <8 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_v8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m1, tu, ma +; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call <8 x bfloat> @llvm.vp.merge.v8bf16(<8 x i1> %m, <8 x bfloat> %va, <8 x bfloat> %vb, i32 %evl) + ret <8 x bfloat> %v +} + +define <8 x bfloat> @vpmerge_vf_v8bf16(bfloat %a, <8 x bfloat> %vb, <8 x i1> %m, i32 zeroext %evl) { +; ZVFH-LABEL: vpmerge_vf_v8bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, m1, tu, ma +; ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vpmerge_vf_v8bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a1, zero, e32, m2, ta, ma +; ZVFHMIN-NEXT: vfmv.v.f v10, fa5 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m1, tu, mu +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t +; ZVFHMIN-NEXT: ret + %elt.head = insertelement <8 x bfloat> poison, bfloat %a, i32 0 + %va = shufflevector <8 x bfloat> %elt.head, <8 x bfloat> poison, <8 x i32> zeroinitializer + %v = call <8 x bfloat> @llvm.vp.merge.v8bf16(<8 x i1> %m, <8 x bfloat> %va, <8 x bfloat> %vb, i32 %evl) + ret <8 x bfloat> %v +} + +declare <16 x bfloat> @llvm.vp.merge.v16bf16(<16 x i1>, <16 x bfloat>, <16 x bfloat>, i32) + +define <16 x bfloat> @vpmerge_vv_v16bf16(<16 x bfloat> %va, <16 x bfloat> %vb, <16 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_v16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m2, tu, ma +; CHECK-NEXT: vmerge.vvm v10, v10, v8, v0 +; CHECK-NEXT: vmv2r.v v8, v10 +; CHECK-NEXT: ret + %v = call <16 x bfloat> @llvm.vp.merge.v16bf16(<16 x i1> %m, <16 x bfloat> %va, <16 x bfloat> %vb, i32 %evl) + ret <16 x bfloat> %v +} + +define <16 x bfloat> @vpmerge_vf_v16bf16(bfloat %a, <16 x bfloat> %vb, <16 x i1> %m, i32 zeroext %evl) { +; ZVFH-LABEL: vpmerge_vf_v16bf16: +; ZVFH: # %bb.0: +; ZVFH-NEXT: vsetvli zero, a0, e16, m2, tu, ma +; ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; ZVFH-NEXT: ret +; +; ZVFHMIN-LABEL: vpmerge_vf_v16bf16: +; ZVFHMIN: # %bb.0: +; ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; ZVFHMIN-NEXT: vsetvli a1, zero, e32, m4, ta, ma +; ZVFHMIN-NEXT: vfmv.v.f v12, fa5 +; ZVFHMIN-NEXT: vsetvli zero, a0, e16, m2, tu, mu +; ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t +; ZVFHMIN-NEXT: ret + %elt.head = insertelement <16 x bfloat> poison, bfloat %a, i32 0 + %va = shufflevector <16 x bfloat> %elt.head, <16 x bfloat> poison, <16 x i32> zeroinitializer + %v = call <16 x bfloat> @llvm.vp.merge.v16bf16(<16 x i1> %m, <16 x bfloat> %va, <16 x bfloat> %vb, i32 %evl) + ret <16 x bfloat> %v +} diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll index 0a2ed3eb1ffb..c5d9cdacae74 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect-vp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d -riscv-v-vector-bits-min=128 \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d -riscv-v-vector-bits-min=128 \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d -riscv-v-vector-bits-min=128 \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d -riscv-v-vector-bits-min=128 \ ; RUN: -verify-machineinstrs < %s | FileCheck %s declare <1 x i1> @llvm.vp.select.v1i1(<1 x i1>, <1 x i1>, <1 x i1>, i32) @@ -683,3 +683,51 @@ define <16 x double> @select_v16f64(<16 x i1> %a, <16 x double> %b, <16 x double %v = call <16 x double> @llvm.vp.select.v16f64(<16 x i1> %a, <16 x double> %b, <16 x double> %c, i32 %evl) ret <16 x double> %v } + +declare <2 x bfloat> @llvm.vp.select.v2bf16(<2 x i1>, <2 x bfloat>, <2 x bfloat>, i32) + +define <2 x bfloat> @select_v2bf16(<2 x i1> %a, <2 x bfloat> %b, <2 x bfloat> %c, i32 zeroext %evl) { +; CHECK-LABEL: select_v2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = call <2 x bfloat> @llvm.vp.select.v2bf16(<2 x i1> %a, <2 x bfloat> %b, <2 x bfloat> %c, i32 %evl) + ret <2 x bfloat> %v +} + +declare <4 x bfloat> @llvm.vp.select.v4bf16(<4 x i1>, <4 x bfloat>, <4 x bfloat>, i32) + +define <4 x bfloat> @select_v4bf16(<4 x i1> %a, <4 x bfloat> %b, <4 x bfloat> %c, i32 zeroext %evl) { +; CHECK-LABEL: select_v4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = call <4 x bfloat> @llvm.vp.select.v4bf16(<4 x i1> %a, <4 x bfloat> %b, <4 x bfloat> %c, i32 %evl) + ret <4 x bfloat> %v +} + +declare <8 x bfloat> @llvm.vp.select.v8bf16(<8 x i1>, <8 x bfloat>, <8 x bfloat>, i32) + +define <8 x bfloat> @select_v8bf16(<8 x i1> %a, <8 x bfloat> %b, <8 x bfloat> %c, i32 zeroext %evl) { +; CHECK-LABEL: select_v8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = call <8 x bfloat> @llvm.vp.select.v8bf16(<8 x i1> %a, <8 x bfloat> %b, <8 x bfloat> %c, i32 %evl) + ret <8 x bfloat> %v +} + +declare <16 x bfloat> @llvm.vp.select.v16bf16(<16 x i1>, <16 x bfloat>, <16 x bfloat>, i32) + +define <16 x bfloat> @select_v16bf16(<16 x i1> %a, <16 x bfloat> %b, <16 x bfloat> %c, i32 zeroext %evl) { +; CHECK-LABEL: select_v16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %v = call <16 x bfloat> @llvm.vp.select.v16bf16(<16 x i1> %a, <16 x bfloat> %b, <16 x bfloat> %c, i32 %evl) + ret <16 x bfloat> %v +} diff --git a/llvm/test/CodeGen/RISCV/rvv/select-fp.ll b/llvm/test/CodeGen/RISCV/rvv/select-fp.ll index f8581d8e21b3..2b9d847a9e87 100644 --- a/llvm/test/CodeGen/RISCV/rvv/select-fp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/select-fp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s define @select_nxv1f16(i1 zeroext %c, %a, %b) { @@ -427,3 +427,183 @@ define @selectcc_nxv8f64(double %a, double %b, %c, %d ret %v } + +define @select_nxv1bf16(i1 zeroext %c, %a, %b) { +; CHECK-LABEL: select_nxv1bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, %a, %b + ret %v +} + +define @selectcc_nxv1bf16(bfloat %a, bfloat %b, %c, %d) { +; CHECK-LABEL: selectcc_nxv1bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, %c, %d + ret %v +} + +define @select_nxv2bf16(i1 zeroext %c, %a, %b) { +; CHECK-LABEL: select_nxv2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, %a, %b + ret %v +} + +define @selectcc_nxv2bf16(bfloat %a, bfloat %b, %c, %d) { +; CHECK-LABEL: selectcc_nxv2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, %c, %d + ret %v +} + +define @select_nxv4bf16(i1 zeroext %c, %a, %b) { +; CHECK-LABEL: select_nxv4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, %a, %b + ret %v +} + +define @selectcc_nxv4bf16(bfloat %a, bfloat %b, %c, %d) { +; CHECK-LABEL: selectcc_nxv4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vmsne.vi v0, v10, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, %c, %d + ret %v +} + +define @select_nxv8bf16(i1 zeroext %c, %a, %b) { +; CHECK-LABEL: select_nxv8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vmsne.vi v0, v12, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, %a, %b + ret %v +} + +define @selectcc_nxv8bf16(bfloat %a, bfloat %b, %c, %d) { +; CHECK-LABEL: selectcc_nxv8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vmsne.vi v0, v12, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, %c, %d + ret %v +} + +define @select_nxv16bf16(i1 zeroext %c, %a, %b) { +; CHECK-LABEL: select_nxv16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m2, ta, ma +; CHECK-NEXT: vmv.v.x v16, a0 +; CHECK-NEXT: vmsne.vi v0, v16, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, %a, %b + ret %v +} + +define @selectcc_nxv16bf16(bfloat %a, bfloat %b, %c, %d) { +; CHECK-LABEL: selectcc_nxv16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetvli a1, zero, e8, m2, ta, ma +; CHECK-NEXT: vmv.v.x v16, a0 +; CHECK-NEXT: vmsne.vi v0, v16, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, %c, %d + ret %v +} + +define @select_nxv32bf16(i1 zeroext %c, %a, %b) { +; CHECK-LABEL: select_nxv32bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m4, ta, ma +; CHECK-NEXT: vmv.v.x v24, a0 +; CHECK-NEXT: vmsne.vi v0, v24, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m8, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v16, v8, v0 +; CHECK-NEXT: ret + %v = select i1 %c, %a, %b + ret %v +} + +define @selectcc_nxv32bf16(bfloat %a, bfloat %b, %c, %d) { +; CHECK-LABEL: selectcc_nxv32bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: fcvt.s.bf16 fa5, fa1 +; CHECK-NEXT: fcvt.s.bf16 fa4, fa0 +; CHECK-NEXT: feq.s a0, fa4, fa5 +; CHECK-NEXT: vsetvli a1, zero, e8, m4, ta, ma +; CHECK-NEXT: vmv.v.x v24, a0 +; CHECK-NEXT: vmsne.vi v0, v24, 0 +; CHECK-NEXT: vsetvli zero, zero, e16, m8, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v16, v8, v0 +; CHECK-NEXT: ret + %cmp = fcmp oeq bfloat %a, %b + %v = select i1 %cmp, %c, %d + ret %v +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll index 094e6c9cc754..e33c795169fa 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFH -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVFH -; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32ZVFHMIN -; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64ZVFHMIN declare @llvm.vp.merge.nxv1i1(, , , i32) @@ -1547,3 +1547,303 @@ define @vpmerge_vf_nxv8f64(double %a, @llvm.vp.merge.nxv8f64( %m, %va, %vb, i32 %evl) ret %v } + +declare @llvm.vp.merge.nxv1bf16(, , , i32) + +define @vpmerge_vv_nxv1bf16( %va, %vb, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_nxv1bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf4, tu, ma +; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call @llvm.vp.merge.nxv1bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +define @vpmerge_vf_nxv1bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { +; RV32ZVFH-LABEL: vpmerge_vf_nxv1bf16: +; RV32ZVFH: # %bb.0: +; RV32ZVFH-NEXT: vsetvli zero, a0, e16, mf4, tu, ma +; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV32ZVFH-NEXT: ret +; +; RV64ZVFH-LABEL: vpmerge_vf_nxv1bf16: +; RV64ZVFH: # %bb.0: +; RV64ZVFH-NEXT: vsetvli zero, a0, e16, mf4, tu, ma +; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV64ZVFH-NEXT: ret +; +; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv1bf16: +; RV32ZVFHMIN: # %bb.0: +; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, mf2, ta, ma +; RV32ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, tu, mu +; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; RV32ZVFHMIN-NEXT: ret +; +; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv1bf16: +; RV64ZVFHMIN: # %bb.0: +; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, mf2, ta, ma +; RV64ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf4, tu, mu +; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; RV64ZVFHMIN-NEXT: ret + %elt.head = insertelement poison, bfloat %a, i32 0 + %va = shufflevector %elt.head, poison, zeroinitializer + %v = call @llvm.vp.merge.nxv1bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +declare @llvm.vp.merge.nxv2bf16(, , , i32) + +define @vpmerge_vv_nxv2bf16( %va, %vb, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_nxv2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, tu, ma +; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call @llvm.vp.merge.nxv2bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +define @vpmerge_vf_nxv2bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { +; RV32ZVFH-LABEL: vpmerge_vf_nxv2bf16: +; RV32ZVFH: # %bb.0: +; RV32ZVFH-NEXT: vsetvli zero, a0, e16, mf2, tu, ma +; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV32ZVFH-NEXT: ret +; +; RV64ZVFH-LABEL: vpmerge_vf_nxv2bf16: +; RV64ZVFH: # %bb.0: +; RV64ZVFH-NEXT: vsetvli zero, a0, e16, mf2, tu, ma +; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV64ZVFH-NEXT: ret +; +; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv2bf16: +; RV32ZVFHMIN: # %bb.0: +; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m1, ta, ma +; RV32ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf2, tu, mu +; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; RV32ZVFHMIN-NEXT: ret +; +; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv2bf16: +; RV64ZVFHMIN: # %bb.0: +; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m1, ta, ma +; RV64ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, mf2, tu, mu +; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; RV64ZVFHMIN-NEXT: ret + %elt.head = insertelement poison, bfloat %a, i32 0 + %va = shufflevector %elt.head, poison, zeroinitializer + %v = call @llvm.vp.merge.nxv2bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +declare @llvm.vp.merge.nxv4bf16(, , , i32) + +define @vpmerge_vv_nxv4bf16( %va, %vb, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_nxv4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m1, tu, ma +; CHECK-NEXT: vmerge.vvm v9, v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call @llvm.vp.merge.nxv4bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +define @vpmerge_vf_nxv4bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { +; RV32ZVFH-LABEL: vpmerge_vf_nxv4bf16: +; RV32ZVFH: # %bb.0: +; RV32ZVFH-NEXT: vsetvli zero, a0, e16, m1, tu, ma +; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV32ZVFH-NEXT: ret +; +; RV64ZVFH-LABEL: vpmerge_vf_nxv4bf16: +; RV64ZVFH: # %bb.0: +; RV64ZVFH-NEXT: vsetvli zero, a0, e16, m1, tu, ma +; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV64ZVFH-NEXT: ret +; +; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv4bf16: +; RV32ZVFHMIN: # %bb.0: +; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m2, ta, ma +; RV32ZVFHMIN-NEXT: vfmv.v.f v10, fa5 +; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, m1, tu, mu +; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t +; RV32ZVFHMIN-NEXT: ret +; +; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv4bf16: +; RV64ZVFHMIN: # %bb.0: +; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m2, ta, ma +; RV64ZVFHMIN-NEXT: vfmv.v.f v10, fa5 +; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, m1, tu, mu +; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t +; RV64ZVFHMIN-NEXT: ret + %elt.head = insertelement poison, bfloat %a, i32 0 + %va = shufflevector %elt.head, poison, zeroinitializer + %v = call @llvm.vp.merge.nxv4bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +declare @llvm.vp.merge.nxv8bf16(, , , i32) + +define @vpmerge_vv_nxv8bf16( %va, %vb, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_nxv8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m2, tu, ma +; CHECK-NEXT: vmerge.vvm v10, v10, v8, v0 +; CHECK-NEXT: vmv2r.v v8, v10 +; CHECK-NEXT: ret + %v = call @llvm.vp.merge.nxv8bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +define @vpmerge_vf_nxv8bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { +; RV32ZVFH-LABEL: vpmerge_vf_nxv8bf16: +; RV32ZVFH: # %bb.0: +; RV32ZVFH-NEXT: vsetvli zero, a0, e16, m2, tu, ma +; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV32ZVFH-NEXT: ret +; +; RV64ZVFH-LABEL: vpmerge_vf_nxv8bf16: +; RV64ZVFH: # %bb.0: +; RV64ZVFH-NEXT: vsetvli zero, a0, e16, m2, tu, ma +; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV64ZVFH-NEXT: ret +; +; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv8bf16: +; RV32ZVFHMIN: # %bb.0: +; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m4, ta, ma +; RV32ZVFHMIN-NEXT: vfmv.v.f v12, fa5 +; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, m2, tu, mu +; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t +; RV32ZVFHMIN-NEXT: ret +; +; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv8bf16: +; RV64ZVFHMIN: # %bb.0: +; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m4, ta, ma +; RV64ZVFHMIN-NEXT: vfmv.v.f v12, fa5 +; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, m2, tu, mu +; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t +; RV64ZVFHMIN-NEXT: ret + %elt.head = insertelement poison, bfloat %a, i32 0 + %va = shufflevector %elt.head, poison, zeroinitializer + %v = call @llvm.vp.merge.nxv8bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +declare @llvm.vp.merge.nxv16bf16(, , , i32) + +define @vpmerge_vv_nxv16bf16( %va, %vb, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_nxv16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m4, tu, ma +; CHECK-NEXT: vmerge.vvm v12, v12, v8, v0 +; CHECK-NEXT: vmv4r.v v8, v12 +; CHECK-NEXT: ret + %v = call @llvm.vp.merge.nxv16bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +define @vpmerge_vf_nxv16bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { +; RV32ZVFH-LABEL: vpmerge_vf_nxv16bf16: +; RV32ZVFH: # %bb.0: +; RV32ZVFH-NEXT: vsetvli zero, a0, e16, m4, tu, ma +; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV32ZVFH-NEXT: ret +; +; RV64ZVFH-LABEL: vpmerge_vf_nxv16bf16: +; RV64ZVFH: # %bb.0: +; RV64ZVFH-NEXT: vsetvli zero, a0, e16, m4, tu, ma +; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV64ZVFH-NEXT: ret +; +; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv16bf16: +; RV32ZVFHMIN: # %bb.0: +; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m8, ta, ma +; RV32ZVFHMIN-NEXT: vfmv.v.f v16, fa5 +; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, tu, mu +; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; RV32ZVFHMIN-NEXT: ret +; +; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv16bf16: +; RV64ZVFHMIN: # %bb.0: +; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m8, ta, ma +; RV64ZVFHMIN-NEXT: vfmv.v.f v16, fa5 +; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, m4, tu, mu +; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; RV64ZVFHMIN-NEXT: ret + %elt.head = insertelement poison, bfloat %a, i32 0 + %va = shufflevector %elt.head, poison, zeroinitializer + %v = call @llvm.vp.merge.nxv16bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +declare @llvm.vp.merge.nxv32bf16(, , , i32) + +define @vpmerge_vv_nxv32bf16( %va, %vb, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpmerge_vv_nxv32bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m8, tu, ma +; CHECK-NEXT: vmerge.vvm v16, v16, v8, v0 +; CHECK-NEXT: vmv8r.v v8, v16 +; CHECK-NEXT: ret + %v = call @llvm.vp.merge.nxv32bf16( %m, %va, %vb, i32 %evl) + ret %v +} + +define @vpmerge_vf_nxv32bf16(bfloat %a, %vb, %m, i32 zeroext %evl) { +; RV32ZVFH-LABEL: vpmerge_vf_nxv32bf16: +; RV32ZVFH: # %bb.0: +; RV32ZVFH-NEXT: vsetvli zero, a0, e16, m8, tu, ma +; RV32ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV32ZVFH-NEXT: ret +; +; RV64ZVFH-LABEL: vpmerge_vf_nxv32bf16: +; RV64ZVFH: # %bb.0: +; RV64ZVFH-NEXT: vsetvli zero, a0, e16, m8, tu, ma +; RV64ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; RV64ZVFH-NEXT: ret +; +; RV32ZVFHMIN-LABEL: vpmerge_vf_nxv32bf16: +; RV32ZVFHMIN: # %bb.0: +; RV32ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV32ZVFHMIN-NEXT: vsetvli a1, zero, e32, m8, ta, ma +; RV32ZVFHMIN-NEXT: vfmv.v.f v24, fa5 +; RV32ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; RV32ZVFHMIN-NEXT: vfncvtbf16.f.f.w v16, v24 +; RV32ZVFHMIN-NEXT: vmv.v.v v20, v16 +; RV32ZVFHMIN-NEXT: vsetvli zero, a0, e16, m8, tu, ma +; RV32ZVFHMIN-NEXT: vmerge.vvm v8, v8, v16, v0 +; RV32ZVFHMIN-NEXT: ret +; +; RV64ZVFHMIN-LABEL: vpmerge_vf_nxv32bf16: +; RV64ZVFHMIN: # %bb.0: +; RV64ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; RV64ZVFHMIN-NEXT: vsetvli a1, zero, e32, m8, ta, ma +; RV64ZVFHMIN-NEXT: vfmv.v.f v24, fa5 +; RV64ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; RV64ZVFHMIN-NEXT: vfncvtbf16.f.f.w v16, v24 +; RV64ZVFHMIN-NEXT: vmv.v.v v20, v16 +; RV64ZVFHMIN-NEXT: vsetvli zero, a0, e16, m8, tu, ma +; RV64ZVFHMIN-NEXT: vmerge.vvm v8, v8, v16, v0 +; RV64ZVFHMIN-NEXT: ret + %elt.head = insertelement poison, bfloat %a, i32 0 + %va = shufflevector %elt.head, poison, zeroinitializer + %v = call @llvm.vp.merge.nxv32bf16( %m, %va, %vb, i32 %evl) + ret %v +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll index 53b8e4a78b75..ee7a94cf5e49 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vselect-fp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFH -; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFH -; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFHMIN -; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfhmin,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ZVFHMIN define @vfmerge_vv_nxv1f16( %va, %vb, %cond) { @@ -512,3 +512,219 @@ define void @vselect_legalize_regression( %a, %sel, ptr %out ret void } + +define @vfmerge_vv_nxv1bf16( %va, %vb, %cond) { +; CHECK-LABEL: vfmerge_vv_nxv1bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %vc = select %cond, %va, %vb + ret %vc +} + +define @vfmerge_fv_nxv1bf16( %va, bfloat %b, %cond) { +; CHECK-ZVFH-LABEL: vfmerge_fv_nxv1bf16: +; CHECK-ZVFH: # %bb.0: +; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; CHECK-ZVFH-NEXT: ret +; +; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv1bf16: +; CHECK-ZVFHMIN: # %bb.0: +; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, mu +; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; CHECK-ZVFHMIN-NEXT: ret + %head = insertelement poison, bfloat %b, i32 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = select %cond, %splat, %va + ret %vc +} + +define @vfmerge_vv_nxv2bf16( %va, %vb, %cond) { +; CHECK-LABEL: vfmerge_vv_nxv2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %vc = select %cond, %va, %vb + ret %vc +} + +define @vfmerge_fv_nxv2bf16( %va, bfloat %b, %cond) { +; CHECK-ZVFH-LABEL: vfmerge_fv_nxv2bf16: +; CHECK-ZVFH: # %bb.0: +; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; CHECK-ZVFH-NEXT: ret +; +; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv2bf16: +; CHECK-ZVFHMIN: # %bb.0: +; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-ZVFHMIN-NEXT: vfmv.v.f v9, fa5 +; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, mu +; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v9, v0.t +; CHECK-ZVFHMIN-NEXT: ret + %head = insertelement poison, bfloat %b, i32 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = select %cond, %splat, %va + ret %vc +} + +define @vfmerge_vv_nxv4bf16( %va, %vb, %cond) { +; CHECK-LABEL: vfmerge_vv_nxv4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %vc = select %cond, %va, %vb + ret %vc +} + +define @vfmerge_fv_nxv4bf16( %va, bfloat %b, %cond) { +; CHECK-ZVFH-LABEL: vfmerge_fv_nxv4bf16: +; CHECK-ZVFH: # %bb.0: +; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; CHECK-ZVFH-NEXT: ret +; +; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv4bf16: +; CHECK-ZVFHMIN: # %bb.0: +; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-ZVFHMIN-NEXT: vfmv.v.f v10, fa5 +; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, m1, ta, mu +; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v10, v0.t +; CHECK-ZVFHMIN-NEXT: ret + %head = insertelement poison, bfloat %b, i32 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = select %cond, %splat, %va + ret %vc +} + +define @vfmerge_vv_nxv8bf16( %va, %vb, %cond) { +; CHECK-LABEL: vfmerge_vv_nxv8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %vc = select %cond, %va, %vb + ret %vc +} + +define @vfmerge_fv_nxv8bf16( %va, bfloat %b, %cond) { +; CHECK-ZVFH-LABEL: vfmerge_fv_nxv8bf16: +; CHECK-ZVFH: # %bb.0: +; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; CHECK-ZVFH-NEXT: ret +; +; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv8bf16: +; CHECK-ZVFHMIN: # %bb.0: +; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-ZVFHMIN-NEXT: vfmv.v.f v12, fa5 +; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, m2, ta, mu +; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v12, v0.t +; CHECK-ZVFHMIN-NEXT: ret + %head = insertelement poison, bfloat %b, i32 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = select %cond, %splat, %va + ret %vc +} + +define @vfmerge_zv_nxv8bf16( %va, %cond) { +; CHECK-LABEL: vfmerge_zv_nxv8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vim v8, v8, 0, v0 +; CHECK-NEXT: ret + %vc = select %cond, splat (bfloat zeroinitializer), %va + ret %vc +} + +define @vmerge_truelhs_nxv8bf16_0( %va, %vb) { +; CHECK-LABEL: vmerge_truelhs_nxv8bf16_0: +; CHECK: # %bb.0: +; CHECK-NEXT: ret + %vc = select splat (i1 1), %va, %vb + ret %vc +} + +define @vmerge_falselhs_nxv8bf16_0( %va, %vb) { +; CHECK-LABEL: vmerge_falselhs_nxv8bf16_0: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv2r.v v8, v10 +; CHECK-NEXT: ret + %vc = select zeroinitializer, %va, %vb + ret %vc +} + +define @vfmerge_vv_nxv16bf16( %va, %vb, %cond) { +; CHECK-LABEL: vfmerge_vv_nxv16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0 +; CHECK-NEXT: ret + %vc = select %cond, %va, %vb + ret %vc +} + +define @vfmerge_fv_nxv16bf16( %va, bfloat %b, %cond) { +; CHECK-ZVFH-LABEL: vfmerge_fv_nxv16bf16: +; CHECK-ZVFH: # %bb.0: +; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; CHECK-ZVFH-NEXT: ret +; +; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv16bf16: +; CHECK-ZVFHMIN: # %bb.0: +; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m8, ta, ma +; CHECK-ZVFHMIN-NEXT: vfmv.v.f v16, fa5 +; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, mu +; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v8, v16, v0.t +; CHECK-ZVFHMIN-NEXT: ret + %head = insertelement poison, bfloat %b, i32 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = select %cond, %splat, %va + ret %vc +} + +define @vfmerge_vv_nxv32bf16( %va, %vb, %cond) { +; CHECK-LABEL: vfmerge_vv_nxv32bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m8, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v16, v8, v0 +; CHECK-NEXT: ret + %vc = select %cond, %va, %vb + ret %vc +} + +define @vfmerge_fv_nxv32bf16( %va, bfloat %b, %cond) { +; CHECK-ZVFH-LABEL: vfmerge_fv_nxv32bf16: +; CHECK-ZVFH: # %bb.0: +; CHECK-ZVFH-NEXT: vsetvli a0, zero, e16, m8, ta, ma +; CHECK-ZVFH-NEXT: vfmerge.vfm v8, v8, fa0, v0 +; CHECK-ZVFH-NEXT: ret +; +; CHECK-ZVFHMIN-LABEL: vfmerge_fv_nxv32bf16: +; CHECK-ZVFHMIN: # %bb.0: +; CHECK-ZVFHMIN-NEXT: fcvt.s.bf16 fa5, fa0 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e32, m8, ta, ma +; CHECK-ZVFHMIN-NEXT: vfmv.v.f v16, fa5 +; CHECK-ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma +; CHECK-ZVFHMIN-NEXT: vfncvtbf16.f.f.w v24, v16 +; CHECK-ZVFHMIN-NEXT: vmv.v.v v28, v24 +; CHECK-ZVFHMIN-NEXT: vsetvli a0, zero, e16, m8, ta, ma +; CHECK-ZVFHMIN-NEXT: vmerge.vvm v8, v8, v24, v0 +; CHECK-ZVFHMIN-NEXT: ret + %head = insertelement poison, bfloat %b, i32 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = select %cond, %splat, %va + ret %vc +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll index ee0617c93148..d1049e14fa29 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfh,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfh,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfh,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfhmin,+v -target-abi=ilp32d \ +; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=ilp32d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfhmin,+v -target-abi=lp64d \ +; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zfh,+zvfhmin,+v,+experimental-zfbfmin,+experimental-zvfbfmin -target-abi=lp64d \ ; RUN: -verify-machineinstrs < %s | FileCheck %s declare @llvm.vp.select.nxv1i1(, , , i32) @@ -922,3 +922,75 @@ define @select_unknown_T_T( %x, @llvm.vp.select.nxv2i1( %x, %y, %y, i32 %evl) ret %a } + +declare @llvm.vp.select.nxv1bf16(, , , i32) + +define @select_nxv1bf16( %a, %b, %c, i32 zeroext %evl) { +; CHECK-LABEL: select_nxv1bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = call @llvm.vp.select.nxv1bf16( %a, %b, %c, i32 %evl) + ret %v +} + +declare @llvm.vp.select.nxv2bf16(, , , i32) + +define @select_nxv2bf16( %a, %b, %c, i32 zeroext %evl) { +; CHECK-LABEL: select_nxv2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = call @llvm.vp.select.nxv2bf16( %a, %b, %c, i32 %evl) + ret %v +} + +declare @llvm.vp.select.nxv4bf16(, , , i32) + +define @select_nxv4bf16( %a, %b, %c, i32 zeroext %evl) { +; CHECK-LABEL: select_nxv4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0 +; CHECK-NEXT: ret + %v = call @llvm.vp.select.nxv4bf16( %a, %b, %c, i32 %evl) + ret %v +} + +declare @llvm.vp.select.nxv8bf16(, , , i32) + +define @select_nxv8bf16( %a, %b, %c, i32 zeroext %evl) { +; CHECK-LABEL: select_nxv8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m2, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0 +; CHECK-NEXT: ret + %v = call @llvm.vp.select.nxv8bf16( %a, %b, %c, i32 %evl) + ret %v +} + +declare @llvm.vp.select.nxv16bf16(, , , i32) + +define @select_nxv16bf16( %a, %b, %c, i32 zeroext %evl) { +; CHECK-LABEL: select_nxv16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m4, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0 +; CHECK-NEXT: ret + %v = call @llvm.vp.select.nxv16bf16( %a, %b, %c, i32 %evl) + ret %v +} + +declare @llvm.vp.select.nxv32bf16(, , , i32) + +define @select_nxv32bf16( %a, %b, %c, i32 zeroext %evl) { +; CHECK-LABEL: select_nxv32bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, m8, ta, ma +; CHECK-NEXT: vmerge.vvm v8, v16, v8, v0 +; CHECK-NEXT: ret + %v = call @llvm.vp.select.nxv32bf16( %a, %b, %c, i32 %evl) + ret %v +} -- GitLab From 8ef5c98e9f9276f09c590e22ab88a20ae86f3859 Mon Sep 17 00:00:00 2001 From: uint256_t Date: Fri, 7 Jun 2024 17:05:50 +0900 Subject: [PATCH 207/462] [docs] Fix benchmarking tips (#94724) This PR fixes an incorrect line for setting scaling_governer in benchmarking tips. --- llvm/docs/Benchmarking.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/docs/Benchmarking.rst b/llvm/docs/Benchmarking.rst index 7deb22144606..cd7b835e47c5 100644 --- a/llvm/docs/Benchmarking.rst +++ b/llvm/docs/Benchmarking.rst @@ -49,7 +49,7 @@ Linux for i in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor do - echo performance > /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + echo performance > $i done * Use https://github.com/lpechacek/cpuset to reserve cpus for just the -- GitLab From 36bc7410cb68e32d4416b951a118d5a799d3a226 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Fri, 7 Jun 2024 10:15:32 +0200 Subject: [PATCH 208/462] [test] Don't generate extra file for regalloc-amdgpu.mir test. --- llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir b/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir index 0603ffc4c5ea..f75db964e2d6 100644 --- a/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir +++ b/llvm/test/tools/llc/new-pm/regalloc-amdgpu.mir @@ -1,5 +1,5 @@ # REQUIRES: amdgpu-registered-target -# RUN: llc -mtriple=amdgcn --passes='regallocfast' --print-pipeline-passes %s | FileCheck %s --check-prefix=PASS +# RUN: llc -mtriple=amdgcn --passes='regallocfast' --print-pipeline-passes --filetype=null %s | FileCheck %s --check-prefix=PASS # RUN: not llc -mtriple=amdgcn --passes='regallocfast' --print-pipeline-passes --filetype=null %s 2>&1 | FileCheck %s --check-prefix=BAD-FILTER # PASS: regallocfast -- GitLab From 1c0063b58a4fc23c94c7f5bf5a937bbdf9703cc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Fri, 7 Jun 2024 10:31:16 +0200 Subject: [PATCH 209/462] [clang][Interp] Remove StoragKind limitation in Pointer assign operators It's not strictly needed and did cause some test failures. --- clang/lib/AST/Interp/Pointer.cpp | 44 ++++++++++++++++---------------- clang/test/AST/Interp/cxx20.cpp | 15 +++++++++++ 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp index a60b4d28b438..e3d21f93f114 100644 --- a/clang/lib/AST/Interp/Pointer.cpp +++ b/clang/lib/AST/Interp/Pointer.cpp @@ -64,26 +64,26 @@ Pointer::~Pointer() { } void Pointer::operator=(const Pointer &P) { - if (!this->isIntegralPointer() || !P.isBlockPointer()) - assert(P.StorageKind == StorageKind || (this->isZero() && P.isZero())); - + // If the current storage type is Block, we need to remove + // this pointer from the block. bool WasBlockPointer = isBlockPointer(); - StorageKind = P.StorageKind; if (StorageKind == Storage::Block) { Block *Old = PointeeStorage.BS.Pointee; - if (WasBlockPointer && PointeeStorage.BS.Pointee) + if (WasBlockPointer && Old) { PointeeStorage.BS.Pointee->removePointer(this); + Old->cleanup(); + } + } - Offset = P.Offset; + StorageKind = P.StorageKind; + Offset = P.Offset; + + if (P.isBlockPointer()) { PointeeStorage.BS = P.PointeeStorage.BS; if (PointeeStorage.BS.Pointee) PointeeStorage.BS.Pointee->addPointer(this); - - if (WasBlockPointer && Old) - Old->cleanup(); - - } else if (StorageKind == Storage::Int) { + } else if (P.isIntegralPointer()) { PointeeStorage.Int = P.PointeeStorage.Int; } else { assert(false && "Unhandled storage kind"); @@ -91,26 +91,26 @@ void Pointer::operator=(const Pointer &P) { } void Pointer::operator=(Pointer &&P) { - if (!this->isIntegralPointer() || !P.isBlockPointer()) - assert(P.StorageKind == StorageKind || (this->isZero() && P.isZero())); - + // If the current storage type is Block, we need to remove + // this pointer from the block. bool WasBlockPointer = isBlockPointer(); - StorageKind = P.StorageKind; if (StorageKind == Storage::Block) { Block *Old = PointeeStorage.BS.Pointee; - if (WasBlockPointer && PointeeStorage.BS.Pointee) + if (WasBlockPointer && Old) { PointeeStorage.BS.Pointee->removePointer(this); + Old->cleanup(); + } + } - Offset = P.Offset; + StorageKind = P.StorageKind; + Offset = P.Offset; + + if (P.isBlockPointer()) { PointeeStorage.BS = P.PointeeStorage.BS; if (PointeeStorage.BS.Pointee) PointeeStorage.BS.Pointee->addPointer(this); - - if (WasBlockPointer && Old) - Old->cleanup(); - - } else if (StorageKind == Storage::Int) { + } else if (P.isIntegralPointer()) { PointeeStorage.Int = P.PointeeStorage.Int; } else { assert(false && "Unhandled storage kind"); diff --git a/clang/test/AST/Interp/cxx20.cpp b/clang/test/AST/Interp/cxx20.cpp index 49ee040be392..c750be866ca7 100644 --- a/clang/test/AST/Interp/cxx20.cpp +++ b/clang/test/AST/Interp/cxx20.cpp @@ -782,3 +782,18 @@ namespace APValues { constexpr const A &v = get; constexpr const A &w = get; } + +namespace self_referencing { + struct S { + S* ptr = nullptr; + constexpr S(int i) : ptr(this) { + if (this == ptr && i) + ptr = nullptr; + } + constexpr ~S() {} + }; + + void test() { + S s(1); + } +} -- GitLab From ac404632f991fc6e7dc75ef553a99676ba8002ce Mon Sep 17 00:00:00 2001 From: WANG Rui Date: Fri, 7 Jun 2024 16:33:17 +0800 Subject: [PATCH 210/462] [NFC][LoongArch] Update test for #94590 --- .../LoongArch/machinelicm-address-pseudos.ll | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll b/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll index 30a19adcfdce..cd308a149591 100644 --- a/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll +++ b/llvm/test/CodeGen/LoongArch/machinelicm-address-pseudos.ll @@ -19,7 +19,7 @@ define void @test_la_pcrel(i32 signext %n) { ; LA32-NEXT: .p2align 4, , 16 ; LA32-NEXT: .LBB0_1: # %loop ; LA32-NEXT: # =>This Inner Loop Header: Depth=1 -; LA32-NEXT: ld.w $a3, $a2, 0 +; LA32-NEXT: ld.w $zero, $a2, 0 ; LA32-NEXT: addi.w $a1, $a1, 1 ; LA32-NEXT: blt $a1, $a0, .LBB0_1 ; LA32-NEXT: # %bb.2: # %ret @@ -33,7 +33,7 @@ define void @test_la_pcrel(i32 signext %n) { ; LA64-NEXT: .p2align 4, , 16 ; LA64-NEXT: .LBB0_1: # %loop ; LA64-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64-NEXT: ld.w $a3, $a2, 0 +; LA64-NEXT: ld.w $zero, $a2, 0 ; LA64-NEXT: addi.w $a1, $a1, 1 ; LA64-NEXT: blt $a1, $a0, .LBB0_1 ; LA64-NEXT: # %bb.2: # %ret @@ -50,7 +50,7 @@ define void @test_la_pcrel(i32 signext %n) { ; LA64LARGE-NEXT: .p2align 4, , 16 ; LA64LARGE-NEXT: .LBB0_1: # %loop ; LA64LARGE-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64LARGE-NEXT: ld.w $a3, $a2, 0 +; LA64LARGE-NEXT: ld.w $zero, $a2, 0 ; LA64LARGE-NEXT: addi.w $a1, $a1, 1 ; LA64LARGE-NEXT: blt $a1, $a0, .LBB0_1 ; LA64LARGE-NEXT: # %bb.2: # %ret @@ -80,7 +80,7 @@ define void @test_la_got(i32 signext %n) { ; LA32-NEXT: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: pcalau12i $a2, %got_pc_hi20(g) ; LA32-NEXT: ld.w $a2, $a2, %got_pc_lo12(g) -; LA32-NEXT: ld.w $a2, $a2, 0 +; LA32-NEXT: ld.w $zero, $a2, 0 ; LA32-NEXT: addi.w $a1, $a1, 1 ; LA32-NEXT: blt $a1, $a0, .LBB1_1 ; LA32-NEXT: # %bb.2: # %ret @@ -94,7 +94,7 @@ define void @test_la_got(i32 signext %n) { ; LA64-NEXT: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: pcalau12i $a2, %got_pc_hi20(g) ; LA64-NEXT: ld.d $a2, $a2, %got_pc_lo12(g) -; LA64-NEXT: ld.w $a2, $a2, 0 +; LA64-NEXT: ld.w $zero, $a2, 0 ; LA64-NEXT: addi.w $a1, $a1, 1 ; LA64-NEXT: blt $a1, $a0, .LBB1_1 ; LA64-NEXT: # %bb.2: # %ret @@ -111,7 +111,7 @@ define void @test_la_got(i32 signext %n) { ; LA64LARGE-NEXT: lu32i.d $t8, %got64_pc_lo20(g) ; LA64LARGE-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(g) ; LA64LARGE-NEXT: ldx.d $a2, $t8, $a2 -; LA64LARGE-NEXT: ld.w $a2, $a2, 0 +; LA64LARGE-NEXT: ld.w $zero, $a2, 0 ; LA64LARGE-NEXT: addi.w $a1, $a1, 1 ; LA64LARGE-NEXT: blt $a1, $a0, .LBB1_1 ; LA64LARGE-NEXT: # %bb.2: # %ret @@ -142,7 +142,7 @@ define void @test_la_tls_ie(i32 signext %n) { ; LA32-NEXT: pcalau12i $a2, %ie_pc_hi20(ie) ; LA32-NEXT: ld.w $a2, $a2, %ie_pc_lo12(ie) ; LA32-NEXT: add.w $a2, $a2, $tp -; LA32-NEXT: ld.w $a2, $a2, 0 +; LA32-NEXT: ld.w $zero, $a2, 0 ; LA32-NEXT: addi.w $a1, $a1, 1 ; LA32-NEXT: blt $a1, $a0, .LBB2_1 ; LA32-NEXT: # %bb.2: # %ret @@ -156,7 +156,7 @@ define void @test_la_tls_ie(i32 signext %n) { ; LA64-NEXT: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: pcalau12i $a2, %ie_pc_hi20(ie) ; LA64-NEXT: ld.d $a2, $a2, %ie_pc_lo12(ie) -; LA64-NEXT: ldx.w $a2, $a2, $tp +; LA64-NEXT: ldx.w $zero, $a2, $tp ; LA64-NEXT: addi.w $a1, $a1, 1 ; LA64-NEXT: blt $a1, $a0, .LBB2_1 ; LA64-NEXT: # %bb.2: # %ret @@ -173,7 +173,7 @@ define void @test_la_tls_ie(i32 signext %n) { ; LA64LARGE-NEXT: lu32i.d $t8, %ie64_pc_lo20(ie) ; LA64LARGE-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(ie) ; LA64LARGE-NEXT: ldx.d $a2, $t8, $a2 -; LA64LARGE-NEXT: ldx.w $a2, $a2, $tp +; LA64LARGE-NEXT: ldx.w $zero, $a2, $tp ; LA64LARGE-NEXT: addi.w $a1, $a1, 1 ; LA64LARGE-NEXT: blt $a1, $a0, .LBB2_1 ; LA64LARGE-NEXT: # %bb.2: # %ret @@ -213,7 +213,7 @@ define void @test_la_tls_ld(i32 signext %n) { ; LA32-NEXT: pcalau12i $a0, %ld_pc_hi20(ld) ; LA32-NEXT: addi.w $a0, $a0, %got_pc_lo12(ld) ; LA32-NEXT: bl %plt(__tls_get_addr) -; LA32-NEXT: ld.w $a0, $a0, 0 +; LA32-NEXT: ld.w $zero, $a0, 0 ; LA32-NEXT: addi.w $s0, $s0, 1 ; LA32-NEXT: blt $s0, $fp, .LBB3_1 ; LA32-NEXT: # %bb.2: # %ret @@ -241,7 +241,7 @@ define void @test_la_tls_ld(i32 signext %n) { ; LA64-NEXT: pcalau12i $a0, %ld_pc_hi20(ld) ; LA64-NEXT: addi.d $a0, $a0, %got_pc_lo12(ld) ; LA64-NEXT: bl %plt(__tls_get_addr) -; LA64-NEXT: ld.w $a0, $a0, 0 +; LA64-NEXT: ld.w $zero, $a0, 0 ; LA64-NEXT: addi.w $s0, $s0, 1 ; LA64-NEXT: blt $s0, $fp, .LBB3_1 ; LA64-NEXT: # %bb.2: # %ret @@ -277,7 +277,7 @@ define void @test_la_tls_ld(i32 signext %n) { ; LA64LARGE-NEXT: lu52i.d $t8, $t8, %pc64_hi12(__tls_get_addr) ; LA64LARGE-NEXT: add.d $ra, $t8, $ra ; LA64LARGE-NEXT: jirl $ra, $ra, 0 -; LA64LARGE-NEXT: ld.w $a0, $a0, 0 +; LA64LARGE-NEXT: ld.w $zero, $a0, 0 ; LA64LARGE-NEXT: addi.w $s0, $s0, 1 ; LA64LARGE-NEXT: blt $s0, $fp, .LBB3_1 ; LA64LARGE-NEXT: # %bb.2: # %ret @@ -312,7 +312,7 @@ define void @test_la_tls_le(i32 signext %n) { ; LA32-NEXT: .p2align 4, , 16 ; LA32-NEXT: .LBB4_1: # %loop ; LA32-NEXT: # =>This Inner Loop Header: Depth=1 -; LA32-NEXT: ld.w $a3, $a2, 0 +; LA32-NEXT: ld.w $zero, $a2, 0 ; LA32-NEXT: addi.w $a1, $a1, 1 ; LA32-NEXT: blt $a1, $a0, .LBB4_1 ; LA32-NEXT: # %bb.2: # %ret @@ -326,7 +326,7 @@ define void @test_la_tls_le(i32 signext %n) { ; LA64-NEXT: .p2align 4, , 16 ; LA64-NEXT: .LBB4_1: # %loop ; LA64-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64-NEXT: ldx.w $a3, $a2, $tp +; LA64-NEXT: ldx.w $zero, $a2, $tp ; LA64-NEXT: addi.w $a1, $a1, 1 ; LA64-NEXT: blt $a1, $a0, .LBB4_1 ; LA64-NEXT: # %bb.2: # %ret @@ -342,7 +342,7 @@ define void @test_la_tls_le(i32 signext %n) { ; LA64LARGE-NEXT: .p2align 4, , 16 ; LA64LARGE-NEXT: .LBB4_1: # %loop ; LA64LARGE-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64LARGE-NEXT: ldx.w $a3, $a2, $tp +; LA64LARGE-NEXT: ldx.w $zero, $a2, $tp ; LA64LARGE-NEXT: addi.w $a1, $a1, 1 ; LA64LARGE-NEXT: blt $a1, $a0, .LBB4_1 ; LA64LARGE-NEXT: # %bb.2: # %ret @@ -378,7 +378,7 @@ define void @test_la_tls_gd(i32 signext %n) nounwind { ; LA32-NEXT: pcalau12i $a0, %gd_pc_hi20(gd) ; LA32-NEXT: addi.w $a0, $a0, %got_pc_lo12(gd) ; LA32-NEXT: bl %plt(__tls_get_addr) -; LA32-NEXT: ld.w $a0, $a0, 0 +; LA32-NEXT: ld.w $zero, $a0, 0 ; LA32-NEXT: addi.w $s0, $s0, 1 ; LA32-NEXT: blt $s0, $fp, .LBB5_1 ; LA32-NEXT: # %bb.2: # %ret @@ -402,7 +402,7 @@ define void @test_la_tls_gd(i32 signext %n) nounwind { ; LA64-NEXT: pcalau12i $a0, %gd_pc_hi20(gd) ; LA64-NEXT: addi.d $a0, $a0, %got_pc_lo12(gd) ; LA64-NEXT: bl %plt(__tls_get_addr) -; LA64-NEXT: ld.w $a0, $a0, 0 +; LA64-NEXT: ld.w $zero, $a0, 0 ; LA64-NEXT: addi.w $s0, $s0, 1 ; LA64-NEXT: blt $s0, $fp, .LBB5_1 ; LA64-NEXT: # %bb.2: # %ret @@ -434,7 +434,7 @@ define void @test_la_tls_gd(i32 signext %n) nounwind { ; LA64LARGE-NEXT: lu52i.d $t8, $t8, %pc64_hi12(__tls_get_addr) ; LA64LARGE-NEXT: add.d $ra, $t8, $ra ; LA64LARGE-NEXT: jirl $ra, $ra, 0 -; LA64LARGE-NEXT: ld.w $a0, $a0, 0 +; LA64LARGE-NEXT: ld.w $zero, $a0, 0 ; LA64LARGE-NEXT: addi.w $s0, $s0, 1 ; LA64LARGE-NEXT: blt $s0, $fp, .LBB5_1 ; LA64LARGE-NEXT: # %bb.2: # %ret -- GitLab From 4f320e6aa2d4fdcb3d749977c840cdf1783173f3 Mon Sep 17 00:00:00 2001 From: Abid Qadeer Date: Fri, 7 Jun 2024 09:59:47 +0100 Subject: [PATCH 211/462] [MLIR] Translate DIStringType. (#94480) This PR handle translation of DIStringType. Mostly mechanical changes to translate DIStringType to/from DIStringTypeAttr. The 'stringLength' field is 'DIVariable' in DIStringType. As there was no `DIVariableAttr` previously, it has been added to ease the translation. --------- Co-authored-by: Tobias Gysi --- mlir/include/mlir-c/Dialect/LLVM.h | 6 ++++ .../mlir/Dialect/LLVMIR/LLVMAttrDefs.td | 19 +++++++++++ mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h | 9 ++++++ mlir/lib/CAPI/Dialect/LLVM.cpp | 12 +++++++ mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp | 15 +++++++-- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 12 +++---- mlir/lib/Target/LLVMIR/DebugImporter.cpp | 15 +++++++++ mlir/lib/Target/LLVMIR/DebugImporter.h | 2 ++ mlir/lib/Target/LLVMIR/DebugTranslation.cpp | 17 ++++++++-- mlir/lib/Target/LLVMIR/DebugTranslation.h | 2 ++ mlir/test/CAPI/llvm.c | 11 +++++-- mlir/test/Target/LLVMIR/Import/debug-info.ll | 32 +++++++++++++++++++ mlir/test/Target/LLVMIR/llvmir-debug.mlir | 27 ++++++++++++++++ 13 files changed, 166 insertions(+), 13 deletions(-) diff --git a/mlir/include/mlir-c/Dialect/LLVM.h b/mlir/include/mlir-c/Dialect/LLVM.h index e754318d6685..902b45444d6c 100644 --- a/mlir/include/mlir-c/Dialect/LLVM.h +++ b/mlir/include/mlir-c/Dialect/LLVM.h @@ -251,6 +251,12 @@ MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDIDerivedTypeAttrGet( MlirAttribute baseType, uint64_t sizeInBits, uint32_t alignInBits, uint64_t offsetInBits, int64_t dwarfAddressSpace, MlirAttribute extraData); +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDIStringTypeAttrGet( + MlirContext ctx, unsigned int tag, MlirAttribute name, uint64_t sizeInBits, + uint32_t alignInBits, MlirAttribute stringLength, + MlirAttribute stringLengthExp, MlirAttribute stringLocationExp, + MlirLLVMTypeEncoding encoding); + /// Constant to represent std::nullopt for dwarfAddressSpace to omit the field. #define MLIR_CAPI_DWARF_ADDRESS_SPACE_NULL -1 diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td index ec58cf368b59..b05366d2a635 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -676,6 +676,25 @@ def LLVM_DILabelAttr : LLVM_Attr<"DILabel", "di_label", let assemblyFormat = "`<` struct(params) `>`"; } +//===----------------------------------------------------------------------===// +// DIStringTypeAttr +//===----------------------------------------------------------------------===// + +def LLVM_DIStringTypeAttr : LLVM_Attr<"DIStringType", "di_string_type", + /*traits=*/[], "DITypeAttr"> { + let parameters = (ins + LLVM_DITagParameter:$tag, + "StringAttr":$name, + OptionalParameter<"uint64_t">:$sizeInBits, + OptionalParameter<"uint32_t">:$alignInBits, + OptionalParameter<"DIVariableAttr">:$stringLength, + OptionalParameter<"DIExpressionAttr">:$stringLengthExp, + OptionalParameter<"DIExpressionAttr">:$stringLocationExp, + LLVM_DIEncodingParameter:$encoding + ); + let assemblyFormat = "`<` struct(params) `>`"; +} + //===----------------------------------------------------------------------===// // MemoryEffectsAttr //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h index 091a817caf3d..3a93be21da37 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h @@ -60,6 +60,15 @@ public: static bool classof(Attribute attr); }; +/// This class represents a LLVM attribute that describes a debug info variable. +class DIVariableAttr : public DINodeAttr { +public: + using DINodeAttr::DINodeAttr; + + /// Support LLVM type casting. + static bool classof(Attribute attr); +}; + /// Base class for LLVM attributes participating in the TBAA graph. class TBAANodeAttr : public Attribute { public: diff --git a/mlir/lib/CAPI/Dialect/LLVM.cpp b/mlir/lib/CAPI/Dialect/LLVM.cpp index f6fb2cbedfac..754c94511524 100644 --- a/mlir/lib/CAPI/Dialect/LLVM.cpp +++ b/mlir/lib/CAPI/Dialect/LLVM.cpp @@ -195,6 +195,18 @@ MlirAttribute mlirLLVMDIDerivedTypeAttrGet( addressSpace, cast(unwrap(extraData)))); } +MlirAttribute mlirLLVMDIStringTypeAttrGet( + MlirContext ctx, unsigned int tag, MlirAttribute name, uint64_t sizeInBits, + uint32_t alignInBits, MlirAttribute stringLength, + MlirAttribute stringLengthExp, MlirAttribute stringLocationExp, + MlirLLVMTypeEncoding encoding) { + return wrap(DIStringTypeAttr::get( + unwrap(ctx), tag, cast(unwrap(name)), sizeInBits, alignInBits, + cast(unwrap(stringLength)), + cast(unwrap(stringLengthExp)), + cast(unwrap(stringLocationExp)), encoding)); +} + MlirAttribute mlirLLVMDIDerivedTypeAttrGetBaseType(MlirAttribute diDerivedType) { return wrap(cast(unwrap(diDerivedType)).getBaseType()); diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp index 9bc71e1ebc48..963a4be25079 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp @@ -60,8 +60,8 @@ bool DINodeAttr::classof(Attribute attr) { DIDerivedTypeAttr, DIFileAttr, DIGlobalVariableAttr, DILabelAttr, DILexicalBlockAttr, DILexicalBlockFileAttr, DILocalVariableAttr, DIModuleAttr, DINamespaceAttr, - DINullTypeAttr, DISubprogramAttr, DISubrangeAttr, - DISubroutineTypeAttr>(attr); + DINullTypeAttr, DIStringTypeAttr, DISubprogramAttr, + DISubrangeAttr, DISubroutineTypeAttr>(attr); } //===----------------------------------------------------------------------===// @@ -82,13 +82,22 @@ bool DILocalScopeAttr::classof(Attribute attr) { DISubprogramAttr>(attr); } +//===----------------------------------------------------------------------===// +// DIVariableAttr +//===----------------------------------------------------------------------===// + +bool DIVariableAttr::classof(Attribute attr) { + return llvm::isa(attr); +} + //===----------------------------------------------------------------------===// // DITypeAttr //===----------------------------------------------------------------------===// bool DITypeAttr::classof(Attribute attr) { return llvm::isa(attr); + DIDerivedTypeAttr, DIStringTypeAttr, DISubroutineTypeAttr>( + attr); } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 60b911948d4a..76cac0b05b47 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -3031,12 +3031,12 @@ struct LLVMOpAsmDialectInterface : public OpAsmDialectInterface { DIDerivedTypeAttr, DIFileAttr, DIGlobalVariableAttr, DIGlobalVariableExpressionAttr, DILabelAttr, DILexicalBlockAttr, DILexicalBlockFileAttr, DILocalVariableAttr, DIModuleAttr, - DINamespaceAttr, DINullTypeAttr, DISubprogramAttr, - DISubroutineTypeAttr, LoopAnnotationAttr, LoopVectorizeAttr, - LoopInterleaveAttr, LoopUnrollAttr, LoopUnrollAndJamAttr, - LoopLICMAttr, LoopDistributeAttr, LoopPipelineAttr, - LoopPeeledAttr, LoopUnswitchAttr, TBAARootAttr, TBAATagAttr, - TBAATypeDescriptorAttr>([&](auto attr) { + DINamespaceAttr, DINullTypeAttr, DIStringTypeAttr, + DISubprogramAttr, DISubroutineTypeAttr, LoopAnnotationAttr, + LoopVectorizeAttr, LoopInterleaveAttr, LoopUnrollAttr, + LoopUnrollAndJamAttr, LoopLICMAttr, LoopDistributeAttr, + LoopPipelineAttr, LoopPeeledAttr, LoopUnswitchAttr, TBAARootAttr, + TBAATagAttr, TBAATypeDescriptorAttr>([&](auto attr) { os << decltype(attr)::getMnemonic(); return AliasResult::OverridableAlias; }) diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.cpp b/mlir/lib/Target/LLVMIR/DebugImporter.cpp index a373c1fb5cda..15737aa681c5 100644 --- a/mlir/lib/Target/LLVMIR/DebugImporter.cpp +++ b/mlir/lib/Target/LLVMIR/DebugImporter.cpp @@ -113,6 +113,15 @@ DIDerivedTypeAttr DebugImporter::translateImpl(llvm::DIDerivedType *node) { node->getOffsetInBits(), node->getDWARFAddressSpace(), extraData); } +DIStringTypeAttr DebugImporter::translateImpl(llvm::DIStringType *node) { + return DIStringTypeAttr::get( + context, node->getTag(), getStringAttrOrNull(node->getRawName()), + node->getSizeInBits(), node->getAlignInBits(), + translate(node->getStringLength()), + translateExpression(node->getStringLengthExp()), + translateExpression(node->getStringLocationExp()), node->getEncoding()); +} + DIFileAttr DebugImporter::translateImpl(llvm::DIFile *node) { return DIFileAttr::get(context, node->getFilename(), node->getDirectory()); } @@ -174,6 +183,10 @@ DILocalVariableAttr DebugImporter::translateImpl(llvm::DILocalVariable *node) { node->getAlignInBits(), translate(node->getType())); } +DIVariableAttr DebugImporter::translateImpl(llvm::DIVariable *node) { + return cast(translate(static_cast(node))); +} + DIScopeAttr DebugImporter::translateImpl(llvm::DIScope *node) { return cast(translate(static_cast(node))); } @@ -295,6 +308,8 @@ DINodeAttr DebugImporter::translate(llvm::DINode *node) { return translateImpl(casted); if (auto *casted = dyn_cast(node)) return translateImpl(casted); + if (auto *casted = dyn_cast(node)) + return translateImpl(casted); if (auto *casted = dyn_cast(node)) return translateImpl(casted); if (auto *casted = dyn_cast(node)) diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.h b/mlir/lib/Target/LLVMIR/DebugImporter.h index 5f402fb0b657..fac86dbe2cdd 100644 --- a/mlir/lib/Target/LLVMIR/DebugImporter.h +++ b/mlir/lib/Target/LLVMIR/DebugImporter.h @@ -63,12 +63,14 @@ private: DICompileUnitAttr translateImpl(llvm::DICompileUnit *node); DICompositeTypeAttr translateImpl(llvm::DICompositeType *node); DIDerivedTypeAttr translateImpl(llvm::DIDerivedType *node); + DIStringTypeAttr translateImpl(llvm::DIStringType *node); DIFileAttr translateImpl(llvm::DIFile *node); DILabelAttr translateImpl(llvm::DILabel *node); DILexicalBlockAttr translateImpl(llvm::DILexicalBlock *node); DILexicalBlockFileAttr translateImpl(llvm::DILexicalBlockFile *node); DIGlobalVariableAttr translateImpl(llvm::DIGlobalVariable *node); DILocalVariableAttr translateImpl(llvm::DILocalVariable *node); + DIVariableAttr translateImpl(llvm::DIVariable *node); DIModuleAttr translateImpl(llvm::DIModule *node); DINamespaceAttr translateImpl(llvm::DINamespace *node); DIScopeAttr translateImpl(llvm::DIScope *node); diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp index aef7a06f96c3..35800e993d89 100644 --- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp @@ -182,6 +182,15 @@ llvm::DIDerivedType *DebugTranslation::translateImpl(DIDerivedTypeAttr attr) { /*Flags=*/llvm::DINode::FlagZero, translate(attr.getExtraData())); } +llvm::DIStringType *DebugTranslation::translateImpl(DIStringTypeAttr attr) { + return llvm::DIStringType::get( + llvmCtx, attr.getTag(), getMDStringOrNull(attr.getName()), + translate(attr.getStringLength()), + getExpressionAttrOrNull(attr.getStringLengthExp()), + getExpressionAttrOrNull(attr.getStringLocationExp()), + attr.getSizeInBits(), attr.getAlignInBits(), attr.getEncoding()); +} + llvm::DIFile *DebugTranslation::translateImpl(DIFileAttr attr) { return llvm::DIFile::get(llvmCtx, getMDStringOrNull(attr.getName()), getMDStringOrNull(attr.getDirectory())); @@ -210,6 +219,10 @@ llvm::DILocalScope *DebugTranslation::translateImpl(DILocalScopeAttr attr) { return cast(translate(DINodeAttr(attr))); } +llvm::DIVariable *DebugTranslation::translateImpl(DIVariableAttr attr) { + return cast(translate(DINodeAttr(attr))); +} + llvm::DILocalVariable * DebugTranslation::translateImpl(DILocalVariableAttr attr) { return llvm::DILocalVariable::get( @@ -377,8 +390,8 @@ llvm::DINode *DebugTranslation::translate(DINodeAttr attr) { DIDerivedTypeAttr, DIFileAttr, DIGlobalVariableAttr, DILabelAttr, DILexicalBlockAttr, DILexicalBlockFileAttr, DILocalVariableAttr, DIModuleAttr, DINamespaceAttr, - DINullTypeAttr, DISubprogramAttr, DISubrangeAttr, - DISubroutineTypeAttr>( + DINullTypeAttr, DIStringTypeAttr, DISubprogramAttr, + DISubrangeAttr, DISubroutineTypeAttr>( [&](auto attr) { return translateImpl(attr); }); if (node && !node->isTemporary()) diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.h b/mlir/lib/Target/LLVMIR/DebugTranslation.h index 04b7ea41add9..16a853736226 100644 --- a/mlir/lib/Target/LLVMIR/DebugTranslation.h +++ b/mlir/lib/Target/LLVMIR/DebugTranslation.h @@ -73,6 +73,7 @@ private: llvm::DICompileUnit *translateImpl(DICompileUnitAttr attr); llvm::DICompositeType *translateImpl(DICompositeTypeAttr attr); llvm::DIDerivedType *translateImpl(DIDerivedTypeAttr attr); + llvm::DIStringType *translateImpl(DIStringTypeAttr attr); llvm::DIFile *translateImpl(DIFileAttr attr); llvm::DILabel *translateImpl(DILabelAttr attr); llvm::DILexicalBlock *translateImpl(DILexicalBlockAttr attr); @@ -80,6 +81,7 @@ private: llvm::DILocalScope *translateImpl(DILocalScopeAttr attr); llvm::DILocalVariable *translateImpl(DILocalVariableAttr attr); llvm::DIGlobalVariable *translateImpl(DIGlobalVariableAttr attr); + llvm::DIVariable *translateImpl(DIVariableAttr attr); llvm::DIModule *translateImpl(DIModuleAttr attr); llvm::DINamespace *translateImpl(DINamespaceAttr attr); llvm::DIScope *translateImpl(DIScopeAttr attr); diff --git a/mlir/test/CAPI/llvm.c b/mlir/test/CAPI/llvm.c index f1d02b43d523..082ddc53e628 100644 --- a/mlir/test/CAPI/llvm.c +++ b/mlir/test/CAPI/llvm.c @@ -293,8 +293,9 @@ static void testDebugInfoAttributes(MlirContext ctx) { mlirLLVMDILexicalBlockFileAttrGet(ctx, compile_unit, file, 3)); // CHECK: #llvm.di_local_variable<{{.*}}> - mlirAttributeDump(mlirLLVMDILocalVariableAttrGet(ctx, compile_unit, foo, file, - 1, 0, 8, di_type)); + MlirAttribute local_var = mlirLLVMDILocalVariableAttrGet( + ctx, compile_unit, foo, file, 1, 0, 8, di_type); + mlirAttributeDump(local_var); // CHECK: #llvm.di_derived_type<{{.*}}> // CHECK-NOT: dwarfAddressSpace mlirAttributeDump(mlirLLVMDIDerivedTypeAttrGet( @@ -337,6 +338,12 @@ static void testDebugInfoAttributes(MlirContext ctx) { // CHECK: #llvm.di_expression<[(1)]> mlirAttributeDump(expression); + MlirAttribute string_type = + mlirLLVMDIStringTypeAttrGet(ctx, 0x0, foo, 16, 0, local_var, expression, + expression, MlirLLVMTypeEncodingSigned); + // CHECK: #llvm.di_string_type<{{.*}}> + mlirAttributeDump(string_type); + // CHECK: #llvm.di_composite_type<{{.*}}> mlirAttributeDump(mlirLLVMDICompositeTypeAttrGet( ctx, 0, id, foo, file, 1, compile_unit, di_type, 0, 64, 8, 1, &di_type, diff --git a/mlir/test/Target/LLVMIR/Import/debug-info.ll b/mlir/test/Target/LLVMIR/Import/debug-info.ll index 0c6a7368b7b8..2173cc33e68a 100644 --- a/mlir/test/Target/LLVMIR/Import/debug-info.ll +++ b/mlir/test/Target/LLVMIR/Import/debug-info.ll @@ -761,3 +761,35 @@ define void @class_field(ptr %arg1) !dbg !18 { !11 = !{!6, !7, !8} ; C -> A, B, C !18 = distinct !DISubprogram(name: "SP", scope: !3, file: !2, spFlags: DISPFlagDefinition, unit: !1) + +; // ----- + +; Verify the string type is handled correctly + +define void @string_type(ptr %arg1) { + call void @llvm.dbg.value(metadata ptr %arg1, metadata !4, metadata !DIExpression()), !dbg !10 + call void @llvm.dbg.value(metadata ptr %arg1, metadata !9, metadata !DIExpression()), !dbg !10 + ret void +} + +!llvm.dbg.cu = !{!1} +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DICompileUnit(language: DW_LANG_C, file: !2) +!2 = !DIFile(filename: "debug-info.ll", directory: "/") +!3 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!4 = !DILocalVariable(scope: !5, name: "string_size", file: !2, type: !3); +!5 = distinct !DISubprogram(name: "test", scope: !2, file: !2, spFlags: DISPFlagDefinition, unit: !1) +!6 = !DIStringType(name: "character(*)", stringLength: !4, size: 32, align: 8, stringLengthExpression: !8, stringLocationExpression: !7) +!7 = !DIExpression(DW_OP_push_object_address, DW_OP_deref) +!8 = !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 8) +!9 = !DILocalVariable(scope: !5, name: "str", file: !2, type: !6); +!10 = !DILocation(line: 1, column: 2, scope: !5) + +; CHECK: #[[VAR:.+]] = #llvm.di_local_variable<{{.*}}name = "string_size"{{.*}}> +; CHECK: #llvm.di_string_type +; CHECK-SAME: stringLocationExp = <[DW_OP_push_object_address, DW_OP_deref]>> diff --git a/mlir/test/Target/LLVMIR/llvmir-debug.mlir b/mlir/test/Target/LLVMIR/llvmir-debug.mlir index 724f45f0d2a8..c6e7ca6f3f21 100644 --- a/mlir/test/Target/LLVMIR/llvmir-debug.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-debug.mlir @@ -564,3 +564,30 @@ llvm.func @subranges(%arg: !llvm.ptr) { // CHECK: ![[ELEMENTS2]] = !{![[ELEMENT2:[0-9]+]]} // CHECK: ![[ELEMENT2]] = !DISubrange(count: ![[LV:[0-9]+]], stride: ![[GV:[0-9]+]]) // CHECK: ![[LV]] = !DILocalVariable(name: "size"{{.*}}) + +// ----- + +#bt = #llvm.di_basic_type +#file = #llvm.di_file<"debug-info.ll" in "/"> +#cu = #llvm.di_compile_unit, sourceLanguage = DW_LANG_C, + file = #file, isOptimized = false, emissionKind = Full> +#sp = #llvm.di_subprogram +#var = #llvm.di_local_variable +#ty = #llvm.di_string_type, + stringLocationExp = <[DW_OP_push_object_address, DW_OP_deref]>> +#var1 = #llvm.di_local_variable + +llvm.func @string_ty(%arg0: !llvm.ptr) { + llvm.intr.dbg.value #var1 = %arg0 : !llvm.ptr + llvm.intr.dbg.value #var = %arg0 : !llvm.ptr + llvm.return +} loc(#loc2) + +#loc1 = loc("test.f90":1:1) +#loc2 = loc(fused<#sp>[#loc1]) + +// CHECK-DAG: !DIStringType(name: "character(*)", stringLength: ![[VAR:[0-9]+]], stringLengthExpression: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 8), stringLocationExpression: !DIExpression(DW_OP_push_object_address, DW_OP_deref), size: 32, align: 8) +// CHECK-DAG: ![[VAR]] = !DILocalVariable(name: "string_size"{{.*}}) -- GitLab From 5f1adf0433c6007f8be885b832c852da67e8524c Mon Sep 17 00:00:00 2001 From: Utkarsh Saxena Date: Fri, 7 Jun 2024 11:08:25 +0200 Subject: [PATCH 212/462] [clangd] Fix crash with null check for Token at Loc (#94528) Fixes https://github.com/llvm/llvm-project/issues/94599 --- clang-tools-extra/clangd/FindSymbols.cpp | 2 +- clang-tools-extra/clangd/IncludeCleaner.cpp | 2 +- .../clangd/SemanticHighlighting.cpp | 11 +++++------ clang-tools-extra/clangd/XRefs.cpp | 9 +++++---- clang-tools-extra/clangd/refactor/Rename.cpp | 2 +- .../clangd/unittests/PreambleTests.cpp | 11 ++++++----- .../clangd/unittests/XRefsTests.cpp | 16 +++++++++++++++- clang/include/clang/Tooling/Syntax/Tokens.h | 4 ++-- clang/lib/Tooling/Syntax/Tokens.cpp | 7 ++++--- clang/unittests/Tooling/Syntax/TokensTest.cpp | 17 +++++++++++++++-- 10 files changed, 55 insertions(+), 26 deletions(-) diff --git a/clang-tools-extra/clangd/FindSymbols.cpp b/clang-tools-extra/clangd/FindSymbols.cpp index 5244a4e89376..55f16b7085a6 100644 --- a/clang-tools-extra/clangd/FindSymbols.cpp +++ b/clang-tools-extra/clangd/FindSymbols.cpp @@ -454,7 +454,7 @@ private: if (!MacroName.isValid() || !MacroName.isFileID()) continue; // All conditions satisfied, add the macro. - if (auto *Tok = AST.getTokens().spelledTokenAt(MacroName)) + if (auto *Tok = AST.getTokens().spelledTokenContaining(MacroName)) CurParent = &CurParent->inMacro( *Tok, SM, AST.getTokens().expansionStartingAt(Tok)); } diff --git a/clang-tools-extra/clangd/IncludeCleaner.cpp b/clang-tools-extra/clangd/IncludeCleaner.cpp index 01b47679790f..dc5b7ec95db5 100644 --- a/clang-tools-extra/clangd/IncludeCleaner.cpp +++ b/clang-tools-extra/clangd/IncludeCleaner.cpp @@ -303,7 +303,7 @@ collectMacroReferences(ParsedAST &AST) { for (const auto &[_, Refs] : AST.getMacros().MacroRefs) { for (const auto &Ref : Refs) { auto Loc = SM.getComposedLoc(SM.getMainFileID(), Ref.StartOffset); - const auto *Tok = AST.getTokens().spelledTokenAt(Loc); + const auto *Tok = AST.getTokens().spelledTokenContaining(Loc); if (!Tok) continue; auto Macro = locateMacroAt(*Tok, PP); diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp index eb025f21f361..a366f1331c2d 100644 --- a/clang-tools-extra/clangd/SemanticHighlighting.cpp +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -447,11 +447,10 @@ public: if (!RLoc.isValid()) return; - const auto *RTok = TB.spelledTokenAt(RLoc); - // Handle `>>`. RLoc is always pointing at the right location, just change - // the end to be offset by 1. - // We'll either point at the beginning of `>>`, hence get a proper spelled - // or point in the middle of `>>` hence get no spelled tok. + const auto *RTok = TB.spelledTokenContaining(RLoc); + // Handle `>>`. RLoc is either part of `>>` or a spelled token on its own + // `>`. If it's the former, slice to have length of 1, if latter use the + // token as-is. if (!RTok || RTok->kind() == tok::greatergreater) { Position Begin = sourceLocToPosition(SourceMgr, RLoc); Position End = sourceLocToPosition(SourceMgr, RLoc.getLocWithOffset(1)); @@ -577,7 +576,7 @@ private: return std::nullopt; // We might have offsets in the main file that don't correspond to any // spelled tokens. - const auto *Tok = TB.spelledTokenAt(Loc); + const auto *Tok = TB.spelledTokenContaining(Loc); if (!Tok) return std::nullopt; return halfOpenToRange(SourceMgr, diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp index cd909266489a..f94cadeffaa2 100644 --- a/clang-tools-extra/clangd/XRefs.cpp +++ b/clang-tools-extra/clangd/XRefs.cpp @@ -844,7 +844,7 @@ std::vector getDocumentLinks(ParsedAST &AST) { if (Inc.Resolved.empty()) continue; auto HashLoc = SM.getComposedLoc(SM.getMainFileID(), Inc.HashOffset); - const auto *HashTok = AST.getTokens().spelledTokenAt(HashLoc); + const auto *HashTok = AST.getTokens().spelledTokenContaining(HashLoc); assert(HashTok && "got inclusion at wrong offset"); const auto *IncludeTok = std::next(HashTok); const auto *FileTok = std::next(IncludeTok); @@ -938,7 +938,7 @@ public: CollectorOpts.CollectMainFileSymbols = true; for (SourceLocation L : Locs) { L = SM.getFileLoc(L); - if (const auto *Tok = TB.spelledTokenAt(L)) + if (const auto *Tok = TB.spelledTokenContaining(L)) References.push_back( {*Tok, Roles, SymbolCollector::getRefContainer(ASTNode.Parent, CollectorOpts)}); @@ -1216,7 +1216,7 @@ DocumentHighlight toHighlight(const ReferenceFinder::Reference &Ref, std::optional toHighlight(SourceLocation Loc, const syntax::TokenBuffer &TB) { Loc = TB.sourceManager().getFileLoc(Loc); - if (const auto *Tok = TB.spelledTokenAt(Loc)) { + if (const auto *Tok = TB.spelledTokenContaining(Loc)) { DocumentHighlight Result; Result.range = halfOpenToRange( TB.sourceManager(), @@ -1353,7 +1353,8 @@ maybeFindIncludeReferences(ParsedAST &AST, Position Pos, Loc = SM.getIncludeLoc(SM.getFileID(Loc)); ReferencesResult::Reference Result; - const auto *Token = AST.getTokens().spelledTokenAt(Loc); + const auto *Token = AST.getTokens().spelledTokenContaining(Loc); + assert(Token && "references expected token here"); Result.Loc.range = Range{sourceLocToPosition(SM, Token->location()), sourceLocToPosition(SM, Token->endLocation())}; Result.Loc.uri = URIMainFile; diff --git a/clang-tools-extra/clangd/refactor/Rename.cpp b/clang-tools-extra/clangd/refactor/Rename.cpp index c0fc4453a3fc..c85e13dbdfe9 100644 --- a/clang-tools-extra/clangd/refactor/Rename.cpp +++ b/clang-tools-extra/clangd/refactor/Rename.cpp @@ -748,7 +748,7 @@ std::vector collectRenameIdentifierRanges( clangd::Range tokenRangeForLoc(ParsedAST &AST, SourceLocation TokLoc, const SourceManager &SM, const LangOptions &LangOpts) { - const auto *Token = AST.getTokens().spelledTokenAt(TokLoc); + const auto *Token = AST.getTokens().spelledTokenContaining(TokLoc); assert(Token && "rename expects spelled tokens"); clangd::Range Result; Result.start = sourceLocToPosition(SM, Token->location()); diff --git a/clang-tools-extra/clangd/unittests/PreambleTests.cpp b/clang-tools-extra/clangd/unittests/PreambleTests.cpp index 6420516e7855..16a2f9448b1e 100644 --- a/clang-tools-extra/clangd/unittests/PreambleTests.cpp +++ b/clang-tools-extra/clangd/unittests/PreambleTests.cpp @@ -417,7 +417,7 @@ TEST(PreamblePatchTest, LocateMacroAtWorks) { ASSERT_TRUE(AST); const auto &SM = AST->getSourceManager(); - auto *MacroTok = AST->getTokens().spelledTokenAt( + auto *MacroTok = AST->getTokens().spelledTokenContaining( SM.getComposedLoc(SM.getMainFileID(), Modified.point("use"))); ASSERT_TRUE(MacroTok); @@ -441,7 +441,7 @@ TEST(PreamblePatchTest, LocateMacroAtDeletion) { ASSERT_TRUE(AST); const auto &SM = AST->getSourceManager(); - auto *MacroTok = AST->getTokens().spelledTokenAt( + auto *MacroTok = AST->getTokens().spelledTokenContaining( SM.getComposedLoc(SM.getMainFileID(), Modified.point())); ASSERT_TRUE(MacroTok); @@ -512,9 +512,10 @@ TEST(PreamblePatchTest, RefsToMacros) { ExpectedLocations.push_back(referenceRangeIs(R)); for (const auto &P : Modified.points()) { - auto *MacroTok = AST->getTokens().spelledTokenAt(SM.getComposedLoc( - SM.getMainFileID(), - llvm::cantFail(positionToOffset(Modified.code(), P)))); + auto *MacroTok = + AST->getTokens().spelledTokenContaining(SM.getComposedLoc( + SM.getMainFileID(), + llvm::cantFail(positionToOffset(Modified.code(), P)))); ASSERT_TRUE(MacroTok); EXPECT_THAT(findReferences(*AST, P, 0).References, testing::ElementsAreArray(ExpectedLocations)); diff --git a/clang-tools-extra/clangd/unittests/XRefsTests.cpp b/clang-tools-extra/clangd/unittests/XRefsTests.cpp index f53cbf01b799..cbceb9a343f8 100644 --- a/clang-tools-extra/clangd/unittests/XRefsTests.cpp +++ b/clang-tools-extra/clangd/unittests/XRefsTests.cpp @@ -2173,6 +2173,11 @@ TEST(FindReferences, WithinAST) { using $def[[MyTypeD^ef]] = int; enum MyEnum : $(MyEnum)[[MyTy^peDef]] { }; )cpp", + // UDL + R"cpp( + bool $decl[[operator]]"" _u^dl(unsigned long long value); + bool x = $(x)[[1_udl]]; + )cpp", }; for (const char *Test : Tests) checkFindRefs(Test); @@ -2358,7 +2363,13 @@ TEST(FindReferences, UsedSymbolsFromInclude) { R"cpp([[#in^clude ]] std::[[vector]] vec; - )cpp"}; + )cpp", + + R"cpp( + [[#include ^"udl_header.h"]] + auto x = [[1_b]]; + )cpp", + }; for (const char *Test : Tests) { Annotations T(Test); auto TU = TestTU::withCode(T.code()); @@ -2375,6 +2386,9 @@ TEST(FindReferences, UsedSymbolsFromInclude) { class vector{}; } )cpp"); + TU.AdditionalFiles["udl_header.h"] = guard(R"cpp( + bool operator"" _b(unsigned long long value); + )cpp"); TU.ExtraArgs.push_back("-isystem" + testPath("system")); auto AST = TU.build(); diff --git a/clang/include/clang/Tooling/Syntax/Tokens.h b/clang/include/clang/Tooling/Syntax/Tokens.h index b1bdefed7c97..f71b8d67bfea 100644 --- a/clang/include/clang/Tooling/Syntax/Tokens.h +++ b/clang/include/clang/Tooling/Syntax/Tokens.h @@ -292,9 +292,9 @@ public: /// "DECL", "(", "a", ")", ";"} llvm::ArrayRef spelledTokens(FileID FID) const; - /// Returns the spelled Token starting at Loc, if there are no such tokens + /// Returns the spelled Token containing the Loc, if there are no such tokens /// returns nullptr. - const syntax::Token *spelledTokenAt(SourceLocation Loc) const; + const syntax::Token *spelledTokenContaining(SourceLocation Loc) const; /// Get all tokens that expand a macro in \p FID. For the following input /// #define FOO B diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp index 8d32c45a4a70..0a656dff3842 100644 --- a/clang/lib/Tooling/Syntax/Tokens.cpp +++ b/clang/lib/Tooling/Syntax/Tokens.cpp @@ -383,12 +383,13 @@ llvm::ArrayRef TokenBuffer::spelledTokens(FileID FID) const { return It->second.SpelledTokens; } -const syntax::Token *TokenBuffer::spelledTokenAt(SourceLocation Loc) const { +const syntax::Token * +TokenBuffer::spelledTokenContaining(SourceLocation Loc) const { assert(Loc.isFileID()); const auto *Tok = llvm::partition_point( spelledTokens(SourceMgr->getFileID(Loc)), - [&](const syntax::Token &Tok) { return Tok.location() < Loc; }); - if (!Tok || Tok->location() != Loc) + [&](const syntax::Token &Tok) { return Tok.endLocation() <= Loc; }); + if (!Tok || Loc < Tok->location()) return nullptr; return Tok; } diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp index 42f516971396..dc8a11dbc345 100644 --- a/clang/unittests/Tooling/Syntax/TokensTest.cpp +++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp @@ -374,11 +374,24 @@ TEST_F(TokenCollectorTest, Locations) { auto StartLoc = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID()); for (auto &R : Code.ranges()) { - EXPECT_THAT(Buffer.spelledTokenAt(StartLoc.getLocWithOffset(R.Begin)), - Pointee(RangeIs(R))); + EXPECT_THAT( + Buffer.spelledTokenContaining(StartLoc.getLocWithOffset(R.Begin)), + Pointee(RangeIs(R))); } } +TEST_F(TokenCollectorTest, LocationInMiddleOfSpelledToken) { + llvm::Annotations Code(R"cpp( + int foo = [[baa^aar]]; + )cpp"); + recordTokens(Code.code()); + // Check spelled tokens. + auto StartLoc = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID()); + EXPECT_THAT( + Buffer.spelledTokenContaining(StartLoc.getLocWithOffset(Code.point())), + Pointee(RangeIs(Code.range()))); +} + TEST_F(TokenCollectorTest, MacroDirectives) { // Macro directives are not stored anywhere at the moment. std::string Code = R"cpp( -- GitLab From 8f1164948d6b8f3d3ea11d0ee0629af82fe8cb74 Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Fri, 7 Jun 2024 10:14:05 +0100 Subject: [PATCH 213/462] [flang][Transforms][NFC] Remove boilerplate from vscale range pass (#94598) Use tablegen to generate the pass constructor. This pass is supposed to add function attributes so it does not need to operate on other top level operations. --- flang/include/flang/Optimizer/Transforms/Passes.h | 1 + flang/include/flang/Optimizer/Transforms/Passes.td | 1 - flang/include/flang/Tools/CLOptions.inc | 2 +- flang/lib/Optimizer/Transforms/VScaleAttr.cpp | 12 ------------ 4 files changed, 2 insertions(+), 14 deletions(-) diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h index 2d43f4d4c55b..4e978e6c9cde 100644 --- a/flang/include/flang/Optimizer/Transforms/Passes.h +++ b/flang/include/flang/Optimizer/Transforms/Passes.h @@ -54,6 +54,7 @@ namespace fir { #define GEN_PASS_DECL_OMPMAPINFOFINALIZATIONPASS #define GEN_PASS_DECL_OMPMARKDECLARETARGETPASS #define GEN_PASS_DECL_OMPFUNCTIONFILTERING +#define GEN_PASS_DECL_VSCALEATTR #include "flang/Optimizer/Transforms/Passes.h.inc" std::unique_ptr createAffineDemotionPass(); diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td index cac590a8da00..4e281e2846c5 100644 --- a/flang/include/flang/Optimizer/Transforms/Passes.td +++ b/flang/include/flang/Optimizer/Transforms/Passes.td @@ -359,7 +359,6 @@ def VScaleAttr : Pass<"vscale-attr", "mlir::func::FuncOp"> { "std::pair", /*default=*/"std::pair{}", "vector scale range">, ]; - let constructor = "::fir::createVScaleAttrPass()"; } def FunctionAttr : Pass<"function-attr", "mlir::func::FuncOp"> { diff --git a/flang/include/flang/Tools/CLOptions.inc b/flang/include/flang/Tools/CLOptions.inc index d0399d65f565..528c51d8c1c4 100644 --- a/flang/include/flang/Tools/CLOptions.inc +++ b/flang/include/flang/Tools/CLOptions.inc @@ -369,7 +369,7 @@ inline void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm, fir::createDebugPasses(pm, config.DebugInfo, config.OptLevel, inputFilename); if (config.VScaleMin != 0) - pm.addPass(fir::createVScaleAttrPass({config.VScaleMin, config.VScaleMax})); + pm.addPass(fir::createVScaleAttr({{config.VScaleMin, config.VScaleMax}})); // Add function attributes fir::FunctionAttrTypes functionAttrs; diff --git a/flang/lib/Optimizer/Transforms/VScaleAttr.cpp b/flang/lib/Optimizer/Transforms/VScaleAttr.cpp index 601a937de37b..d311167c58b4 100644 --- a/flang/lib/Optimizer/Transforms/VScaleAttr.cpp +++ b/flang/lib/Optimizer/Transforms/VScaleAttr.cpp @@ -38,7 +38,6 @@ #include namespace fir { -#define GEN_PASS_DECL_VSCALEATTR #define GEN_PASS_DEF_VSCALEATTR #include "flang/Optimizer/Transforms/Passes.h.inc" } // namespace fir @@ -77,14 +76,3 @@ void VScaleAttrPass::runOnOperation() { LLVM_DEBUG(llvm::dbgs() << "=== End " DEBUG_TYPE " ===\n"); } - -std::unique_ptr -fir::createVScaleAttrPass(std::pair vscaleAttr) { - VScaleAttrOptions opts; - opts.vscaleRange = vscaleAttr; - return std::make_unique(opts); -} - -std::unique_ptr fir::createVScaleAttrPass() { - return std::make_unique(); -} -- GitLab From 0749b01c81b7397f674cfe3726e13aa9dfb702ea Mon Sep 17 00:00:00 2001 From: Chen Zheng Date: Fri, 7 Jun 2024 04:43:25 -0400 Subject: [PATCH 214/462] [PowerPC] modify the frameaddress case, NFC --- llvm/test/CodeGen/PowerPC/frameaddr.ll | 212 +++++++++++++++++++++++-- 1 file changed, 195 insertions(+), 17 deletions(-) diff --git a/llvm/test/CodeGen/PowerPC/frameaddr.ll b/llvm/test/CodeGen/PowerPC/frameaddr.ll index 74263393442c..ade38ea457a3 100644 --- a/llvm/test/CodeGen/PowerPC/frameaddr.ll +++ b/llvm/test/CodeGen/PowerPC/frameaddr.ll @@ -1,38 +1,217 @@ -; RUN: llc -verify-machineinstrs < %s -mcpu=pwr7 | FileCheck %s -target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64" -target triple = "powerpc64-unknown-linux-gnu" - -declare void @llvm.eh.sjlj.longjmp(ptr) #1 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=powerpc-ibm-aix-xcoff -verify-machineinstrs < %s -mcpu=pwr7 | FileCheck %s --check-prefix=AIX32 +; RUN: llc -mtriple=powerpc64-ibm-aix-xcoff -verify-machineinstrs < %s -mcpu=pwr7 | FileCheck %s --check-prefix=AIX64 +; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs < %s -mcpu=pwr7 | FileCheck %s --check-prefix=LE define ptr @main() #0 { +; AIX32-LABEL: main: +; AIX32: # %bb.0: # %entry +; AIX32-NEXT: mr 3, 1 +; AIX32-NEXT: blr +; +; AIX64-LABEL: main: +; AIX64: # %bb.0: # %entry +; AIX64-NEXT: mr 3, 1 +; AIX64-NEXT: blr +; +; LE-LABEL: main: +; LE: # %bb.0: # %entry +; LE-NEXT: mr 3, 1 +; LE-NEXT: blr entry: %0 = call ptr @llvm.frameaddress(i32 0) ret ptr %0 +} -; CHECK: @main -; CHECK: mr 3, 1 +define ptr @foo_naked() #3 { ; naked +; AIX32-LABEL: foo_naked: +; AIX32: # %bb.0: # %entry +; AIX32-NEXT: lwz 3, 0(1) +; AIX32-NEXT: blr +; +; AIX64-LABEL: foo_naked: +; AIX64: # %bb.0: # %entry +; AIX64-NEXT: ld 3, 0(1) +; AIX64-NEXT: blr +; +; LE-LABEL: foo_naked: +; LE: # %bb.0: # %entry +; LE-NEXT: ld 3, 0(1) +; LE-NEXT: blr +entry: + %0 = call ptr @llvm.frameaddress(i32 1) + ret ptr %0 } -define ptr @foo() #3 { ; naked +define ptr @foo1() #0 { +; AIX32-LABEL: foo1: +; AIX32: # %bb.0: # %entry +; AIX32-NEXT: lwz 3, 0(1) +; AIX32-NEXT: blr +; +; AIX64-LABEL: foo1: +; AIX64: # %bb.0: # %entry +; AIX64-NEXT: ld 3, 0(1) +; AIX64-NEXT: blr +; +; LE-LABEL: foo1: +; LE: # %bb.0: # %entry +; LE-NEXT: ld 3, 0(1) +; LE-NEXT: blr entry: - %0 = call ptr @llvm.frameaddress(i32 0) + %0 = call ptr @llvm.frameaddress(i32 1) ret ptr %0 +} -; CHECK: @foo -; CHECK: mr 3, 1 +define ptr @foo2() #0 { +; AIX32-LABEL: foo2: +; AIX32: # %bb.0: # %entry +; AIX32-NEXT: lwz 3, 0(1) +; AIX32-NEXT: lwz 3, 0(3) +; AIX32-NEXT: blr +; +; AIX64-LABEL: foo2: +; AIX64: # %bb.0: # %entry +; AIX64-NEXT: ld 3, 0(1) +; AIX64-NEXT: ld 3, 0(3) +; AIX64-NEXT: blr +; +; LE-LABEL: foo2: +; LE: # %bb.0: # %entry +; LE-NEXT: ld 3, 0(1) +; LE-NEXT: ld 3, 0(3) +; LE-NEXT: blr +entry: + %0 = call ptr @llvm.frameaddress(i32 2) + ret ptr %0 } -define ptr @bar() #0 { +define ptr @bar0() #0 { +; AIX32-LABEL: bar0: +; AIX32: # %bb.0: # %entry +; AIX32-NEXT: mflr 0 +; AIX32-NEXT: stw 31, -4(1) +; AIX32-NEXT: stw 0, 8(1) +; AIX32-NEXT: lis 0, -2 +; AIX32-NEXT: ori 0, 0, 31008 +; AIX32-NEXT: stwux 1, 1, 0 +; AIX32-NEXT: mr 31, 1 +; AIX32-NEXT: addi 3, 31, 60 +; AIX32-NEXT: bl .use[PR] +; AIX32-NEXT: nop +; AIX32-NEXT: mr 3, 31 +; AIX32-NEXT: lwz 1, 0(1) +; AIX32-NEXT: lwz 0, 8(1) +; AIX32-NEXT: lwz 31, -4(1) +; AIX32-NEXT: mtlr 0 +; AIX32-NEXT: blr +; +; AIX64-LABEL: bar0: +; AIX64: # %bb.0: # %entry +; AIX64-NEXT: mflr 0 +; AIX64-NEXT: std 31, -8(1) +; AIX64-NEXT: std 0, 16(1) +; AIX64-NEXT: lis 0, -2 +; AIX64-NEXT: ori 0, 0, 30944 +; AIX64-NEXT: stdux 1, 1, 0 +; AIX64-NEXT: mr 31, 1 +; AIX64-NEXT: addi 3, 31, 120 +; AIX64-NEXT: bl .use[PR] +; AIX64-NEXT: nop +; AIX64-NEXT: mr 3, 31 +; AIX64-NEXT: ld 1, 0(1) +; AIX64-NEXT: ld 0, 16(1) +; AIX64-NEXT: ld 31, -8(1) +; AIX64-NEXT: mtlr 0 +; AIX64-NEXT: blr +; +; LE-LABEL: bar0: +; LE: # %bb.0: # %entry +; LE-NEXT: mflr 0 +; LE-NEXT: std 31, -8(1) +; LE-NEXT: std 0, 16(1) +; LE-NEXT: lis 0, -2 +; LE-NEXT: ori 0, 0, 31024 +; LE-NEXT: stdux 1, 1, 0 +; LE-NEXT: mr 31, 1 +; LE-NEXT: addi 3, 31, 40 +; LE-NEXT: bl use +; LE-NEXT: nop +; LE-NEXT: mr 3, 31 +; LE-NEXT: ld 1, 0(1) +; LE-NEXT: ld 0, 16(1) +; LE-NEXT: ld 31, -8(1) +; LE-NEXT: mtlr 0 +; LE-NEXT: blr entry: %x = alloca [100000 x i8] ; [#uses=1] call void @use(ptr %x) nounwind %0 = call ptr @llvm.frameaddress(i32 0) ret ptr %0 +} -; Note that if we start eliminating non-leaf frame pointers by default, this -; will need to be updated. -; CHECK: @bar -; CHECK: mr 3, 31 +define ptr @bar1() #0 { +; AIX32-LABEL: bar1: +; AIX32: # %bb.0: # %entry +; AIX32-NEXT: mflr 0 +; AIX32-NEXT: stw 31, -4(1) +; AIX32-NEXT: stw 0, 8(1) +; AIX32-NEXT: lis 0, -2 +; AIX32-NEXT: ori 0, 0, 31008 +; AIX32-NEXT: stwux 1, 1, 0 +; AIX32-NEXT: mr 31, 1 +; AIX32-NEXT: addi 3, 31, 60 +; AIX32-NEXT: bl .use[PR] +; AIX32-NEXT: nop +; AIX32-NEXT: lwz 3, 0(31) +; AIX32-NEXT: lwz 1, 0(1) +; AIX32-NEXT: lwz 0, 8(1) +; AIX32-NEXT: lwz 31, -4(1) +; AIX32-NEXT: mtlr 0 +; AIX32-NEXT: blr +; +; AIX64-LABEL: bar1: +; AIX64: # %bb.0: # %entry +; AIX64-NEXT: mflr 0 +; AIX64-NEXT: std 31, -8(1) +; AIX64-NEXT: std 0, 16(1) +; AIX64-NEXT: lis 0, -2 +; AIX64-NEXT: ori 0, 0, 30944 +; AIX64-NEXT: stdux 1, 1, 0 +; AIX64-NEXT: mr 31, 1 +; AIX64-NEXT: addi 3, 31, 120 +; AIX64-NEXT: bl .use[PR] +; AIX64-NEXT: nop +; AIX64-NEXT: ld 3, 0(31) +; AIX64-NEXT: ld 1, 0(1) +; AIX64-NEXT: ld 0, 16(1) +; AIX64-NEXT: ld 31, -8(1) +; AIX64-NEXT: mtlr 0 +; AIX64-NEXT: blr +; +; LE-LABEL: bar1: +; LE: # %bb.0: # %entry +; LE-NEXT: mflr 0 +; LE-NEXT: std 31, -8(1) +; LE-NEXT: std 0, 16(1) +; LE-NEXT: lis 0, -2 +; LE-NEXT: ori 0, 0, 31024 +; LE-NEXT: stdux 1, 1, 0 +; LE-NEXT: mr 31, 1 +; LE-NEXT: addi 3, 31, 40 +; LE-NEXT: bl use +; LE-NEXT: nop +; LE-NEXT: ld 3, 0(31) +; LE-NEXT: ld 1, 0(1) +; LE-NEXT: ld 0, 16(1) +; LE-NEXT: ld 31, -8(1) +; LE-NEXT: mtlr 0 +; LE-NEXT: blr +entry: + %x = alloca [100000 x i8] ; [#uses=1] + call void @use(ptr %x) nounwind + %0 = call ptr @llvm.frameaddress(i32 1) + ret ptr %0 } declare void @use(ptr) @@ -43,4 +222,3 @@ attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-lea attributes #1 = { noreturn nounwind } attributes #2 = { nounwind readnone } attributes #3 = { nounwind naked "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" } - -- GitLab From 3453dedfaf565429bc06c6d58533926f793ad650 Mon Sep 17 00:00:00 2001 From: Chen Zheng Date: Fri, 7 Jun 2024 04:58:55 -0400 Subject: [PATCH 215/462] [PowerPC] return correct frame address for frameaddress intrinsic --- llvm/lib/Target/PowerPC/PPCFrameLowering.cpp | 3 ++- .../CodeGen/PowerPC/2010-05-03-retaddr1.ll | 2 ++ llvm/test/CodeGen/PowerPC/frameaddr.ll | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp index 8444266459c4..201b2d162372 100644 --- a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp @@ -317,7 +317,8 @@ PPCFrameLowering::determineFrameLayout(const MachineFunction &MF, !MFI.adjustsStack() && // No calls. !MustSaveLR(MF, LR) && // No need to save LR. !FI->mustSaveTOC() && // No need to save TOC. - !RegInfo->hasBasePointer(MF); // No special alignment. + !RegInfo->hasBasePointer(MF) && // No special alignment. + !MFI.isFrameAddressTaken(); // Note: for PPC32 SVR4ABI, we can still generate stackless // code if all local vars are reg-allocated. diff --git a/llvm/test/CodeGen/PowerPC/2010-05-03-retaddr1.ll b/llvm/test/CodeGen/PowerPC/2010-05-03-retaddr1.ll index e4deef157be8..87565df1506b 100644 --- a/llvm/test/CodeGen/PowerPC/2010-05-03-retaddr1.ll +++ b/llvm/test/CodeGen/PowerPC/2010-05-03-retaddr1.ll @@ -7,7 +7,9 @@ declare ptr @llvm.frameaddress(i32) nounwind readnone define ptr @g2() nounwind readnone { ; CHECK-LABEL: g2: ; CHECK: # %bb.0: # %entry +; CHECK-NEXT: stwu 1, -16(1) ; CHECK-NEXT: lwz 3, 0(1) +; CHECK-NEXT: addi 1, 1, 16 ; CHECK-NEXT: blr entry: %0 = tail call ptr @llvm.frameaddress(i32 1) ; [#uses=1] diff --git a/llvm/test/CodeGen/PowerPC/frameaddr.ll b/llvm/test/CodeGen/PowerPC/frameaddr.ll index ade38ea457a3..180f736c5ddb 100644 --- a/llvm/test/CodeGen/PowerPC/frameaddr.ll +++ b/llvm/test/CodeGen/PowerPC/frameaddr.ll @@ -6,17 +6,23 @@ define ptr @main() #0 { ; AIX32-LABEL: main: ; AIX32: # %bb.0: # %entry +; AIX32-NEXT: stwu 1, -32(1) ; AIX32-NEXT: mr 3, 1 +; AIX32-NEXT: addi 1, 1, 32 ; AIX32-NEXT: blr ; ; AIX64-LABEL: main: ; AIX64: # %bb.0: # %entry +; AIX64-NEXT: stdu 1, -48(1) ; AIX64-NEXT: mr 3, 1 +; AIX64-NEXT: addi 1, 1, 48 ; AIX64-NEXT: blr ; ; LE-LABEL: main: ; LE: # %bb.0: # %entry +; LE-NEXT: stdu 1, -32(1) ; LE-NEXT: mr 3, 1 +; LE-NEXT: addi 1, 1, 32 ; LE-NEXT: blr entry: %0 = call ptr @llvm.frameaddress(i32 0) @@ -46,17 +52,23 @@ entry: define ptr @foo1() #0 { ; AIX32-LABEL: foo1: ; AIX32: # %bb.0: # %entry +; AIX32-NEXT: stwu 1, -32(1) ; AIX32-NEXT: lwz 3, 0(1) +; AIX32-NEXT: addi 1, 1, 32 ; AIX32-NEXT: blr ; ; AIX64-LABEL: foo1: ; AIX64: # %bb.0: # %entry +; AIX64-NEXT: stdu 1, -48(1) ; AIX64-NEXT: ld 3, 0(1) +; AIX64-NEXT: addi 1, 1, 48 ; AIX64-NEXT: blr ; ; LE-LABEL: foo1: ; LE: # %bb.0: # %entry +; LE-NEXT: stdu 1, -32(1) ; LE-NEXT: ld 3, 0(1) +; LE-NEXT: addi 1, 1, 32 ; LE-NEXT: blr entry: %0 = call ptr @llvm.frameaddress(i32 1) @@ -66,20 +78,26 @@ entry: define ptr @foo2() #0 { ; AIX32-LABEL: foo2: ; AIX32: # %bb.0: # %entry +; AIX32-NEXT: stwu 1, -32(1) ; AIX32-NEXT: lwz 3, 0(1) ; AIX32-NEXT: lwz 3, 0(3) +; AIX32-NEXT: addi 1, 1, 32 ; AIX32-NEXT: blr ; ; AIX64-LABEL: foo2: ; AIX64: # %bb.0: # %entry +; AIX64-NEXT: stdu 1, -48(1) ; AIX64-NEXT: ld 3, 0(1) ; AIX64-NEXT: ld 3, 0(3) +; AIX64-NEXT: addi 1, 1, 48 ; AIX64-NEXT: blr ; ; LE-LABEL: foo2: ; LE: # %bb.0: # %entry +; LE-NEXT: stdu 1, -32(1) ; LE-NEXT: ld 3, 0(1) ; LE-NEXT: ld 3, 0(3) +; LE-NEXT: addi 1, 1, 32 ; LE-NEXT: blr entry: %0 = call ptr @llvm.frameaddress(i32 2) -- GitLab From c0b468523c9c5517e61a197e7c1fe6cb52f8999c Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 7 Jun 2024 10:18:45 +0100 Subject: [PATCH 216/462] [ARM] Add NEON support for ISD::ABDS/ABDU nodes. (#94504) As noted on #94466, NEON has ABDS/ABDU instructions but only handles them via intrinsics, plus some VABDL custom patterns. This patch flags basic ABDS/ABDU for neon types as legal and updates all tablegen patterns to use abds/abdu instead. Fixes #94466 --- llvm/lib/Target/ARM/ARMISelLowering.cpp | 16 +- llvm/lib/Target/ARM/ARMInstrNEON.td | 24 +-- llvm/test/CodeGen/ARM/neon_vabd.ll | 201 ++++++++---------------- 3 files changed, 87 insertions(+), 154 deletions(-) diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 5212d2c620b7..78aaaca4e185 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -205,9 +205,9 @@ void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT) { setOperationAction(ISD::SDIVREM, VT, Expand); setOperationAction(ISD::UDIVREM, VT, Expand); - if (!VT.isFloatingPoint() && - VT != MVT::v2i64 && VT != MVT::v1i64) - for (auto Opcode : {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) + if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64) + for (auto Opcode : {ISD::ABS, ISD::ABDS, ISD::ABDU, ISD::SMIN, ISD::SMAX, + ISD::UMIN, ISD::UMAX}) setOperationAction(Opcode, VT, Legal); if (!VT.isFloatingPoint()) for (auto Opcode : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT}) @@ -4174,7 +4174,15 @@ ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG, } case Intrinsic::arm_neon_vabs: return DAG.getNode(ISD::ABS, SDLoc(Op), Op.getValueType(), - Op.getOperand(1)); + Op.getOperand(1)); + case Intrinsic::arm_neon_vabds: + if (Op.getValueType().isInteger()) + return DAG.getNode(ISD::ABDS, SDLoc(Op), Op.getValueType(), + Op.getOperand(1), Op.getOperand(2)); + return SDValue(); + case Intrinsic::arm_neon_vabdu: + return DAG.getNode(ISD::ABDU, SDLoc(Op), Op.getValueType(), + Op.getOperand(1), Op.getOperand(2)); case Intrinsic::arm_neon_vmulls: case Intrinsic::arm_neon_vmullu: { unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls) diff --git a/llvm/lib/Target/ARM/ARMInstrNEON.td b/llvm/lib/Target/ARM/ARMInstrNEON.td index 21a5817252ae..c600478b6640 100644 --- a/llvm/lib/Target/ARM/ARMInstrNEON.td +++ b/llvm/lib/Target/ARM/ARMInstrNEON.td @@ -5640,10 +5640,10 @@ def VBITq : N3VX<1, 0, 0b10, 0b0001, 1, 1, // VABD : Vector Absolute Difference defm VABDs : N3VInt_QHS<0, 0, 0b0111, 0, N3RegFrm, IIC_VSUBi4D, IIC_VSUBi4D, IIC_VSUBi4Q, IIC_VSUBi4Q, - "vabd", "s", int_arm_neon_vabds, 1>; + "vabd", "s", abds, 1>; defm VABDu : N3VInt_QHS<1, 0, 0b0111, 0, N3RegFrm, IIC_VSUBi4D, IIC_VSUBi4D, IIC_VSUBi4Q, IIC_VSUBi4Q, - "vabd", "u", int_arm_neon_vabdu, 1>; + "vabd", "u", abdu, 1>; def VABDfd : N3VDInt<1, 0, 0b10, 0b1101, 0, N3RegFrm, IIC_VBIND, "vabd", "f32", v2f32, v2f32, int_arm_neon_vabds, 1>; def VABDfq : N3VQInt<1, 0, 0b10, 0b1101, 0, N3RegFrm, IIC_VBINQ, @@ -5657,20 +5657,22 @@ def VABDhq : N3VQInt<1, 0, 0b11, 0b1101, 0, N3RegFrm, IIC_VBINQ, // VABDL : Vector Absolute Difference Long (Q = | D - D |) defm VABDLs : N3VLIntExt_QHS<0,1,0b0111,0, IIC_VSUBi4Q, - "vabdl", "s", int_arm_neon_vabds, zext, 1>; + "vabdl", "s", abds, zext, 1>; defm VABDLu : N3VLIntExt_QHS<1,1,0b0111,0, IIC_VSUBi4Q, - "vabdl", "u", int_arm_neon_vabdu, zext, 1>; + "vabdl", "u", abdu, zext, 1>; let Predicates = [HasNEON] in { -def : Pat<(v8i16 (abs (sub (zext (v8i8 DPR:$opA)), (zext (v8i8 DPR:$opB))))), +def : Pat<(v8i16 (zext (abdu (v8i8 DPR:$opA), (v8i8 DPR:$opB)))), (VABDLuv8i16 DPR:$opA, DPR:$opB)>; -def : Pat<(v4i32 (abs (sub (zext (v4i16 DPR:$opA)), (zext (v4i16 DPR:$opB))))), +def : Pat<(v4i32 (zext (abdu (v4i16 DPR:$opA), (v4i16 DPR:$opB)))), (VABDLuv4i32 DPR:$opA, DPR:$opB)>; +def : Pat<(v2i64 (zext (abdu (v2i32 DPR:$opA), (v2i32 DPR:$opB)))), + (VABDLuv2i64 DPR:$opA, DPR:$opB)>; } // ISD::ABS is not legal for v2i64, so VABDL needs to be matched from the // shift/xor pattern for ABS. - +// TODO: Remove me. def abd_shr : PatFrag<(ops node:$in1, node:$in2, node:$shift), (ARMvshrsImm (sub (zext node:$in1), @@ -5686,15 +5688,15 @@ def : Pat<(xor (v2i64 (abd_shr (v2i32 DPR:$opA), (v2i32 DPR:$opB), 63)), // VABA : Vector Absolute Difference and Accumulate defm VABAs : N3VIntOp_QHS<0,0,0b0111,1, IIC_VABAD, IIC_VABAQ, - "vaba", "s", int_arm_neon_vabds, add>; + "vaba", "s", abds, add>; defm VABAu : N3VIntOp_QHS<1,0,0b0111,1, IIC_VABAD, IIC_VABAQ, - "vaba", "u", int_arm_neon_vabdu, add>; + "vaba", "u", abdu, add>; // VABAL : Vector Absolute Difference and Accumulate Long (Q += | D - D |) defm VABALs : N3VLIntExtOp_QHS<0,1,0b0101,0, IIC_VABAD, - "vabal", "s", int_arm_neon_vabds, zext, add>; + "vabal", "s", abds, zext, add>; defm VABALu : N3VLIntExtOp_QHS<1,1,0b0101,0, IIC_VABAD, - "vabal", "u", int_arm_neon_vabdu, zext, add>; + "vabal", "u", abdu, zext, add>; // Vector Maximum and Minimum. diff --git a/llvm/test/CodeGen/ARM/neon_vabd.ll b/llvm/test/CodeGen/ARM/neon_vabd.ll index 14ad1a108a72..907e11c0cf19 100644 --- a/llvm/test/CodeGen/ARM/neon_vabd.ll +++ b/llvm/test/CodeGen/ARM/neon_vabd.ll @@ -10,9 +10,7 @@ define <8 x i8> @sabd_8b(<8 x i8> %a, <8 x i8> %b) { ; CHECK: @ %bb.0: ; CHECK-NEXT: vmov d16, r2, r3 ; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vsubl.s8 q8, d17, d16 -; CHECK-NEXT: vabs.s16 q8, q8 -; CHECK-NEXT: vmovn.i16 d16, q8 +; CHECK-NEXT: vabd.s8 d16, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: mov pc, lr %a.sext = sext <8 x i8> %a to <8 x i16> @@ -26,18 +24,13 @@ define <8 x i8> @sabd_8b(<8 x i8> %a, <8 x i8> %b) { define <16 x i8> @sabd_16b(<16 x i8> %a, <16 x i8> %b) { ; CHECK-LABEL: sabd_16b: ; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 ; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d16, d17}, [r12] -; CHECK-NEXT: vmov d18, r2, r3 -; CHECK-NEXT: vmov d19, r0, r1 -; CHECK-NEXT: vsubl.s8 q10, d18, d17 -; CHECK-NEXT: vsubl.s8 q8, d19, d16 -; CHECK-NEXT: vabs.s16 q9, q10 -; CHECK-NEXT: vabs.s16 q8, q8 -; CHECK-NEXT: vmovn.i16 d19, q9 -; CHECK-NEXT: vmovn.i16 d18, q8 -; CHECK-NEXT: vmov r2, r3, d19 -; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vabd.s8 q8, q8, q9 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr %a.sext = sext <16 x i8> %a to <16 x i16> %b.sext = sext <16 x i8> %b to <16 x i16> @@ -52,9 +45,7 @@ define <4 x i16> @sabd_4h(<4 x i16> %a, <4 x i16> %b) { ; CHECK: @ %bb.0: ; CHECK-NEXT: vmov d16, r2, r3 ; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vsubl.s16 q8, d17, d16 -; CHECK-NEXT: vabs.s32 q8, q8 -; CHECK-NEXT: vmovn.i32 d16, q8 +; CHECK-NEXT: vabd.s16 d16, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: mov pc, lr %a.sext = sext <4 x i16> %a to <4 x i32> @@ -74,8 +65,7 @@ define <4 x i16> @sabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) { ; CHECK-NEXT: vshl.i16 d17, d17, #8 ; CHECK-NEXT: vshr.s16 d16, d16, #8 ; CHECK-NEXT: vshr.s16 d17, d17, #8 -; CHECK-NEXT: vsub.i16 d16, d17, d16 -; CHECK-NEXT: vabs.s16 d16, d16 +; CHECK-NEXT: vabd.s16 d16, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: mov pc, lr %a.sext = sext <4 x i8> %a to <4 x i16> @@ -88,18 +78,13 @@ define <4 x i16> @sabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) { define <8 x i16> @sabd_8h(<8 x i16> %a, <8 x i16> %b) { ; CHECK-LABEL: sabd_8h: ; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 ; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d16, d17}, [r12] -; CHECK-NEXT: vmov d18, r2, r3 -; CHECK-NEXT: vmov d19, r0, r1 -; CHECK-NEXT: vsubl.s16 q10, d18, d17 -; CHECK-NEXT: vsubl.s16 q8, d19, d16 -; CHECK-NEXT: vabs.s32 q9, q10 -; CHECK-NEXT: vabs.s32 q8, q8 -; CHECK-NEXT: vmovn.i32 d19, q9 -; CHECK-NEXT: vmovn.i32 d18, q8 -; CHECK-NEXT: vmov r2, r3, d19 -; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vabd.s16 q8, q8, q9 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr %a.sext = sext <8 x i16> %a to <8 x i32> %b.sext = sext <8 x i16> %b to <8 x i32> @@ -114,8 +99,7 @@ define <8 x i16> @sabd_8h_promoted_ops(<8 x i8> %a, <8 x i8> %b) { ; CHECK: @ %bb.0: ; CHECK-NEXT: vmov d16, r2, r3 ; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vsubl.s8 q8, d17, d16 -; CHECK-NEXT: vabs.s16 q8, q8 +; CHECK-NEXT: vabdl.s8 q8, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr @@ -131,11 +115,7 @@ define <2 x i32> @sabd_2s(<2 x i32> %a, <2 x i32> %b) { ; CHECK: @ %bb.0: ; CHECK-NEXT: vmov d16, r2, r3 ; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vsubl.s32 q8, d17, d16 -; CHECK-NEXT: vshr.s64 q9, q8, #63 -; CHECK-NEXT: veor q8, q8, q9 -; CHECK-NEXT: vsub.i64 q8, q8, q9 -; CHECK-NEXT: vmovn.i64 d16, q8 +; CHECK-NEXT: vabd.s32 d16, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: mov pc, lr %a.sext = sext <2 x i32> %a to <2 x i64> @@ -155,8 +135,7 @@ define <2 x i32> @sabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) { ; CHECK-NEXT: vshl.i32 d17, d17, #16 ; CHECK-NEXT: vshr.s32 d16, d16, #16 ; CHECK-NEXT: vshr.s32 d17, d17, #16 -; CHECK-NEXT: vsub.i32 d16, d17, d16 -; CHECK-NEXT: vabs.s32 d16, d16 +; CHECK-NEXT: vabd.s32 d16, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: mov pc, lr %a.sext = sext <2 x i16> %a to <2 x i32> @@ -169,22 +148,13 @@ define <2 x i32> @sabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) { define <4 x i32> @sabd_4s(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: sabd_4s: ; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 ; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d16, d17}, [r12] -; CHECK-NEXT: vmov d18, r2, r3 -; CHECK-NEXT: vmov d19, r0, r1 -; CHECK-NEXT: vsubl.s32 q10, d18, d17 -; CHECK-NEXT: vsubl.s32 q8, d19, d16 -; CHECK-NEXT: vshr.s64 q9, q10, #63 -; CHECK-NEXT: vshr.s64 q11, q8, #63 -; CHECK-NEXT: veor q10, q10, q9 -; CHECK-NEXT: veor q8, q8, q11 -; CHECK-NEXT: vsub.i64 q9, q10, q9 -; CHECK-NEXT: vsub.i64 q8, q8, q11 -; CHECK-NEXT: vmovn.i64 d19, q9 -; CHECK-NEXT: vmovn.i64 d18, q8 -; CHECK-NEXT: vmov r2, r3, d19 -; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vabd.s32 q8, q8, q9 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr %a.sext = sext <4 x i32> %a to <4 x i64> %b.sext = sext <4 x i32> %b to <4 x i64> @@ -199,8 +169,7 @@ define <4 x i32> @sabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) { ; CHECK: @ %bb.0: ; CHECK-NEXT: vmov d16, r2, r3 ; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vsubl.s16 q8, d17, d16 -; CHECK-NEXT: vabs.s32 q8, q8 +; CHECK-NEXT: vabdl.s16 q8, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr @@ -259,10 +228,7 @@ define <2 x i64> @sabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) { ; CHECK: @ %bb.0: ; CHECK-NEXT: vmov d16, r2, r3 ; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vsubl.s32 q8, d17, d16 -; CHECK-NEXT: vshr.s64 q9, q8, #63 -; CHECK-NEXT: veor q8, q8, q9 -; CHECK-NEXT: vsub.i64 q8, q8, q9 +; CHECK-NEXT: vabdl.s32 q8, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr @@ -282,8 +248,7 @@ define <8 x i8> @uabd_8b(<8 x i8> %a, <8 x i8> %b) { ; CHECK: @ %bb.0: ; CHECK-NEXT: vmov d16, r2, r3 ; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabdl.u8 q8, d17, d16 -; CHECK-NEXT: vmovn.i16 d16, q8 +; CHECK-NEXT: vabd.u8 d16, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: mov pc, lr %a.zext = zext <8 x i8> %a to <8 x i16> @@ -297,16 +262,13 @@ define <8 x i8> @uabd_8b(<8 x i8> %a, <8 x i8> %b) { define <16 x i8> @uabd_16b(<16 x i8> %a, <16 x i8> %b) { ; CHECK-LABEL: uabd_16b: ; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 ; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d16, d17}, [r12] -; CHECK-NEXT: vmov d18, r2, r3 -; CHECK-NEXT: vmov d19, r0, r1 -; CHECK-NEXT: vabdl.u8 q10, d18, d17 -; CHECK-NEXT: vabdl.u8 q8, d19, d16 -; CHECK-NEXT: vmovn.i16 d19, q10 -; CHECK-NEXT: vmovn.i16 d18, q8 -; CHECK-NEXT: vmov r2, r3, d19 -; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vabd.u8 q8, q8, q9 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr %a.zext = zext <16 x i8> %a to <16 x i16> %b.zext = zext <16 x i8> %b to <16 x i16> @@ -321,8 +283,7 @@ define <4 x i16> @uabd_4h(<4 x i16> %a, <4 x i16> %b) { ; CHECK: @ %bb.0: ; CHECK-NEXT: vmov d16, r2, r3 ; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabdl.u16 q8, d17, d16 -; CHECK-NEXT: vmovn.i32 d16, q8 +; CHECK-NEXT: vabd.u16 d16, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: mov pc, lr %a.zext = zext <4 x i16> %a to <4 x i32> @@ -340,8 +301,7 @@ define <4 x i16> @uabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) { ; CHECK-NEXT: vmov d17, r0, r1 ; CHECK-NEXT: vbic.i16 d16, #0xff00 ; CHECK-NEXT: vbic.i16 d17, #0xff00 -; CHECK-NEXT: vsub.i16 d16, d17, d16 -; CHECK-NEXT: vabs.s16 d16, d16 +; CHECK-NEXT: vabd.u16 d16, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: mov pc, lr %a.zext = zext <4 x i8> %a to <4 x i16> @@ -354,16 +314,13 @@ define <4 x i16> @uabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) { define <8 x i16> @uabd_8h(<8 x i16> %a, <8 x i16> %b) { ; CHECK-LABEL: uabd_8h: ; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 ; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d16, d17}, [r12] -; CHECK-NEXT: vmov d18, r2, r3 -; CHECK-NEXT: vmov d19, r0, r1 -; CHECK-NEXT: vabdl.u16 q10, d18, d17 -; CHECK-NEXT: vabdl.u16 q8, d19, d16 -; CHECK-NEXT: vmovn.i32 d19, q10 -; CHECK-NEXT: vmovn.i32 d18, q8 -; CHECK-NEXT: vmov r2, r3, d19 -; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vabd.u16 q8, q8, q9 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr %a.zext = zext <8 x i16> %a to <8 x i32> %b.zext = zext <8 x i16> %b to <8 x i32> @@ -394,11 +351,7 @@ define <2 x i32> @uabd_2s(<2 x i32> %a, <2 x i32> %b) { ; CHECK: @ %bb.0: ; CHECK-NEXT: vmov d16, r2, r3 ; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vsubl.u32 q8, d17, d16 -; CHECK-NEXT: vshr.s64 q9, q8, #63 -; CHECK-NEXT: veor q8, q8, q9 -; CHECK-NEXT: vsub.i64 q8, q8, q9 -; CHECK-NEXT: vmovn.i64 d16, q8 +; CHECK-NEXT: vabd.u32 d16, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: mov pc, lr %a.zext = zext <2 x i32> %a to <2 x i64> @@ -417,8 +370,7 @@ define <2 x i32> @uabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) { ; CHECK-NEXT: vmov d18, r0, r1 ; CHECK-NEXT: vand d17, d17, d16 ; CHECK-NEXT: vand d16, d18, d16 -; CHECK-NEXT: vsub.i32 d16, d16, d17 -; CHECK-NEXT: vabs.s32 d16, d16 +; CHECK-NEXT: vabd.u32 d16, d16, d17 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: mov pc, lr %a.zext = zext <2 x i16> %a to <2 x i32> @@ -431,22 +383,13 @@ define <2 x i32> @uabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) { define <4 x i32> @uabd_4s(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: uabd_4s: ; CHECK: @ %bb.0: +; CHECK-NEXT: vmov d17, r2, r3 ; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d16, d17}, [r12] -; CHECK-NEXT: vmov d18, r2, r3 -; CHECK-NEXT: vmov d19, r0, r1 -; CHECK-NEXT: vsubl.u32 q10, d18, d17 -; CHECK-NEXT: vsubl.u32 q8, d19, d16 -; CHECK-NEXT: vshr.s64 q9, q10, #63 -; CHECK-NEXT: vshr.s64 q11, q8, #63 -; CHECK-NEXT: veor q10, q10, q9 -; CHECK-NEXT: veor q8, q8, q11 -; CHECK-NEXT: vsub.i64 q9, q10, q9 -; CHECK-NEXT: vsub.i64 q8, q8, q11 -; CHECK-NEXT: vmovn.i64 d19, q9 -; CHECK-NEXT: vmovn.i64 d18, q8 -; CHECK-NEXT: vmov r2, r3, d19 -; CHECK-NEXT: vmov r0, r1, d18 +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vabd.u32 q8, q8, q9 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr %a.zext = zext <4 x i32> %a to <4 x i64> %b.zext = zext <4 x i32> %b to <4 x i64> @@ -519,10 +462,7 @@ define <2 x i64> @uabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) { ; CHECK: @ %bb.0: ; CHECK-NEXT: vmov d16, r2, r3 ; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vsubl.u32 q8, d17, d16 -; CHECK-NEXT: vshr.s64 q9, q8, #63 -; CHECK-NEXT: veor q8, q8, q9 -; CHECK-NEXT: vsub.i64 q8, q8, q9 +; CHECK-NEXT: vabdl.u32 q8, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr @@ -610,8 +550,7 @@ define <16 x i8> @sabd_v16i8_nsw(<16 x i8> %a, <16 x i8> %b) { ; CHECK-NEXT: mov r12, sp ; CHECK-NEXT: vld1.64 {d18, d19}, [r12] ; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vsub.i8 q8, q8, q9 -; CHECK-NEXT: vabs.s8 q8, q8 +; CHECK-NEXT: vabd.s8 q8, q8, q9 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr @@ -627,8 +566,7 @@ define <8 x i16> @sabd_v8i16_nsw(<8 x i16> %a, <8 x i16> %b) { ; CHECK-NEXT: mov r12, sp ; CHECK-NEXT: vld1.64 {d18, d19}, [r12] ; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vsub.i16 q8, q8, q9 -; CHECK-NEXT: vabs.s16 q8, q8 +; CHECK-NEXT: vabd.s16 q8, q8, q9 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr @@ -644,8 +582,7 @@ define <4 x i32> @sabd_v4i32_nsw(<4 x i32> %a, <4 x i32> %b) { ; CHECK-NEXT: mov r12, sp ; CHECK-NEXT: vld1.64 {d18, d19}, [r12] ; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vsub.i32 q8, q8, q9 -; CHECK-NEXT: vabs.s32 q8, q8 +; CHECK-NEXT: vabd.s32 q8, q8, q9 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr @@ -680,9 +617,7 @@ define <16 x i8> @smaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) { ; CHECK-NEXT: mov r12, sp ; CHECK-NEXT: vld1.64 {d18, d19}, [r12] ; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vmin.s8 q10, q8, q9 -; CHECK-NEXT: vmax.s8 q8, q8, q9 -; CHECK-NEXT: vsub.i8 q8, q8, q10 +; CHECK-NEXT: vabd.s8 q8, q8, q9 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr @@ -699,9 +634,7 @@ define <8 x i16> @smaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) { ; CHECK-NEXT: mov r12, sp ; CHECK-NEXT: vld1.64 {d18, d19}, [r12] ; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vmin.s16 q10, q8, q9 -; CHECK-NEXT: vmax.s16 q8, q8, q9 -; CHECK-NEXT: vsub.i16 q8, q8, q10 +; CHECK-NEXT: vabd.s16 q8, q8, q9 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr @@ -718,9 +651,7 @@ define <4 x i32> @smaxmin_v4i32(<4 x i32> %0, <4 x i32> %1) { ; CHECK-NEXT: mov r12, sp ; CHECK-NEXT: vld1.64 {d18, d19}, [r12] ; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vmin.s32 q10, q8, q9 -; CHECK-NEXT: vmax.s32 q8, q8, q9 -; CHECK-NEXT: vsub.i32 q8, q8, q10 +; CHECK-NEXT: vabd.s32 q8, q8, q9 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr @@ -799,9 +730,7 @@ define <16 x i8> @umaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) { ; CHECK-NEXT: mov r12, sp ; CHECK-NEXT: vld1.64 {d18, d19}, [r12] ; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vmin.u8 q10, q8, q9 -; CHECK-NEXT: vmax.u8 q8, q8, q9 -; CHECK-NEXT: vsub.i8 q8, q8, q10 +; CHECK-NEXT: vabd.u8 q8, q8, q9 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr @@ -818,9 +747,7 @@ define <8 x i16> @umaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) { ; CHECK-NEXT: mov r12, sp ; CHECK-NEXT: vld1.64 {d18, d19}, [r12] ; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vmin.u16 q10, q8, q9 -; CHECK-NEXT: vmax.u16 q8, q8, q9 -; CHECK-NEXT: vsub.i16 q8, q8, q10 +; CHECK-NEXT: vabd.u16 q8, q8, q9 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr @@ -837,9 +764,7 @@ define <4 x i32> @umaxmin_v4i32(<4 x i32> %0, <4 x i32> %1) { ; CHECK-NEXT: mov r12, sp ; CHECK-NEXT: vld1.64 {d18, d19}, [r12] ; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vmin.u32 q10, q8, q9 -; CHECK-NEXT: vmax.u32 q8, q8, q9 -; CHECK-NEXT: vsub.i32 q8, q8, q10 +; CHECK-NEXT: vabd.u32 q8, q8, q9 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr @@ -874,12 +799,10 @@ define <16 x i8> @umaxmin_v16i8_com1(<16 x i8> %0, <16 x i8> %1) { ; CHECK-LABEL: umaxmin_v16i8_com1: ; CHECK: @ %bb.0: ; CHECK-NEXT: vmov d17, r2, r3 +; CHECK-NEXT: mov r12, sp +; CHECK-NEXT: vld1.64 {d18, d19}, [r12] ; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: mov r0, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vmin.u8 q10, q9, q8 -; CHECK-NEXT: vmax.u8 q8, q8, q9 -; CHECK-NEXT: vsub.i8 q8, q8, q10 +; CHECK-NEXT: vabd.u8 q8, q8, q9 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: vmov r2, r3, d17 ; CHECK-NEXT: mov pc, lr -- GitLab From 0d1b3671a91c3929fc3e3613491ff4b57f1adcc3 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Fri, 7 Jun 2024 02:27:59 -0700 Subject: [PATCH 217/462] [CMake][Release] Use the TXZ cpack generator for binaries (#90138) --- .github/workflows/release-binaries.yml | 2 +- clang/cmake/caches/Release.cmake | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 8fa3bf3d9df5..7de4d00334d1 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -216,7 +216,7 @@ jobs: - id: package-info run: | - filename="LLVM-${{ needs.prepare.outputs.release-version }}-Linux.tar.gz" + filename="LLVM-${{ needs.prepare.outputs.release-version }}-Linux.tar.xz" echo "filename=$filename" >> $GITHUB_OUTPUT echo "path=/mnt/build/tools/clang/stage2-bins/$filename" >> $GITHUB_OUTPUT diff --git a/clang/cmake/caches/Release.cmake b/clang/cmake/caches/Release.cmake index c0bfcbdfc1c2..1dfb1bc535bf 100644 --- a/clang/cmake/caches/Release.cmake +++ b/clang/cmake/caches/Release.cmake @@ -91,4 +91,6 @@ endif() # Final Stage Config (stage2) set_final_stage_var(LLVM_ENABLE_RUNTIMES "${LLVM_RELEASE_ENABLE_RUNTIMES}" STRING) set_final_stage_var(LLVM_ENABLE_PROJECTS "${LLVM_RELEASE_ENABLE_PROJECTS}" STRING) +set_final_stage_var(CPACK_GENERATOR "TXZ" STRING) +set_final_stage_var(CPACK_ARCHIVE_THREADS "0" STRING) -- GitLab From 1721c14e8e0d75cc611067b6f4e84028ea7c47d5 Mon Sep 17 00:00:00 2001 From: John Brawn Date: Fri, 7 Jun 2024 10:38:23 +0100 Subject: [PATCH 218/462] [DebugInfo] Add DW_OP_LLVM_extract_bits (#93990) This operation extracts a number of bits at a given offset and sign or zero extends them, which is done by emitting it as a left shift followed by a right shift. This is being added for use in clang for C++ structured bindings of bitfields that have offset or size that aren't a byte multiple. A new operation is being added, instead of shifts being used directly, as it makes correctly handling it in optimisations (which will be done in a later patch) much easier. --- llvm/docs/LangRef.rst | 9 ++ llvm/include/llvm/BinaryFormat/Dwarf.h | 14 +-- llvm/lib/BinaryFormat/Dwarf.cpp | 6 ++ .../CodeGen/AsmPrinter/DwarfExpression.cpp | 32 +++++++ llvm/lib/IR/DebugInfoMetadata.cpp | 4 + .../DebugInfo/X86/DW_OP_LLVM_extract_bits.ll | 92 +++++++++++++++++++ 6 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 llvm/test/DebugInfo/X86/DW_OP_LLVM_extract_bits.ll diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 9d7ade8eb523..b1e0cee25c98 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -6308,6 +6308,15 @@ The current supported opcode vocabulary is limited: (``16`` and ``DW_ATE_signed`` here, respectively) to which the top of the expression stack is to be converted. Maps into a ``DW_OP_convert`` operation that references a base type constructed from the supplied values. +- ``DW_OP_LLVM_extract_bits_sext, 16, 8,`` specifies the offset and size + (``16`` and ``8`` here, respectively) of bits that are to be extracted and + sign-extended from the value at the top of the expression stack. If the top of + the expression stack is a memory location then these bits are extracted from + the value pointed to by that memory location. Maps into a ``DW_OP_shl`` + followed by ``DW_OP_shra``. +- ``DW_OP_LLVM_extract_bits_zext`` behaves similarly to + ``DW_OP_LLVM_extract_bits_sext``, but zero-extends instead of sign-extending. + Maps into a ``DW_OP_shl`` followed by ``DW_OP_shr``. - ``DW_OP_LLVM_tag_offset, tag_offset`` specifies that a memory tag should be optionally applied to the pointer. The memory tag is derived from the given tag offset in an implementation-defined manner. diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.h b/llvm/include/llvm/BinaryFormat/Dwarf.h index 74c4d6ff3a71..607f3eb9d4c2 100644 --- a/llvm/include/llvm/BinaryFormat/Dwarf.h +++ b/llvm/include/llvm/BinaryFormat/Dwarf.h @@ -138,12 +138,14 @@ enum LocationAtom { #include "llvm/BinaryFormat/Dwarf.def" DW_OP_lo_user = 0xe0, DW_OP_hi_user = 0xff, - DW_OP_LLVM_fragment = 0x1000, ///< Only used in LLVM metadata. - DW_OP_LLVM_convert = 0x1001, ///< Only used in LLVM metadata. - DW_OP_LLVM_tag_offset = 0x1002, ///< Only used in LLVM metadata. - DW_OP_LLVM_entry_value = 0x1003, ///< Only used in LLVM metadata. - DW_OP_LLVM_implicit_pointer = 0x1004, ///< Only used in LLVM metadata. - DW_OP_LLVM_arg = 0x1005, ///< Only used in LLVM metadata. + DW_OP_LLVM_fragment = 0x1000, ///< Only used in LLVM metadata. + DW_OP_LLVM_convert = 0x1001, ///< Only used in LLVM metadata. + DW_OP_LLVM_tag_offset = 0x1002, ///< Only used in LLVM metadata. + DW_OP_LLVM_entry_value = 0x1003, ///< Only used in LLVM metadata. + DW_OP_LLVM_implicit_pointer = 0x1004, ///< Only used in LLVM metadata. + DW_OP_LLVM_arg = 0x1005, ///< Only used in LLVM metadata. + DW_OP_LLVM_extract_bits_sext = 0x1006, ///< Only used in LLVM metadata. + DW_OP_LLVM_extract_bits_zext = 0x1007, ///< Only used in LLVM metadata. }; enum LlvmUserLocationAtom { diff --git a/llvm/lib/BinaryFormat/Dwarf.cpp b/llvm/lib/BinaryFormat/Dwarf.cpp index 732426617268..0bf4f201dbe1 100644 --- a/llvm/lib/BinaryFormat/Dwarf.cpp +++ b/llvm/lib/BinaryFormat/Dwarf.cpp @@ -155,6 +155,10 @@ StringRef llvm::dwarf::OperationEncodingString(unsigned Encoding) { return "DW_OP_LLVM_implicit_pointer"; case DW_OP_LLVM_arg: return "DW_OP_LLVM_arg"; + case DW_OP_LLVM_extract_bits_sext: + return "DW_OP_LLVM_extract_bits_sext"; + case DW_OP_LLVM_extract_bits_zext: + return "DW_OP_LLVM_extract_bits_zext"; } } @@ -169,6 +173,8 @@ unsigned llvm::dwarf::getOperationEncoding(StringRef OperationEncodingString) { .Case("DW_OP_LLVM_entry_value", DW_OP_LLVM_entry_value) .Case("DW_OP_LLVM_implicit_pointer", DW_OP_LLVM_implicit_pointer) .Case("DW_OP_LLVM_arg", DW_OP_LLVM_arg) + .Case("DW_OP_LLVM_extract_bits_sext", DW_OP_LLVM_extract_bits_sext) + .Case("DW_OP_LLVM_extract_bits_zext", DW_OP_LLVM_extract_bits_zext) .Default(0); } diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp index a74d43897d45..cc96d3c481f7 100644 --- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp @@ -18,6 +18,7 @@ #include "llvm/CodeGen/Register.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/IR/DataLayout.h" +#include "llvm/MC/MCAsmInfo.h" #include "llvm/Support/ErrorHandling.h" #include @@ -546,6 +547,37 @@ bool DwarfExpression::addExpression( LocationKind = Unknown; return true; } + case dwarf::DW_OP_LLVM_extract_bits_sext: + case dwarf::DW_OP_LLVM_extract_bits_zext: { + unsigned SizeInBits = Op->getArg(1); + unsigned BitOffset = Op->getArg(0); + + // If we have a memory location then dereference to get the value + if (isMemoryLocation()) + emitOp(dwarf::DW_OP_deref); + + // Extract the bits by a shift left (to shift out the bits after what we + // want to extract) followed by shift right (to shift the bits to position + // 0 and also sign/zero extend). These operations are done in the DWARF + // "generic type" whose size is the size of a pointer. + unsigned PtrSizeInBytes = CU.getAsmPrinter()->MAI->getCodePointerSize(); + unsigned LeftShift = PtrSizeInBytes * 8 - (SizeInBits + BitOffset); + unsigned RightShift = LeftShift + BitOffset; + if (LeftShift) { + emitOp(dwarf::DW_OP_constu); + emitUnsigned(LeftShift); + emitOp(dwarf::DW_OP_shl); + } + emitOp(dwarf::DW_OP_constu); + emitUnsigned(RightShift); + emitOp(OpNum == dwarf::DW_OP_LLVM_extract_bits_sext ? dwarf::DW_OP_shra + : dwarf::DW_OP_shr); + + // The value is now at the top of the stack, so set the location to + // implicit so that we get a stack_value at the end. + LocationKind = Implicit; + break; + } case dwarf::DW_OP_plus_uconst: assert(!isRegisterLocation()); emitOp(dwarf::DW_OP_plus_uconst); diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp index 9bd1d7880c9f..2b45932093f0 100644 --- a/llvm/lib/IR/DebugInfoMetadata.cpp +++ b/llvm/lib/IR/DebugInfoMetadata.cpp @@ -1404,6 +1404,8 @@ unsigned DIExpression::ExprOperand::getSize() const { switch (Op) { case dwarf::DW_OP_LLVM_convert: case dwarf::DW_OP_LLVM_fragment: + case dwarf::DW_OP_LLVM_extract_bits_sext: + case dwarf::DW_OP_LLVM_extract_bits_zext: case dwarf::DW_OP_bregx: return 3; case dwarf::DW_OP_constu: @@ -1474,6 +1476,8 @@ bool DIExpression::isValid() const { case dwarf::DW_OP_LLVM_convert: case dwarf::DW_OP_LLVM_arg: case dwarf::DW_OP_LLVM_tag_offset: + case dwarf::DW_OP_LLVM_extract_bits_sext: + case dwarf::DW_OP_LLVM_extract_bits_zext: case dwarf::DW_OP_constu: case dwarf::DW_OP_plus_uconst: case dwarf::DW_OP_plus: diff --git a/llvm/test/DebugInfo/X86/DW_OP_LLVM_extract_bits.ll b/llvm/test/DebugInfo/X86/DW_OP_LLVM_extract_bits.ll new file mode 100644 index 000000000000..18fdfa579b9f --- /dev/null +++ b/llvm/test/DebugInfo/X86/DW_OP_LLVM_extract_bits.ll @@ -0,0 +1,92 @@ +; RUN: llc -mtriple=x86_64-unknown-linux-gnu %s -o %t -filetype=obj +; RUN: llvm-dwarfdump --debug-info %t | FileCheck %s + +%struct.struct_t = type { i8 } + +@g = dso_local global %struct.struct_t zeroinitializer, align 1, !dbg !0 + +; CHECK-LABEL: DW_TAG_subprogram +; CHECK: DW_AT_name ("test1") +; CHECK: DW_TAG_variable +; CHECK: DW_AT_location (DW_OP_fbreg -1, DW_OP_deref, DW_OP_constu 0x3d, DW_OP_shl, DW_OP_constu 0x3d, DW_OP_shr, DW_OP_stack_value) +; CHECK: DW_AT_name ("x") +; CHECK: DW_TAG_variable +; CHECK: DW_AT_location (DW_OP_fbreg -1, DW_OP_deref, DW_OP_constu 0x39, DW_OP_shl, DW_OP_constu 0x3c, DW_OP_shra, DW_OP_stack_value) +; CHECK: DW_AT_name ("y") + +define i32 @test1() !dbg !13 { +entry: + %0 = alloca %struct.struct_t, align 1 + tail call void @llvm.dbg.declare(metadata ptr %0, metadata !16, metadata !DIExpression(DW_OP_LLVM_extract_bits_zext, 0, 3)), !dbg !17 + tail call void @llvm.dbg.declare(metadata ptr %0, metadata !18, metadata !DIExpression(DW_OP_LLVM_extract_bits_sext, 3, 4)), !dbg !17 + ret i32 0, !dbg !17 +} + +; CHECK-LABEL: DW_TAG_subprogram +; CHECK: DW_AT_name ("test2") +; CHECK: DW_TAG_variable +; CHECK: DW_AT_location (DW_OP_breg0 {{R[^+]+}}+0, DW_OP_constu 0xff, DW_OP_and, DW_OP_constu 0x3d, DW_OP_shl, DW_OP_constu 0x3d, DW_OP_shr, DW_OP_stack_value) +; CHECK: DW_AT_name ("x") +; CHECK: DW_TAG_variable +; CHECK: DW_AT_location (DW_OP_breg0 {{R[^+]+}}+0, DW_OP_constu 0xff, DW_OP_and, DW_OP_constu 0x39, DW_OP_shl, DW_OP_constu 0x3c, DW_OP_shra, DW_OP_stack_value) +; CHECK: DW_AT_name ("y") + +define i8 @test2() !dbg !20 { +entry: + %0 = load i8, ptr @g, align 1 + tail call void @llvm.dbg.value(metadata i8 %0, metadata !21, metadata !DIExpression(DW_OP_LLVM_extract_bits_zext, 0, 3)), !dbg !22 + tail call void @llvm.dbg.value(metadata i8 %0, metadata !23, metadata !DIExpression(DW_OP_LLVM_extract_bits_sext, 3, 4)), !dbg !22 + ret i8 %0, !dbg !22 +} + +; CHECK-LABEL: DW_TAG_subprogram +; CHECK: DW_AT_name ("test3") +; CHECK: DW_TAG_variable +; CHECK: DW_AT_location (DW_OP_breg0 {{R[^+]+}}+0, DW_OP_constu 0x3f, DW_OP_shr, DW_OP_stack_value) +; CHECK: DW_AT_name ("x") +; CHECK: DW_TAG_variable +; CHECK: DW_AT_location (DW_OP_breg0 {{R[^+]+}}+0, DW_OP_constu 0x3f, DW_OP_shra, DW_OP_stack_value) +; CHECK: DW_AT_name ("y") + +define i64 @test3(ptr %p) !dbg !24 { +entry: + %0 = load i64, ptr %p, align 8 + tail call void @llvm.dbg.value(metadata i64 %0, metadata !25, metadata !DIExpression(DW_OP_LLVM_extract_bits_zext, 63, 1)), !dbg !26 + tail call void @llvm.dbg.value(metadata i64 %0, metadata !27, metadata !DIExpression(DW_OP_LLVM_extract_bits_sext, 63, 1)), !dbg !26 + ret i64 %0, !dbg !26 +} + +declare void @llvm.dbg.declare(metadata, metadata, metadata) +declare void @llvm.dbg.value(metadata, metadata, metadata) + +!llvm.dbg.cu = !{!2} +!llvm.module.flags = !{!11, !12} + +!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) +!1 = distinct !DIGlobalVariable(name: "g", scope: !2, file: !3, type: !5, isLocal: false, isDefinition: true) +!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !3, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, globals: !4, splitDebugInlining: false, nameTableKind: None) +!3 = !DIFile(filename: "DW_OP_bit_piece.cpp", directory: "./") +!4 = !{!0} +!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "struct_t", file: !3, baseType: !6) +!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, file: !3, size: 8, flags: DIFlagTypePassByValue, elements: !7, identifier: "_ZTS8struct_t") +!7 = !{!8, !10} +!8 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !6, file: !3, baseType: !9, size: 3, flags: DIFlagBitField, extraData: i64 0) +!9 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned) +!10 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !6, file: !3, baseType: !9, size: 4, offset: 3, flags: DIFlagBitField, extraData: i64 0) +!11 = !{i32 7, !"Dwarf Version", i32 5} +!12 = !{i32 2, !"Debug Info Version", i32 3} +!13 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: !3, file: !3, type: !14, spFlags: DISPFlagDefinition, unit: !2) +!14 = !DISubroutineType(types: !15) +!15 = !{!9} +!16 = !DILocalVariable(name: "x", scope: !13, file: !3, type: !9) +!17 = !DILocation(line: 0, scope: !13) +!18 = !DILocalVariable(name: "y", scope: !13, file: !3, type: !19) +!19 = !DIBasicType(name: "signed int", size: 32, encoding: DW_ATE_signed) +!20 = distinct !DISubprogram(name: "test2", linkageName: "test2", scope: !3, file: !3, type: !14, spFlags: DISPFlagDefinition, unit: !2) +!21 = !DILocalVariable(name: "x", scope: !20, file: !3, type: !9) +!22 = !DILocation(line: 0, scope: !20) +!23 = !DILocalVariable(name: "y", scope: !20, file: !3, type: !19) +!24 = distinct !DISubprogram(name: "test3", linkageName: "test3", scope: !3, file: !3, type: !14, spFlags: DISPFlagDefinition, unit: !2) +!25 = !DILocalVariable(name: "x", scope: !24, file: !3, type: !9) +!26 = !DILocation(line: 0, scope: !24) +!27 = !DILocalVariable(name: "y", scope: !24, file: !3, type: !19) -- GitLab From 192cd685129d6f42e8b191e12bddf74ade48c270 Mon Sep 17 00:00:00 2001 From: Fotis Kounelis Date: Fri, 7 Jun 2024 10:46:01 +0100 Subject: [PATCH 219/462] Add checks before hoisting out in loop pipelining (#90872) Currently, during a loop pipelining transformation, operations may be hoisted out without any checks on the loop bounds, which leads to incorrect transformations and unexpected behaviour. The following [issue ](https://github.com/llvm/llvm-project/issues/90870) describes the problem more extensively, including an example. The proposed fix adds some check in the loop bounds before and applies the maximum hoisting. --- .../SCF/TransformOps/SCFTransformOps.cpp | 11 +++- mlir/test/Dialect/SCF/transform-ops.mlir | 57 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp index 30699ecdde0a..15b8aca5e267 100644 --- a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp +++ b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp @@ -261,6 +261,8 @@ loopScheduling(scf::ForOp forOp, return 1; }; + std::optional ubConstant = getConstantIntValue(forOp.getUpperBound()); + std::optional lbConstant = getConstantIntValue(forOp.getLowerBound()); DenseMap opCycles; std::map> wrappedSchedule; for (Operation &op : forOp.getBody()->getOperations()) { @@ -271,7 +273,14 @@ loopScheduling(scf::ForOp forOp, Operation *def = operand.getDefiningOp(); if (!def) continue; - earlyCycle = std::max(earlyCycle, opCycles[def] + getLatency(def)); + if (ubConstant && lbConstant) { + unsigned ubInt = ubConstant.value(); + unsigned lbInt = lbConstant.value(); + auto minLatency = std::min(ubInt - lbInt - 1, getLatency(def)); + earlyCycle = std::max(earlyCycle, opCycles[def] + minLatency); + } else { + earlyCycle = std::max(earlyCycle, opCycles[def] + getLatency(def)); + } } opCycles[&op] = earlyCycle; wrappedSchedule[earlyCycle % iterationInterval].push_back(&op); diff --git a/mlir/test/Dialect/SCF/transform-ops.mlir b/mlir/test/Dialect/SCF/transform-ops.mlir index a4daa86583c3..2d6b48fd3e57 100644 --- a/mlir/test/Dialect/SCF/transform-ops.mlir +++ b/mlir/test/Dialect/SCF/transform-ops.mlir @@ -300,3 +300,60 @@ module attributes {transform.with_named_sequence} { transform.yield } } + + +// ----- + +// CHECK-LABEL: func.func @loop_pipeline +func.func @loop_pipeline(%arg0: memref<4x16xf32>, %arg1: vector<16xf32>) -> vector<16xf32> { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %cst = arith.constant 0.000000e+00 : f32 + %c3 = arith.constant 3 : index + // CHECK: vector.transfer_read + // CHECK: vector.transfer_read + // CHECK: vector.transfer_read + // CHECK: arith.addf + // CHECK: arith.addf + // CHECK: arith.addf + %0 = scf.for %arg2 = %c0 to %c3 step %c1 iter_args(%arg3 = %arg1) -> (vector<16xf32>) { + %1 = vector.transfer_read %arg0[%arg2, %c0], %cst {in_bounds = [true]} : memref<4x16xf32>, vector<16xf32> + %2 = arith.addf %1, %arg3 : vector<16xf32> + scf.yield %2 : vector<16xf32> + } + return %0 : vector<16xf32> +} +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["scf.for"]} in %arg1 : (!transform.any_op) -> !transform.op<"scf.for"> + %1 = transform.loop.pipeline %0 {iteration_interval = 1 : i64, read_latency = 5 : i64, scheduling_type = "full-loops"} : (!transform.op<"scf.for">) -> !transform.any_op + transform.yield + } +} + + +// ----- + +// CHECK-LABEL: func.func @loop_pipeline_lb_gt_0 +func.func @loop_pipeline_lb_gt_0(%arg0: memref<4x16xf32>, %arg1: vector<16xf32>) -> vector<16xf32> { + %c1 = arith.constant 1 : index + %cst = arith.constant 0.000000e+00 : f32 + %c3 = arith.constant 3 : index + // CHECK: vector.transfer_read + // CHECK: vector.transfer_read + // CHECK: arith.addf + // CHECK: arith.addf + %0 = scf.for %arg2 = %c1 to %c3 step %c1 iter_args(%arg3 = %arg1) -> (vector<16xf32>) { + %1 = vector.transfer_read %arg0[%arg2, %c1], %cst {in_bounds = [true]} : memref<4x16xf32>, vector<16xf32> + %2 = arith.addf %1, %arg3 : vector<16xf32> + scf.yield %2 : vector<16xf32> + } + return %0 : vector<16xf32> +} +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["scf.for"]} in %arg1 : (!transform.any_op) -> !transform.op<"scf.for"> + %1 = transform.loop.pipeline %0 {iteration_interval = 1 : i64, read_latency = 5 : i64, scheduling_type = "full-loops"} : (!transform.op<"scf.for">) -> !transform.any_op + transform.yield + } +} -- GitLab From 5d6acf8196a44225991ab2fb6dfc9cc72296b348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Fri, 7 Jun 2024 10:42:07 +0200 Subject: [PATCH 220/462] [clang][Interp][NFC] Properly assign block pointer Pointee --- clang/lib/AST/Interp/Pointer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp index e3d21f93f114..85857d4ee1c8 100644 --- a/clang/lib/AST/Interp/Pointer.cpp +++ b/clang/lib/AST/Interp/Pointer.cpp @@ -80,6 +80,7 @@ void Pointer::operator=(const Pointer &P) { if (P.isBlockPointer()) { PointeeStorage.BS = P.PointeeStorage.BS; + PointeeStorage.BS.Pointee = P.PointeeStorage.BS.Pointee; if (PointeeStorage.BS.Pointee) PointeeStorage.BS.Pointee->addPointer(this); @@ -107,6 +108,7 @@ void Pointer::operator=(Pointer &&P) { if (P.isBlockPointer()) { PointeeStorage.BS = P.PointeeStorage.BS; + PointeeStorage.BS.Pointee = P.PointeeStorage.BS.Pointee; if (PointeeStorage.BS.Pointee) PointeeStorage.BS.Pointee->addPointer(this); -- GitLab From 3a31eaeac8482fa5e242ee00cd4e77b203db539e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Fri, 7 Jun 2024 11:42:51 +0200 Subject: [PATCH 221/462] [clang][Interp] Fix refers_to_enclosing_variable_or_capture DREs They do not count into lambda captures, so visit them lazily. --- clang/lib/AST/Interp/ByteCodeExprGen.cpp | 7 +++++++ clang/test/AST/Interp/lambda.cpp | 3 +++ 2 files changed, 10 insertions(+) diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index 1d0c36d0bf09..8dc71b2527f3 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -3894,6 +3894,13 @@ bool ByteCodeExprGen::visitDeclRef(const ValueDecl *D, const Expr *E) { if (IsPtr) return this->emitGetThisFieldPtr(Offset, E); return this->emitGetPtrThisField(Offset, E); + } else if (const auto *DRE = dyn_cast(E); + DRE && DRE->refersToEnclosingVariableOrCapture() && + isa(D)) { + if (!this->visitVarDecl(cast(D))) + return false; + // Retry. + return this->visitDeclRef(D, E); } // Try to lazily visit (or emit dummy pointers for) declarations diff --git a/clang/test/AST/Interp/lambda.cpp b/clang/test/AST/Interp/lambda.cpp index 77e035ce2547..71e7077550b2 100644 --- a/clang/test/AST/Interp/lambda.cpp +++ b/clang/test/AST/Interp/lambda.cpp @@ -264,3 +264,6 @@ namespace CaptureDefaults { }; static_assert(f2() == 3, ""); } + +constexpr auto t4 = ([x=42]() consteval { return x; }()); +static_assert(t4 == 42, ""); -- GitLab From 1934c1aa3613fe2ded87ca4cd739694378e92601 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 7 Jun 2024 11:40:05 +0200 Subject: [PATCH 222/462] [SimplifyCFG] Remove bogus UTC line from test (NFC) The check lines in this test were clearly not generated by UTC. --- llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll b/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll index 463a85cfd583..6def8f4eeb08 100644 --- a/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll +++ b/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll @@ -1,4 +1,3 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -S -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -switch-to-lookup -mtriple=arm -relocation-model=static < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE ; RUN: opt -S -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -switch-to-lookup -mtriple=arm -relocation-model=pic < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE ; RUN: opt -S -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -switch-to-lookup -mtriple=arm -relocation-model=ropi < %s | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE -- GitLab From 8719cb88e3443bf717391b5020beab61238a372c Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 7 Jun 2024 11:50:43 +0200 Subject: [PATCH 223/462] [SimplifyCFG] Regenerate switch to lookup tests (NFC) Regenerate these with --check-globals. The manual global CHECKS get dropped during regeneration otherwise. Annoyingly UTC insists on putting the globals directly before the first function, so the first comment is a bit out of place now. --- .../RISCV/switch_to_lookup_table-rv32.ll | 28 +++-- .../RISCV/switch_to_lookup_table-rv64.ll | 28 +++-- .../SimplifyCFG/X86/switch_to_lookup_table.ll | 112 +++++++++--------- 3 files changed, 89 insertions(+), 79 deletions(-) diff --git a/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv32.ll b/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv32.ll index 6858746e8724..18a3c9a6f754 100644 --- a/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv32.ll +++ b/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv32.ll @@ -1,26 +1,25 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals ; RUN: opt < %s -passes=simplifycfg -switch-to-lookup=true -keep-loops=false -S -mattr=+f | FileCheck %s ; RUN: opt < %s -passes='simplifycfg' -S -mattr=+f | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-n32-S128" target triple = "riscv32-unknown-elf" -; The table for @f -; CHECK: @switch.table.f = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1], align 4 - -; The char table for char -; CHECK: @switch.table.char = private unnamed_addr constant [9 x i8] c"7{\00\FF\1B>\01!T", align 1 - -; The float table for @h -; CHECK: @switch.table.h = private unnamed_addr constant [4 x float] [float 0x40091EB860000000, float 0x3FF3BE76C0000000, float 0x4012449BA0000000, float 0x4001AE1480000000], align 4 - -; The table for @foostring -; CHECK: @switch.table.foostring = private unnamed_addr constant [4 x ptr] [ptr @.str, ptr @.str1, ptr @.str2, ptr @.str3], align 4 - ; A simple int-to-int selection switch. ; It is dense enough to be replaced by table lookup. ; The result is directly by a ret from an otherwise empty bb, ; so we return early, directly from the lookup bb. +;. +; CHECK: @.str = private unnamed_addr constant [4 x i8] c"foo\00", align 1 +; CHECK: @.str1 = private unnamed_addr constant [4 x i8] c"bar\00", align 1 +; CHECK: @.str2 = private unnamed_addr constant [4 x i8] c"baz\00", align 1 +; CHECK: @.str3 = private unnamed_addr constant [4 x i8] c"qux\00", align 1 +; CHECK: @.str4 = private unnamed_addr constant [6 x i8] c"error\00", align 1 +; CHECK: @switch.table.f = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1], align 4 +; CHECK: @switch.table.char = private unnamed_addr constant [9 x i8] c"7{\00\FF\1B>\01!T", align 1 +; CHECK: @switch.table.h = private unnamed_addr constant [4 x float] [float 0x40091EB860000000, float 0x3FF3BE76C0000000, float 0x4012449BA0000000, float 0x4001AE1480000000], align 4 +; CHECK: @switch.table.foostring = private unnamed_addr constant [4 x ptr] [ptr @.str, ptr @.str1, ptr @.str2, ptr @.str3], align 4 +;. define i32 @f(i32 %c) { ; CHECK-LABEL: @f( ; CHECK-NEXT: entry: @@ -189,3 +188,6 @@ return: ret ptr %retval.0 } +;. +; CHECK: attributes #[[ATTR0:[0-9]+]] = { "target-features"="+f" } +;. diff --git a/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll b/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll index f680cdbe6e7c..6c9450bb8998 100644 --- a/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll +++ b/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll @@ -1,26 +1,25 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals ; RUN: opt < %s -passes=simplifycfg -switch-to-lookup=true -keep-loops=false -S -mattr=+f | FileCheck %s ; RUN: opt < %s -passes='simplifycfg' -S -mattr=+f | FileCheck %s target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128" target triple = "riscv64-unknown-elf" -; The table for @f -; CHECK: @switch.table.f = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1], align 4 - -; The char table for char -; CHECK: @switch.table.char = private unnamed_addr constant [9 x i8] c"7{\00\FF\1B>\01!T", align 1 - -; The float table for @h -; CHECK: @switch.table.h = private unnamed_addr constant [4 x float] [float 0x40091EB860000000, float 0x3FF3BE76C0000000, float 0x4012449BA0000000, float 0x4001AE1480000000], align 4 - -; The table for @foostring -; CHECK: @switch.table.foostring = private unnamed_addr constant [4 x ptr] [ptr @.str, ptr @.str1, ptr @.str2, ptr @.str3], align 8 - ; A simple int-to-int selection switch. ; It is dense enough to be replaced by table lookup. ; The result is directly by a ret from an otherwise empty bb, ; so we return early, directly from the lookup bb. +;. +; CHECK: @.str = private unnamed_addr constant [4 x i8] c"foo\00", align 1 +; CHECK: @.str1 = private unnamed_addr constant [4 x i8] c"bar\00", align 1 +; CHECK: @.str2 = private unnamed_addr constant [4 x i8] c"baz\00", align 1 +; CHECK: @.str3 = private unnamed_addr constant [4 x i8] c"qux\00", align 1 +; CHECK: @.str4 = private unnamed_addr constant [6 x i8] c"error\00", align 1 +; CHECK: @switch.table.f = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1], align 4 +; CHECK: @switch.table.char = private unnamed_addr constant [9 x i8] c"7{\00\FF\1B>\01!T", align 1 +; CHECK: @switch.table.h = private unnamed_addr constant [4 x float] [float 0x40091EB860000000, float 0x3FF3BE76C0000000, float 0x4012449BA0000000, float 0x4001AE1480000000], align 4 +; CHECK: @switch.table.foostring = private unnamed_addr constant [4 x ptr] [ptr @.str, ptr @.str1, ptr @.str2, ptr @.str3], align 8 +;. define i32 @f(i32 %c) { ; CHECK-LABEL: @f( ; CHECK-NEXT: entry: @@ -189,3 +188,6 @@ return: ret ptr %retval.0 } +;. +; CHECK: attributes #[[ATTR0:[0-9]+]] = { "target-features"="+f" } +;. diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll index 9d6502072c16..d6450a2b4a34 100644 --- a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll +++ b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll @@ -1,38 +1,44 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals ; RUN: opt < %s -passes=simplifycfg -switch-to-lookup=true -keep-loops=false -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s ; RUN: opt < %s -passes='simplifycfg' -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" -; The table for @f -; CHECK: @switch.table.f = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1], align 4 +; A simple int-to-int selection switch. +; It is dense enough to be replaced by table lookup. +; The result is directly by a ret from an otherwise empty bb, +; so we return early, directly from the lookup bb. -; The char table for char +;. +; CHECK: @.str = private unnamed_addr constant [4 x i8] c"foo\00", align 1 +; CHECK: @.str1 = private unnamed_addr constant [4 x i8] c"bar\00", align 1 +; CHECK: @.str2 = private unnamed_addr constant [4 x i8] c"baz\00", align 1 +; CHECK: @.str3 = private unnamed_addr constant [4 x i8] c"qux\00", align 1 +; CHECK: @.str4 = private unnamed_addr constant [6 x i8] c"error\00", align 1 +; CHECK: @tls_a = thread_local global i32 0 +; CHECK: @tls_b = thread_local global i32 0 +; CHECK: @tls_c = thread_local global i32 0 +; CHECK: @tls_d = thread_local global i32 0 +; CHECK: @dllimport_a = external dllimport global [3 x i32] +; CHECK: @dllimport_b = external dllimport global [3 x i32] +; CHECK: @dllimport_c = external dllimport global [3 x i32] +; CHECK: @dllimport_d = external dllimport global [3 x i32] +; CHECK: @switch.table.f = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1], align 4 ; CHECK: @switch.table.char = private unnamed_addr constant [9 x i8] c"7{\00\FF\1B>\01!T", align 1 - -; The float table for @h ; CHECK: @switch.table.h = private unnamed_addr constant [4 x float] [float 0x40091EB860000000, float 0x3FF3BE76C0000000, float 0x4012449BA0000000, float 0x4001AE1480000000], align 4 - -; The table for @foostring ; CHECK: @switch.table.foostring = private unnamed_addr constant [4 x ptr] [ptr @.str, ptr @.str1, ptr @.str2, ptr @.str3], align 8 - -; The table for @earlyreturncrash ; CHECK: @switch.table.earlyreturncrash = private unnamed_addr constant [4 x i32] [i32 42, i32 9, i32 88, i32 5], align 4 - -; The table for @large -; CHECK: @switch.table.large = private unnamed_addr constant [199 x i32] [i32 1, i32 4, i32 9, - -; The table for @cprop +; CHECK: @switch.table.earlyreturncrash.1 = private unnamed_addr constant [4 x i32] [i32 3, i32 4, i32 1, i32 5], align 4 +; CHECK: @switch.table.large = private unnamed_addr constant [199 x i32] [i32 1, i32 4, i32 9, i32 16, i32 25, i32 36, i32 49, i32 64, i32 81, i32 100, i32 121, i32 144, i32 169, i32 196, i32 225, i32 256, i32 289, i32 342, i32 361, i32 400, i32 441, i32 484, i32 529, i32 576, i32 625, i32 676, i32 729, i32 784, i32 841, i32 900, i32 961, i32 1024, i32 1089, i32 1156, i32 1260, i32 1296, i32 1369, i32 1444, i32 1521, i32 1600, i32 1681, i32 1764, i32 1849, i32 1980, i32 2025, i32 2116, i32 2209, i32 2304, i32 2401, i32 2500, i32 2601, i32 2704, i32 2809, i32 2970, i32 3025, i32 3136, i32 3249, i32 3364, i32 3481, i32 3600, i32 3721, i32 3844, i32 3969, i32 4096, i32 4225, i32 4356, i32 4489, i32 4624, i32 4761, i32 4900, i32 5112, i32 5184, i32 5329, i32 5476, i32 5625, i32 5776, i32 5929, i32 6084, i32 6241, i32 6400, i32 6561, i32 6724, i32 6889, i32 7056, i32 7225, i32 7396, i32 7569, i32 7744, i32 7921, i32 8100, i32 8281, i32 8464, i32 8649, i32 8836, i32 9025, i32 9216, i32 9409, i32 9604, i32 9801, i32 10000, i32 10201, i32 10404, i32 10609, i32 10816, i32 11025, i32 11236, i32 11449, i32 11664, i32 11881, i32 12100, i32 12321, i32 12544, i32 12769, i32 12996, i32 13225, i32 13456, i32 13689, i32 13924, i32 14161, i32 14400, i32 14641, i32 14884, i32 15129, i32 15376, i32 15625, i32 15876, i32 16129, i32 16384, i32 16641, i32 16900, i32 17161, i32 17424, i32 17689, i32 17956, i32 18225, i32 18496, i32 18769, i32 19044, i32 19321, i32 19600, i32 19881, i32 20164, i32 20449, i32 20736, i32 21025, i32 21316, i32 21609, i32 21904, i32 22201, i32 22500, i32 22801, i32 23104, i32 23409, i32 23716, i32 24025, i32 24336, i32 24649, i32 24964, i32 25281, i32 25600, i32 25921, i32 26244, i32 26569, i32 26896, i32 27225, i32 27556, i32 27889, i32 28224, i32 28561, i32 28900, i32 29241, i32 29584, i32 29929, i32 30276, i32 30625, i32 30976, i32 31329, i32 31684, i32 32041, i32 32400, i32 32761, i32 33124, i32 33489, i32 33856, i32 34225, i32 34596, i32 34969, i32 35344, i32 35721, i32 36100, i32 36481, i32 36864, i32 37249, i32 37636, i32 38025, i32 38416, i32 38809, i32 39204, i32 39601], align 4 ; CHECK: @switch.table.cprop = private unnamed_addr constant [7 x i32] [i32 5, i32 42, i32 126, i32 -452, i32 128, i32 6, i32 7], align 4 - -; The table for @unreachable_case ; CHECK: @switch.table.unreachable_case = private unnamed_addr constant [9 x i32] [i32 0, i32 0, i32 0, i32 2, i32 -1, i32 1, i32 1, i32 1, i32 1], align 4 - -; A simple int-to-int selection switch. -; It is dense enough to be replaced by table lookup. -; The result is directly by a ret from an otherwise empty bb, -; so we return early, directly from the lookup bb. - +; CHECK: @switch.table.unreachable_default = private unnamed_addr constant [4 x i32] [i32 42, i32 52, i32 1, i32 2], align 4 +; CHECK: @switch.table.nodefaultnoholes = private unnamed_addr constant [4 x i32] [i32 55, i32 123, i32 0, i32 -1], align 4 +; CHECK: @switch.table.nodefaultwithholes = private unnamed_addr constant [6 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 55, i32 -1], align 4 +; CHECK: @switch.table.threecases = private unnamed_addr constant [3 x i32] [i32 10, i32 7, i32 5], align 4 +; CHECK: @switch.table.covered_switch_with_bit_tests = private unnamed_addr constant [8 x i32] [i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 1, i32 1], align 4 +; CHECK: @switch.table.signed_overflow1 = private unnamed_addr constant [4 x i32] [i32 3333, i32 4444, i32 1111, i32 2222], align 4 +;. define i32 @f(i32 %c) { ; CHECK-LABEL: @f( ; CHECK-NEXT: entry: @@ -308,10 +314,10 @@ define i32 @overflow(i32 %type) { ; CHECK-LABEL: @overflow( ; CHECK-NEXT: entry: ; CHECK-NEXT: switch i32 [[TYPE:%.*]], label [[IF_END:%.*]] [ -; CHECK-NEXT: i32 3, label [[SW_BB3:%.*]] -; CHECK-NEXT: i32 -2147483645, label [[SW_BB3]] -; CHECK-NEXT: i32 1, label [[SW_BB1:%.*]] -; CHECK-NEXT: i32 2, label [[SW_BB2:%.*]] +; CHECK-NEXT: i32 3, label [[SW_BB3:%.*]] +; CHECK-NEXT: i32 -2147483645, label [[SW_BB3]] +; CHECK-NEXT: i32 1, label [[SW_BB1:%.*]] +; CHECK-NEXT: i32 2, label [[SW_BB2:%.*]] ; CHECK-NEXT: ] ; CHECK: sw.bb1: ; CHECK-NEXT: br label [[IF_END]] @@ -931,11 +937,11 @@ define i96 @illegaltype(i32 %c) { ; CHECK-LABEL: @illegaltype( ; CHECK-NEXT: entry: ; CHECK-NEXT: switch i32 [[C:%.*]], label [[SW_DEFAULT:%.*]] [ -; CHECK-NEXT: i32 42, label [[RETURN:%.*]] -; CHECK-NEXT: i32 43, label [[SW_BB1:%.*]] -; CHECK-NEXT: i32 44, label [[SW_BB2:%.*]] -; CHECK-NEXT: i32 45, label [[SW_BB3:%.*]] -; CHECK-NEXT: i32 46, label [[SW_BB4:%.*]] +; CHECK-NEXT: i32 42, label [[RETURN:%.*]] +; CHECK-NEXT: i32 43, label [[SW_BB1:%.*]] +; CHECK-NEXT: i32 44, label [[SW_BB2:%.*]] +; CHECK-NEXT: i32 45, label [[SW_BB3:%.*]] +; CHECK-NEXT: i32 46, label [[SW_BB4:%.*]] ; CHECK-NEXT: ] ; CHECK: sw.bb1: ; CHECK-NEXT: br label [[RETURN]] @@ -1050,9 +1056,9 @@ define i32 @threecasesholes(i32 %c) { ; CHECK-LABEL: @threecasesholes( ; CHECK-NEXT: entry: ; CHECK-NEXT: switch i32 [[C:%.*]], label [[SW_DEFAULT:%.*]] [ -; CHECK-NEXT: i32 0, label [[RETURN:%.*]] -; CHECK-NEXT: i32 1, label [[SW_BB1:%.*]] -; CHECK-NEXT: i32 3, label [[SW_BB2:%.*]] +; CHECK-NEXT: i32 0, label [[RETURN:%.*]] +; CHECK-NEXT: i32 1, label [[SW_BB1:%.*]] +; CHECK-NEXT: i32 3, label [[SW_BB2:%.*]] ; CHECK-NEXT: ] ; CHECK: sw.bb1: ; CHECK-NEXT: br label [[RETURN]] @@ -1142,9 +1148,9 @@ define ptr @tls(i32 %x) { ; CHECK-LABEL: @tls( ; CHECK-NEXT: entry: ; CHECK-NEXT: switch i32 [[X:%.*]], label [[SW_DEFAULT:%.*]] [ -; CHECK-NEXT: i32 0, label [[RETURN:%.*]] -; CHECK-NEXT: i32 1, label [[SW_BB1:%.*]] -; CHECK-NEXT: i32 2, label [[SW_BB2:%.*]] +; CHECK-NEXT: i32 0, label [[RETURN:%.*]] +; CHECK-NEXT: i32 1, label [[SW_BB1:%.*]] +; CHECK-NEXT: i32 2, label [[SW_BB2:%.*]] ; CHECK-NEXT: ] ; CHECK: sw.bb1: ; CHECK-NEXT: br label [[RETURN]] @@ -1182,9 +1188,9 @@ define ptr @dllimport(i32 %x) { ; CHECK-LABEL: @dllimport( ; CHECK-NEXT: entry: ; CHECK-NEXT: switch i32 [[X:%.*]], label [[SW_DEFAULT:%.*]] [ -; CHECK-NEXT: i32 0, label [[RETURN:%.*]] -; CHECK-NEXT: i32 1, label [[SW_BB1:%.*]] -; CHECK-NEXT: i32 2, label [[SW_BB2:%.*]] +; CHECK-NEXT: i32 0, label [[RETURN:%.*]] +; CHECK-NEXT: i32 1, label [[SW_BB1:%.*]] +; CHECK-NEXT: i32 2, label [[SW_BB2:%.*]] ; CHECK-NEXT: ] ; CHECK: sw.bb1: ; CHECK-NEXT: br label [[RETURN]] @@ -1476,11 +1482,11 @@ define void @pr20210(i8 %x, i1 %y) { ; CHECK-NEXT: br i1 [[Y:%.*]], label [[SW:%.*]], label [[INTERMEDIATE:%.*]] ; CHECK: sw: ; CHECK-NEXT: switch i8 [[X:%.*]], label [[END:%.*]] [ -; CHECK-NEXT: i8 7, label [[INTERMEDIATE]] -; CHECK-NEXT: i8 3, label [[INTERMEDIATE]] -; CHECK-NEXT: i8 2, label [[INTERMEDIATE]] -; CHECK-NEXT: i8 1, label [[INTERMEDIATE]] -; CHECK-NEXT: i8 0, label [[INTERMEDIATE]] +; CHECK-NEXT: i8 7, label [[INTERMEDIATE]] +; CHECK-NEXT: i8 3, label [[INTERMEDIATE]] +; CHECK-NEXT: i8 2, label [[INTERMEDIATE]] +; CHECK-NEXT: i8 1, label [[INTERMEDIATE]] +; CHECK-NEXT: i8 0, label [[INTERMEDIATE]] ; CHECK-NEXT: ] ; CHECK: intermediate: ; CHECK-NEXT: [[Z:%.*]] = zext i8 [[X]] to i32 @@ -1612,15 +1618,15 @@ define void @wineh_test(i64 %val) personality ptr @__CxxFrameHandler3 { ; CHECK-LABEL: @wineh_test( ; CHECK-NEXT: entry: ; CHECK-NEXT: invoke void @throw(i1 false) -; CHECK-NEXT: to label [[UNREACHABLE:%.*]] unwind label [[CLEANUP1:%.*]] +; CHECK-NEXT: to label [[UNREACHABLE:%.*]] unwind label [[CLEANUP1:%.*]] ; CHECK: unreachable: ; CHECK-NEXT: unreachable ; CHECK: cleanup1: ; CHECK-NEXT: [[CLEANUPPAD1:%.*]] = cleanuppad within none [] ; CHECK-NEXT: switch i64 [[VAL:%.*]], label [[CLEANUPDONE2:%.*]] [ -; CHECK-NEXT: i64 0, label [[CLEANUPDONE1:%.*]] -; CHECK-NEXT: i64 1, label [[CLEANUPDONE1]] -; CHECK-NEXT: i64 6, label [[CLEANUPDONE1]] +; CHECK-NEXT: i64 0, label [[CLEANUPDONE1:%.*]] +; CHECK-NEXT: i64 1, label [[CLEANUPDONE1]] +; CHECK-NEXT: i64 6, label [[CLEANUPDONE1]] ; CHECK-NEXT: ] ; CHECK: cleanupdone1: ; CHECK-NEXT: cleanupret from [[CLEANUPPAD1]] unwind label [[CLEANUP2:%.*]] @@ -1733,9 +1739,9 @@ define i32 @signed_overflow2(i8 %n) { ; CHECK-NEXT: start: ; CHECK-NEXT: [[TRUNC:%.*]] = trunc i8 [[N:%.*]] to i2 ; CHECK-NEXT: switch i2 [[TRUNC]], label [[BB1:%.*]] [ -; CHECK-NEXT: i2 1, label [[BB6:%.*]] -; CHECK-NEXT: i2 -2, label [[BB4:%.*]] -; CHECK-NEXT: i2 -1, label [[BB5:%.*]] +; CHECK-NEXT: i2 1, label [[BB6:%.*]] +; CHECK-NEXT: i2 -2, label [[BB4:%.*]] +; CHECK-NEXT: i2 -1, label [[BB5:%.*]] ; CHECK-NEXT: ] ; CHECK: bb1: ; CHECK-NEXT: unreachable -- GitLab From b87a80d4ebca9e1c065f0d2762e500078c4badca Mon Sep 17 00:00:00 2001 From: Mubashar Ahmad Date: Fri, 7 Jun 2024 10:57:00 +0100 Subject: [PATCH 224/462] [mlir][vector] Add n-d deinterleave lowering (#94237) This patch implements the lowering for vector deinterleave for vector of n-dimensions. Process involves unrolling the n-d vector to a series of one-dimensional vectors. The deinterleave operation is then used on these vectors. From: ``` %0, %1 = vector.deinterleave %a : vector<2x8xi8> -> vector<2x4xi8> ``` To: ``` %cst = arith.constant dense<0> : vector<2x4xi32> %0 = vector.extract %arg0[0] : vector<8xi32> from vector<2x8xi32> %res1, %res2 = vector.deinterleave %0 : vector<8xi32> -> vector<4xi32> %1 = vector.insert %res1, %cst [0] : vector<4xi32> into vector<2x4xi32> %2 = vector.insert %res2, %cst [0] : vector<4xi32> into vector<2x4xi32> %3 = vector.extract %arg0[1] : vector<8xi32> from vector<2x8xi32> %res1_0, %res2_1 = vector.deinterleave %3 : vector<8xi32> -> vector<4xi32> %4 = vector.insert %res1_0, %1 [1] : vector<4xi32> into vector<2x4xi32> %5 = vector.insert %res2_1, %2 [1] : vector<4xi32> into vector<2x4xi32> ...etc. ``` --- .../Transforms/LowerVectorInterleave.cpp | 70 ++++++++++++++++++- .../VectorToLLVM/vector-to-llvm.mlir | 16 +++++ ...ctor-deinterleave-lowering-transforms.mlir | 68 ++++++++++++++++++ 3 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 mlir/test/Dialect/Vector/vector-deinterleave-lowering-transforms.mlir diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp index 77c97b2f1497..f7e01c7b12e4 100644 --- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp @@ -79,6 +79,73 @@ private: int64_t targetRank = 1; }; +/// A one-shot unrolling of vector.deinterleave to the `targetRank`. +/// +/// Example: +/// +/// ```mlir +/// %0, %1 = vector.deinterleave %a : vector<1x2x3x8xi64> -> vector<1x2x3x4xi64> +/// ``` +/// Would be unrolled to: +/// ```mlir +/// %result = arith.constant dense<0> : vector<1x2x3x4xi64> +/// %0 = vector.extract %a[0, 0, 0] ─┐ +/// : vector<8xi64> from vector<1x2x3x8xi64> | +/// %1, %2 = vector.deinterleave %0 | +/// : vector<8xi64> -> vector<4xi64> | -- Initial deinterleave +/// %3 = vector.insert %1, %result [0, 0, 0] | operation unrolled. +/// : vector<4xi64> into vector<1x2x3x4xi64> | +/// %4 = vector.insert %2, %result [0, 0, 0] | +/// : vector<4xi64> into vector<1x2x3x4xi64> ┘ +/// %5 = vector.extract %a[0, 0, 1] ─┐ +/// : vector<8xi64> from vector<1x2x3x8xi64> | +/// %6, %7 = vector.deinterleave %5 | +/// : vector<8xi64> -> vector<4xi64> | -- Recursive pattern for +/// %8 = vector.insert %6, %3 [0, 0, 1] | subsequent unrolled +/// : vector<4xi64> into vector<1x2x3x4xi64> | deinterleave +/// %9 = vector.insert %7, %4 [0, 0, 1] | operations. Repeated +/// : vector<4xi64> into vector<1x2x3x4xi64> ┘ 5x in this case. +/// ``` +/// +/// Note: If any leading dimension before the `targetRank` is scalable the +/// unrolling will stop before the scalable dimension. +class UnrollDeinterleaveOp final + : public OpRewritePattern { +public: + UnrollDeinterleaveOp(int64_t targetRank, MLIRContext *context, + PatternBenefit benefit = 1) + : OpRewritePattern(context, benefit), targetRank(targetRank) {}; + + LogicalResult matchAndRewrite(vector::DeinterleaveOp op, + PatternRewriter &rewriter) const override { + VectorType resultType = op.getResultVectorType(); + auto unrollIterator = vector::createUnrollIterator(resultType, targetRank); + if (!unrollIterator) + return failure(); + + auto loc = op.getLoc(); + Value emptyResult = rewriter.create( + loc, resultType, rewriter.getZeroAttr(resultType)); + Value evenResult = emptyResult; + Value oddResult = emptyResult; + + for (auto position : *unrollIterator) { + auto extractSrc = + rewriter.create(loc, op.getSource(), position); + auto deinterleave = + rewriter.create(loc, extractSrc); + evenResult = rewriter.create( + loc, deinterleave.getRes1(), evenResult, position); + oddResult = rewriter.create(loc, deinterleave.getRes2(), + oddResult, position); + } + rewriter.replaceOp(op, ValueRange{evenResult, oddResult}); + return success(); + } + +private: + int64_t targetRank = 1; +}; /// Rewrite vector.interleave op into an equivalent vector.shuffle op, when /// applicable: `sourceType` must be 1D and non-scalable. /// @@ -116,7 +183,8 @@ struct InterleaveToShuffle final : OpRewritePattern { void mlir::vector::populateVectorInterleaveLoweringPatterns( RewritePatternSet &patterns, int64_t targetRank, PatternBenefit benefit) { - patterns.add(targetRank, patterns.getContext(), benefit); + patterns.add( + targetRank, patterns.getContext(), benefit); } void mlir::vector::populateVectorInterleaveToShufflePatterns( diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir index 12121ea0dd70..54dcf0705390 100644 --- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir +++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir @@ -2565,6 +2565,22 @@ func.func @vector_deinterleave_1d_scalable(%a: vector<[4]xi32>) -> (vector<[2]xi return %0, %1 : vector<[2]xi32>, vector<[2]xi32> } +// CHECK-LABEL: @vector_deinterleave_2d +// CHECK-SAME: %[[SRC:.*]]: vector<2x8xf32>) -> (vector<2x4xf32>, vector<2x4xf32>) +func.func @vector_deinterleave_2d(%a: vector<2x8xf32>) -> (vector<2x4xf32>, vector<2x4xf32>) { + // CHECK: llvm.shufflevector + // CHECK-NOT: vector.deinterleave %{{.*}} : vector<2x8xf32> + %0, %1 = vector.deinterleave %a : vector<2x8xf32> -> vector<2x4xf32> + return %0, %1 : vector<2x4xf32>, vector<2x4xf32> +} + +func.func @vector_deinterleave_2d_scalable(%a: vector<2x[8]xf32>) -> (vector<2x[4]xf32>, vector<2x[4]xf32>) { + // CHECK: llvm.intr.vector.deinterleave2 + // CHECK-NOT: vector.deinterleave %{{.*}} : vector<2x[8]xf32> + %0, %1 = vector.deinterleave %a : vector<2x[8]xf32> -> vector<2x[4]xf32> + return %0, %1 : vector<2x[4]xf32>, vector<2x[4]xf32> +} + // ----- // CHECK-LABEL: func.func @vector_bitcast_2d diff --git a/mlir/test/Dialect/Vector/vector-deinterleave-lowering-transforms.mlir b/mlir/test/Dialect/Vector/vector-deinterleave-lowering-transforms.mlir new file mode 100644 index 000000000000..53f4a8970c79 --- /dev/null +++ b/mlir/test/Dialect/Vector/vector-deinterleave-lowering-transforms.mlir @@ -0,0 +1,68 @@ +// RUN: mlir-opt %s --transform-interpreter | FileCheck %s + +// CHECK-LABEL: @vector_deinterleave_2d +// CHECK-SAME: %[[SRC:.*]]: vector<2x8xi32>) -> (vector<2x4xi32>, vector<2x4xi32>) +func.func @vector_deinterleave_2d(%a: vector<2x8xi32>) -> (vector<2x4xi32>, vector<2x4xi32>) { + // CHECK: %[[CST:.*]] = arith.constant dense<0> + // CHECK: %[[SRC_0:.*]] = vector.extract %[[SRC]][0] + // CHECK: %[[UNZIP_0:.*]], %[[UNZIP_1:.*]] = vector.deinterleave %[[SRC_0]] + // CHECK: %[[RES_0:.*]] = vector.insert %[[UNZIP_0]], %[[CST]] [0] + // CHECK: %[[RES_1:.*]] = vector.insert %[[UNZIP_1]], %[[CST]] [0] + // CHECK: %[[SRC_1:.*]] = vector.extract %[[SRC]][1] + // CHECK: %[[UNZIP_2:.*]], %[[UNZIP_3:.*]] = vector.deinterleave %[[SRC_1]] + // CHECK: %[[RES_2:.*]] = vector.insert %[[UNZIP_2]], %[[RES_0]] [1] + // CHECK: %[[RES_3:.*]] = vector.insert %[[UNZIP_3]], %[[RES_1]] [1] + // CHECK-NEXT: return %[[RES_2]], %[[RES_3]] : vector<2x4xi32>, vector<2x4xi32> + %0, %1 = vector.deinterleave %a : vector<2x8xi32> -> vector<2x4xi32> + return %0, %1 : vector<2x4xi32>, vector<2x4xi32> +} + +// CHECK-LABEL: @vector_deinterleave_2d_scalable +// CHECK-SAME: %[[SRC:.*]]: vector<2x[8]xi32>) -> (vector<2x[4]xi32>, vector<2x[4]xi32>) +func.func @vector_deinterleave_2d_scalable(%a: vector<2x[8]xi32>) -> (vector<2x[4]xi32>, vector<2x[4]xi32>) { + // CHECK: %[[CST:.*]] = arith.constant dense<0> + // CHECK: %[[SRC_0:.*]] = vector.extract %[[SRC]][0] + // CHECK: %[[UNZIP_0:.*]], %[[UNZIP_1:.*]] = vector.deinterleave %[[SRC_0]] + // CHECK: %[[RES_0:.*]] = vector.insert %[[UNZIP_0]], %[[CST]] [0] + // CHECK: %[[RES_1:.*]] = vector.insert %[[UNZIP_1]], %[[CST]] [0] + // CHECK: %[[SRC_1:.*]] = vector.extract %[[SRC]][1] + // CHECK: %[[UNZIP_2:.*]], %[[UNZIP_3:.*]] = vector.deinterleave %[[SRC_1]] + // CHECK: %[[RES_2:.*]] = vector.insert %[[UNZIP_2]], %[[RES_0]] [1] + // CHECK: %[[RES_3:.*]] = vector.insert %[[UNZIP_3]], %[[RES_1]] [1] + // CHECK-NEXT: return %[[RES_2]], %[[RES_3]] : vector<2x[4]xi32>, vector<2x[4]xi32> + %0, %1 = vector.deinterleave %a : vector<2x[8]xi32> -> vector<2x[4]xi32> + return %0, %1 : vector<2x[4]xi32>, vector<2x[4]xi32> +} + +// CHECK-LABEL: @vector_deinterleave_4d +// CHECK-SAME: %[[SRC:.*]]: vector<1x2x3x8xi64>) -> (vector<1x2x3x4xi64>, vector<1x2x3x4xi64>) +func.func @vector_deinterleave_4d(%a: vector<1x2x3x8xi64>) -> (vector<1x2x3x4xi64>, vector<1x2x3x4xi64>) { + // CHECK: %[[SRC_0:.*]] = vector.extract %[[SRC]][0, 0, 0] : vector<8xi64> from vector<1x2x3x8xi64> + // CHECK: %[[UNZIP_0:.*]], %[[UNZIP_1:.*]] = vector.deinterleave %[[SRC_0]] : vector<8xi64> -> vector<4xi64> + // CHECK: %[[RES_0:.*]] = vector.insert %[[UNZIP_0]], %{{.*}} [0, 0, 0] : vector<4xi64> into vector<1x2x3x4xi64> + // CHECK: %[[RES_1:.*]] = vector.insert %[[UNZIP_1]], %{{.*}} [0, 0, 0] : vector<4xi64> into vector<1x2x3x4xi64> + // CHECK-COUNT-5: vector.deinterleave %{{.*}} : vector<8xi64> -> vector<4xi64> + %0, %1 = vector.deinterleave %a : vector<1x2x3x8xi64> -> vector<1x2x3x4xi64> + return %0, %1 : vector<1x2x3x4xi64>, vector<1x2x3x4xi64> +} + +// CHECK-LABEL: @vector_deinterleave_nd_with_scalable_dim +func.func @vector_deinterleave_nd_with_scalable_dim( + %a: vector<1x3x[2]x2x3x8xf16>) -> (vector<1x3x[2]x2x3x4xf16>, vector<1x3x[2]x2x3x4xf16>) { + // The scalable dim blocks unrolling so only the first two dims are unrolled. + // CHECK-COUNT-3: vector.deinterleave %{{.*}} : vector<[2]x2x3x8xf16> + %0, %1 = vector.deinterleave %a: vector<1x3x[2]x2x3x8xf16> -> vector<1x3x[2]x2x3x4xf16> + return %0, %1 : vector<1x3x[2]x2x3x4xf16>, vector<1x3x[2]x2x3x4xf16> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) { + %f = transform.structured.match ops{["func.func"]} in %module_op + : (!transform.any_op) -> !transform.any_op + + transform.apply_patterns to %f { + transform.apply_patterns.vector.lower_interleave + } : !transform.any_op + transform.yield + } +} -- GitLab From 1a5239251ead73ee57f4e2f7fc93433ac7cf18b1 Mon Sep 17 00:00:00 2001 From: Oliver Stannard Date: Fri, 7 Jun 2024 10:58:10 +0100 Subject: [PATCH 225/462] [ARM] r11 is reserved when using -mframe-chain=aapcs (#86951) When using the -mframe-chain=aapcs or -mframe-chain=aapcs-leaf options, we cannot use r11 as an allocatable register, even if -fomit-frame-pointer is also used. This is so that r11 will always point to a valid frame record, even if we don't create one in every function. --- clang/include/clang/Basic/CodeGenOptions.def | 2 +- clang/include/clang/Basic/CodeGenOptions.h | 9 +- clang/include/clang/Driver/Options.td | 4 +- clang/lib/CodeGen/CGCall.cpp | 1 + clang/lib/CodeGen/CodeGenModule.cpp | 3 + clang/lib/Driver/ToolChains/Arch/ARM.cpp | 2 - clang/lib/Driver/ToolChains/Clang.cpp | 3 + clang/lib/Driver/ToolChains/CommonArgs.cpp | 115 ++++++++++++++---- llvm/docs/LangRef.rst | 6 +- llvm/include/llvm/Support/CodeGen.h | 2 +- llvm/include/llvm/Target/TargetOptions.h | 5 + llvm/lib/CodeGen/CommandFlags.cpp | 5 + llvm/lib/CodeGen/TargetOptionsImpl.cpp | 21 +++- llvm/lib/IR/Function.cpp | 3 + llvm/lib/IR/Verifier.cpp | 2 +- llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp | 2 +- llvm/lib/Target/ARM/ARMFeatures.td | 11 +- llvm/lib/Target/ARM/ARMFrameLowering.cpp | 8 +- llvm/lib/Target/ARM/ARMFrameLowering.h | 1 + .../CodeGen/ARM/frame-chain-reserved-fp.ll | 25 ++-- llvm/test/CodeGen/ARM/frame-chain.ll | 11 +- llvm/test/CodeGen/Thumb/frame-access.ll | 4 +- .../CodeGen/Thumb/frame-chain-reserved-fp.ll | 27 ++-- llvm/test/CodeGen/Thumb/frame-chain.ll | 9 +- 24 files changed, 188 insertions(+), 93 deletions(-) diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def index 07b0ca1691a6..7ffc40a00504 100644 --- a/clang/include/clang/Basic/CodeGenOptions.def +++ b/clang/include/clang/Basic/CodeGenOptions.def @@ -61,7 +61,7 @@ CODEGENOPT(SeparateNamedSections, 1, 0) ///< Set for -fseparate-named-sections. CODEGENOPT(EnableAIXExtendedAltivecABI, 1, 0) ///< Set for -mabi=vec-extabi. Enables the extended Altivec ABI on AIX. CODEGENOPT(XCOFFReadOnlyPointers, 1, 0) ///< Set for -mxcoff-roptr. CODEGENOPT(AllTocData, 1, 0) ///< AIX -mtocdata -ENUM_CODEGENOPT(FramePointer, FramePointerKind, 2, FramePointerKind::None) /// frame-pointer: all,non-leaf,none +ENUM_CODEGENOPT(FramePointer, FramePointerKind, 2, FramePointerKind::None) /// frame-pointer: all,non-leaf,reserved,none CODEGENOPT(ClearASTBeforeBackend , 1, 0) ///< Free the AST before running backend code generation. Only works with -disable-free. CODEGENOPT(DisableFree , 1, 0) ///< Don't free memory. diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h index 9469a424045b..00523a84d389 100644 --- a/clang/include/clang/Basic/CodeGenOptions.h +++ b/clang/include/clang/Basic/CodeGenOptions.h @@ -127,15 +127,18 @@ public: std::string BinutilsVersion; enum class FramePointerKind { - None, // Omit all frame pointers. - NonLeaf, // Keep non-leaf frame pointers. - All, // Keep all frame pointers. + None, // Omit all frame pointers. + Reserved, // Maintain valid frame pointer chain. + NonLeaf, // Keep non-leaf frame pointers. + All, // Keep all frame pointers. }; static StringRef getFramePointerKindName(FramePointerKind Kind) { switch (Kind) { case FramePointerKind::None: return "none"; + case FramePointerKind::Reserved: + return "reserved"; case FramePointerKind::NonLeaf: return "non-leaf"; case FramePointerKind::All: diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index f04c220d6e1d..d44faa55c456 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -7724,8 +7724,8 @@ def pic_is_pie : Flag<["-"], "pic-is-pie">, MarshallingInfoFlag>; def mframe_pointer_EQ : Joined<["-"], "mframe-pointer=">, - HelpText<"Specify which frame pointers to retain.">, Values<"all,non-leaf,none">, - NormalizedValuesScope<"CodeGenOptions::FramePointerKind">, NormalizedValues<["All", "NonLeaf", "None"]>, + HelpText<"Specify which frame pointers to retain.">, Values<"all,non-leaf,reserved,none">, + NormalizedValuesScope<"CodeGenOptions::FramePointerKind">, NormalizedValues<["All", "NonLeaf", "Reserved", "None"]>, MarshallingInfoEnum, "None">; diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index 97449a5e51e7..65d82285b907 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -1917,6 +1917,7 @@ static void getTrivialDefaultFunctionAttributes( case CodeGenOptions::FramePointerKind::None: // This is the default behavior. break; + case CodeGenOptions::FramePointerKind::Reserved: case CodeGenOptions::FramePointerKind::NonLeaf: case CodeGenOptions::FramePointerKind::All: FuncAttrs.addAttribute("frame-pointer", diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index be7bf0b72dc0..75b144909038 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -1328,6 +1328,9 @@ void CodeGenModule::Release() { case CodeGenOptions::FramePointerKind::None: // 0 ("none") is the default. break; + case CodeGenOptions::FramePointerKind::Reserved: + getModule().setFramePointer(llvm::FramePointerKind::Reserved); + break; case CodeGenOptions::FramePointerKind::NonLeaf: getModule().setFramePointer(llvm::FramePointerKind::NonLeaf); break; diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.cpp b/clang/lib/Driver/ToolChains/Arch/ARM.cpp index a68368c47586..8ae22cc37a13 100644 --- a/clang/lib/Driver/ToolChains/Arch/ARM.cpp +++ b/clang/lib/Driver/ToolChains/Arch/ARM.cpp @@ -799,8 +799,6 @@ fp16_fml_fallthrough: StringRef FrameChainOption = A->getValue(); if (FrameChainOption.starts_with("aapcs")) Features.push_back("+aapcs-frame-chain"); - if (FrameChainOption == "aapcs+leaf") - Features.push_back("+aapcs-frame-chain-leaf"); } // CMSE: Check for target 8M (for -mcmse to be applicable) is performed later. diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 4e1c52462e58..b8d8ff3db5d1 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5678,6 +5678,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, case CodeGenOptions::FramePointerKind::None: FPKeepKindStr = "-mframe-pointer=none"; break; + case CodeGenOptions::FramePointerKind::Reserved: + FPKeepKindStr = "-mframe-pointer=reserved"; + break; case CodeGenOptions::FramePointerKind::NonLeaf: FPKeepKindStr = "-mframe-pointer=non-leaf"; break; diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 71e993119436..2a4c1369f5a7 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -164,6 +164,14 @@ static bool useFramePointerForTargetByDefault(const llvm::opt::ArgList &Args, return true; } +static bool useLeafFramePointerForTargetByDefault(const llvm::Triple &Triple) { + if (Triple.isAArch64() || Triple.isPS() || Triple.isVE() || + (Triple.isAndroid() && Triple.isRISCV64())) + return false; + + return true; +} + static bool mustUseNonLeafFramePointerForTarget(const llvm::Triple &Triple) { switch (Triple.getArch()) { default: @@ -176,38 +184,91 @@ static bool mustUseNonLeafFramePointerForTarget(const llvm::Triple &Triple) { } } +// True if a target-specific option requires the frame chain to be preserved, +// even if new frame records are not created. +static bool mustMaintainValidFrameChain(const llvm::opt::ArgList &Args, + const llvm::Triple &Triple) { + if (Triple.isARM() || Triple.isThumb()) { + // For 32-bit Arm, the -mframe-chain=aapcs and -mframe-chain=aapcs+leaf + // options require the frame pointer register to be reserved (or point to a + // new AAPCS-compilant frame record), even with -fno-omit-frame-pointer. + if (Arg *A = Args.getLastArg(options::OPT_mframe_chain)) { + StringRef V = A->getValue(); + return V != "none"; + } + return false; + } + return false; +} + +// True if a target-specific option causes -fno-omit-frame-pointer to also +// cause frame records to be created in leaf functions. +static bool framePointerImpliesLeafFramePointer(const llvm::opt::ArgList &Args, + const llvm::Triple &Triple) { + if (Triple.isARM() || Triple.isThumb()) { + // For 32-bit Arm, the -mframe-chain=aapcs+leaf option causes the + // -fno-omit-frame-pointer optiion to imply -mno-omit-leaf-frame-pointer, + // but does not by itself imply either option. + if (Arg *A = Args.getLastArg(options::OPT_mframe_chain)) { + StringRef V = A->getValue(); + return V == "aapcs+leaf"; + } + return false; + } + return false; +} + clang::CodeGenOptions::FramePointerKind getFramePointerKind(const llvm::opt::ArgList &Args, const llvm::Triple &Triple) { - // We have 4 states: + // There are three things to consider here: + // * Should a frame record be created for non-leaf functions? + // * Should a frame record be created for leaf functions? + // * Is the frame pointer register reserved, i.e. must it always point to + // either a new, valid frame record or be un-modified? // - // 00) leaf retained, non-leaf retained - // 01) leaf retained, non-leaf omitted (this is invalid) - // 10) leaf omitted, non-leaf retained - // (what -momit-leaf-frame-pointer was designed for) - // 11) leaf omitted, non-leaf omitted + // Not all combinations of these are valid: + // * It's not useful to have leaf frame records without non-leaf ones. + // * It's not useful to have frame records without reserving the frame + // pointer. // - // "omit" options taking precedence over "no-omit" options is the only way - // to make 3 valid states representable - llvm::opt::Arg *A = - Args.getLastArg(clang::driver::options::OPT_fomit_frame_pointer, - clang::driver::options::OPT_fno_omit_frame_pointer); - - bool OmitFP = A && A->getOption().matches( - clang::driver::options::OPT_fomit_frame_pointer); - bool NoOmitFP = A && A->getOption().matches( - clang::driver::options::OPT_fno_omit_frame_pointer); - bool OmitLeafFP = - Args.hasFlag(clang::driver::options::OPT_momit_leaf_frame_pointer, - clang::driver::options::OPT_mno_omit_leaf_frame_pointer, - Triple.isAArch64() || Triple.isPS() || Triple.isVE() || - (Triple.isAndroid() && Triple.isRISCV64())); - if (NoOmitFP || mustUseNonLeafFramePointerForTarget(Triple) || - (!OmitFP && useFramePointerForTargetByDefault(Args, Triple))) { - if (OmitLeafFP) - return clang::CodeGenOptions::FramePointerKind::NonLeaf; - return clang::CodeGenOptions::FramePointerKind::All; - } + // | Non-leaf | Leaf | Reserved | + // | N | N | N | FramePointerKind::None + // | N | N | Y | FramePointerKind::Reserved + // | N | Y | N | Invalid + // | N | Y | Y | Invalid + // | Y | N | N | Invalid + // | Y | N | Y | FramePointerKind::NonLeaf + // | Y | Y | N | Invalid + // | Y | Y | Y | FramePointerKind::All + // + // The FramePointerKind::Reserved case is currently only reachable for Arm, + // which has the -mframe-chain= option which can (in combination with + // -fno-omit-frame-pointer) specify that the frame chain must be valid, + // without requiring new frame records to be created. + + bool DefaultFP = useFramePointerForTargetByDefault(Args, Triple); + bool EnableFP = + mustUseNonLeafFramePointerForTarget(Triple) || + Args.hasFlag(clang::driver::options::OPT_fno_omit_frame_pointer, + clang::driver::options::OPT_fomit_frame_pointer, DefaultFP); + + bool DefaultLeafFP = + useLeafFramePointerForTargetByDefault(Triple) || + (EnableFP && framePointerImpliesLeafFramePointer(Args, Triple)); + bool EnableLeafFP = Args.hasFlag( + clang::driver::options::OPT_mno_omit_leaf_frame_pointer, + clang::driver::options::OPT_momit_leaf_frame_pointer, DefaultLeafFP); + + bool FPRegReserved = EnableFP || mustMaintainValidFrameChain(Args, Triple); + + if (EnableFP) { + if (EnableLeafFP) + return clang::CodeGenOptions::FramePointerKind::All; + return clang::CodeGenOptions::FramePointerKind::NonLeaf; + } + if (FPRegReserved) + return clang::CodeGenOptions::FramePointerKind::Reserved; return clang::CodeGenOptions::FramePointerKind::None; } diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index b1e0cee25c98..9fb2c048a5c8 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -1924,7 +1924,11 @@ example: even if this attribute says the frame pointer can be eliminated. The allowed string values are: - * ``"none"`` (default) - the frame pointer can be eliminated. + * ``"none"`` (default) - the frame pointer can be eliminated, and it's + register can be used for other purposes. + * ``"reserved"`` - the frame pointer register must either be updated to + point to a valid frame record for the current function, or not be + modified. * ``"non-leaf"`` - the frame pointer should be kept if the function calls other functions. * ``"all"`` - the frame pointer should be kept. diff --git a/llvm/include/llvm/Support/CodeGen.h b/llvm/include/llvm/Support/CodeGen.h index 46f99811763a..0e42789ba932 100644 --- a/llvm/include/llvm/Support/CodeGen.h +++ b/llvm/include/llvm/Support/CodeGen.h @@ -87,7 +87,7 @@ namespace llvm { }; // Specify what functions should keep the frame pointer. - enum class FramePointerKind { None, NonLeaf, All }; + enum class FramePointerKind { None, NonLeaf, All, Reserved }; // Specify what type of zeroing callee-used registers. namespace ZeroCallUsedRegs { diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h index 98a8b7ba337c..d3464b5202ff 100644 --- a/llvm/include/llvm/Target/TargetOptions.h +++ b/llvm/include/llvm/Target/TargetOptions.h @@ -161,6 +161,11 @@ namespace llvm { /// optimization should be disabled for the given machine function. bool DisableFramePointerElim(const MachineFunction &MF) const; + /// FramePointerIsReserved - This returns true if the frame pointer must + /// always either point to a new frame record or be un-modified in the given + /// function. + bool FramePointerIsReserved(const MachineFunction &MF) const; + /// If greater than 0, override the default value of /// MCAsmInfo::BinutilsVersion. std::pair BinutilsVersion{0, 0}; diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp index 677460a2d8e4..8fc65d78ff2c 100644 --- a/llvm/lib/CodeGen/CommandFlags.cpp +++ b/llvm/lib/CodeGen/CommandFlags.cpp @@ -211,6 +211,9 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() { "Disable frame pointer elimination"), clEnumValN(FramePointerKind::NonLeaf, "non-leaf", "Disable frame pointer elimination for non-leaf frame"), + clEnumValN(FramePointerKind::Reserved, "reserved", + "Enable frame pointer elimination, but reserve the frame " + "pointer register"), clEnumValN(FramePointerKind::None, "none", "Enable frame pointer elimination"))); CGBINDOPT(FramePointerUsage); @@ -693,6 +696,8 @@ void codegen::setFunctionAttributes(StringRef CPU, StringRef Features, NewAttrs.addAttribute("frame-pointer", "all"); else if (getFramePointerUsage() == FramePointerKind::NonLeaf) NewAttrs.addAttribute("frame-pointer", "non-leaf"); + else if (getFramePointerUsage() == FramePointerKind::Reserved) + NewAttrs.addAttribute("frame-pointer", "reserved"); else if (getFramePointerUsage() == FramePointerKind::None) NewAttrs.addAttribute("frame-pointer", "none"); } diff --git a/llvm/lib/CodeGen/TargetOptionsImpl.cpp b/llvm/lib/CodeGen/TargetOptionsImpl.cpp index af5d10103f78..5bf1d265092f 100644 --- a/llvm/lib/CodeGen/TargetOptionsImpl.cpp +++ b/llvm/lib/CodeGen/TargetOptionsImpl.cpp @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/StringSwitch.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/TargetFrameLowering.h" @@ -21,7 +22,7 @@ using namespace llvm; /// DisableFramePointerElim - This returns true if frame pointer elimination /// optimization should be disabled for the given machine function. bool TargetOptions::DisableFramePointerElim(const MachineFunction &MF) const { - // Check to see if the target want to forcably keep frame pointer. + // Check to see if the target want to forcibly keep frame pointer. if (MF.getSubtarget().getFrameLowering()->keepFramePointer(MF)) return true; @@ -34,11 +35,27 @@ bool TargetOptions::DisableFramePointerElim(const MachineFunction &MF) const { return true; if (FP == "non-leaf") return MF.getFrameInfo().hasCalls(); - if (FP == "none") + if (FP == "none" || FP == "reserved") return false; llvm_unreachable("unknown frame pointer flag"); } +bool TargetOptions::FramePointerIsReserved(const MachineFunction &MF) const { + // Check to see if the target want to forcibly keep frame pointer. + if (MF.getSubtarget().getFrameLowering()->keepFramePointer(MF)) + return true; + + const Function &F = MF.getFunction(); + + if (!F.hasFnAttribute("frame-pointer")) + return false; + + StringRef FP = F.getFnAttribute("frame-pointer").getValueAsString(); + return StringSwitch(FP) + .Cases("all", "non-leaf", "reserved", true) + .Case("none", false); +} + /// HonorSignDependentRoundingFPMath - Return true if the codegen must assume /// that the rounding mode of the FPU can change from its default. bool TargetOptions::HonorSignDependentRoundingFPMath() const { diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp index 13fa1afeaaff..3f735020e874 100644 --- a/llvm/lib/IR/Function.cpp +++ b/llvm/lib/IR/Function.cpp @@ -383,6 +383,9 @@ Function *Function::createWithDefaultAttr(FunctionType *Ty, case FramePointerKind::None: // 0 ("none") is the default. break; + case FramePointerKind::Reserved: + B.addAttribute("frame-pointer", "reserved"); + break; case FramePointerKind::NonLeaf: B.addAttribute("frame-pointer", "non-leaf"); break; diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 684e54444621..e5927203f33a 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -2322,7 +2322,7 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, if (Attrs.hasFnAttr("frame-pointer")) { StringRef FP = Attrs.getFnAttr("frame-pointer").getValueAsString(); - if (FP != "all" && FP != "non-leaf" && FP != "none") + if (FP != "all" && FP != "non-leaf" && FP != "none" && FP != "reserved") CheckFailed("invalid value for 'frame-pointer' attribute: " + FP, V); } diff --git a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp index 9adf758b46c4..c149db3144c7 100644 --- a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp +++ b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp @@ -207,7 +207,7 @@ getReservedRegs(const MachineFunction &MF) const { markSuperRegs(Reserved, ARM::PC); markSuperRegs(Reserved, ARM::FPSCR); markSuperRegs(Reserved, ARM::APSR_NZCV); - if (TFI->hasFP(MF)) + if (TFI->isFPReserved(MF)) markSuperRegs(Reserved, STI.getFramePointerReg()); if (hasBasePointer(MF)) markSuperRegs(Reserved, BasePtr); diff --git a/llvm/lib/Target/ARM/ARMFeatures.td b/llvm/lib/Target/ARM/ARMFeatures.td index 84481af650be..8b0ade54b46d 100644 --- a/llvm/lib/Target/ARM/ARMFeatures.td +++ b/llvm/lib/Target/ARM/ARMFeatures.td @@ -548,16 +548,15 @@ def FeatureFixCortexA57AES1742098 : SubtargetFeature<"fix-cortex-a57-aes-1742098 "FixCortexA57AES1742098", "true", "Work around Cortex-A57 Erratum 1742098 / Cortex-A72 Erratum 1655431 (AES)">; +// If frame pointers are in use, they must follow the AAPCS definition, which +// always uses R11 as the frame pointer. If this is not set, we can use R7 as +// the frame pointer for Thumb1-only code, which is more efficient, but less +// compatible. Note that this feature does not control whether frame pointers +// are emitted, that is controlled by the "frame-pointer" function attribute. def FeatureAAPCSFrameChain : SubtargetFeature<"aapcs-frame-chain", "CreateAAPCSFrameChain", "true", "Create an AAPCS compliant frame chain">; -def FeatureAAPCSFrameChainLeaf : SubtargetFeature<"aapcs-frame-chain-leaf", - "CreateAAPCSFrameChainLeaf", "true", - "Create an AAPCS compliant frame chain " - "for leaf functions", - [FeatureAAPCSFrameChain]>; - // Assume that lock-free 32-bit atomics are available, even if the target // and operating system combination would not usually provide them. The user // is responsible for providing any necessary __sync implementations. Code diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp index 11496a6e032d..831b6b0fc722 100644 --- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp +++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp @@ -215,7 +215,7 @@ bool ARMFrameLowering::hasFP(const MachineFunction &MF) const { /// isFPReserved - Return true if the frame pointer register should be /// considered a reserved register on the scope of the specified function. bool ARMFrameLowering::isFPReserved(const MachineFunction &MF) const { - return hasFP(MF) || MF.getSubtarget().createAAPCSFrameChain(); + return hasFP(MF) || MF.getTarget().Options.FramePointerIsReserved(MF); } /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is @@ -2233,10 +2233,10 @@ bool ARMFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const { return true; } -static bool requiresAAPCSFrameRecord(const MachineFunction &MF) { +bool ARMFrameLowering::requiresAAPCSFrameRecord( + const MachineFunction &MF) const { const auto &Subtarget = MF.getSubtarget(); - return Subtarget.createAAPCSFrameChainLeaf() || - (Subtarget.createAAPCSFrameChain() && MF.getFrameInfo().hasCalls()); + return Subtarget.createAAPCSFrameChain() && hasFP(MF); } // Thumb1 may require a spill when storing to a frame index through FP (or any diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.h b/llvm/lib/Target/ARM/ARMFrameLowering.h index 3c7358d8cd53..6a31b73957f1 100644 --- a/llvm/lib/Target/ARM/ARMFrameLowering.h +++ b/llvm/lib/Target/ARM/ARMFrameLowering.h @@ -47,6 +47,7 @@ public: bool hasFP(const MachineFunction &MF) const override; bool isFPReserved(const MachineFunction &MF) const; + bool requiresAAPCSFrameRecord(const MachineFunction &MF) const; bool hasReservedCallFrame(const MachineFunction &MF) const override; bool canSimplifyCallFramePseudos(const MachineFunction &MF) const override; StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, diff --git a/llvm/test/CodeGen/ARM/frame-chain-reserved-fp.ll b/llvm/test/CodeGen/ARM/frame-chain-reserved-fp.ll index 6540381d624b..a265dd10c0af 100644 --- a/llvm/test/CodeGen/ARM/frame-chain-reserved-fp.ll +++ b/llvm/test/CodeGen/ARM/frame-chain-reserved-fp.ll @@ -1,25 +1,24 @@ -; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all 2>&1 | FileCheck %s --check-prefix=RESERVED-R11 -; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=RESERVED-R11 -; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-R11 -; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-NONE -; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=RESERVED-R11 -; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-R11 -; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none 2>&1 | FileCheck %s --check-prefix=RESERVED-NONE -; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=RESERVED-R11 -; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-R11 +; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED +; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED +; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED +; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED +; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-FREE +; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-FREE +; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=reserved 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED +; RUN: not llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=reserved -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED declare void @leaf(i32 %input) define void @reserved_r7(i32 %input) { -; RESERVED-NONE-NOT: error: write to reserved register 'R7' -; RESERVED-R11-NOT: error: write to reserved register 'R7' +; R7-RESERVED: error: write to reserved register 'R7' +; R7-FREE-NOT: error: write to reserved register 'R7' %1 = call i32 asm sideeffect "mov $0, $1", "={r7},r"(i32 %input) ret void } define void @reserved_r11(i32 %input) { -; RESERVED-NONE-NOT: error: write to reserved register 'R11' -; RESERVED-R11: error: write to reserved register 'R11' +; R11-RESERVED: error: write to reserved register 'R11' +; R11-FREE-NOT: error: write to reserved register 'R11' %1 = call i32 asm sideeffect "mov $0, $1", "={r11},r"(i32 %input) ret void } diff --git a/llvm/test/CodeGen/ARM/frame-chain.ll b/llvm/test/CodeGen/ARM/frame-chain.ll index a0f03e51b461..e37213e4aaf8 100644 --- a/llvm/test/CodeGen/ARM/frame-chain.ll +++ b/llvm/test/CodeGen/ARM/frame-chain.ll @@ -1,12 +1,11 @@ ; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all | FileCheck %s --check-prefixes=FP,LEAF-FP -; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-FP -; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain-leaf | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-FP-AAPCS +; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-FP-AAPCS ; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf | FileCheck %s --check-prefixes=FP,LEAF-NOFP -; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-NOFP -; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain-leaf | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-NOFP-AAPCS +; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-NOFP-AAPCS ; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none | FileCheck %s --check-prefixes=NOFP,LEAF-NOFP -; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP -; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain-leaf | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP-AAPCS +; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP-AAPCS +; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=reserved | FileCheck %s --check-prefixes=NOFP,LEAF-NOFP +; RUN: llc -mtriple arm-arm-none-eabi -filetype asm -o - %s -frame-pointer=reserved -mattr=+aapcs-frame-chain | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP-AAPCS define dso_local noundef i32 @leaf(i32 noundef %0) { ; LEAF-FP-LABEL: leaf: diff --git a/llvm/test/CodeGen/Thumb/frame-access.ll b/llvm/test/CodeGen/Thumb/frame-access.ll index 422c595472ee..07a6b6d0b949 100644 --- a/llvm/test/CodeGen/Thumb/frame-access.ll +++ b/llvm/test/CodeGen/Thumb/frame-access.ll @@ -1,7 +1,7 @@ ; RUN: llc -mtriple=thumbv6m-eabi -frame-pointer=none %s -o - --verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP,CHECK-ATPCS ; RUN: llc -mtriple=thumbv6m-eabi -frame-pointer=all %s -o - --verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-FP-ATPCS,CHECK-ATPCS -; RUN: llc -mtriple=thumbv6m-eabi -frame-pointer=none -mattr=+aapcs-frame-chain-leaf %s -o - --verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP,CHECK-AAPCS -; RUN: llc -mtriple=thumbv6m-eabi -frame-pointer=all -mattr=+aapcs-frame-chain-leaf %s -o - --verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-FP-AAPCS,CHECK-AAPCS +; RUN: llc -mtriple=thumbv6m-eabi -frame-pointer=none -mattr=+aapcs-frame-chain %s -o - --verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP,CHECK-AAPCS +; RUN: llc -mtriple=thumbv6m-eabi -frame-pointer=all -mattr=+aapcs-frame-chain %s -o - --verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-FP-AAPCS,CHECK-AAPCS ; struct S { int x[128]; } s; ; int f(int *, int, int, int, struct S); diff --git a/llvm/test/CodeGen/Thumb/frame-chain-reserved-fp.ll b/llvm/test/CodeGen/Thumb/frame-chain-reserved-fp.ll index 37dd16bd9dd9..d62eb20b8459 100644 --- a/llvm/test/CodeGen/Thumb/frame-chain-reserved-fp.ll +++ b/llvm/test/CodeGen/Thumb/frame-chain-reserved-fp.ll @@ -1,27 +1,24 @@ -; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all 2>&1 | FileCheck %s --check-prefix=RESERVED-R7 -; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=RESERVED-R11 -; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-R11 -; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-NONE -; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=RESERVED-R11 -; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-R11 -; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none 2>&1 | FileCheck %s --check-prefix=RESERVED-NONE -; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=RESERVED-R11 -; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain-leaf 2>&1 | FileCheck %s --check-prefix=RESERVED-R11 +; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all 2>&1 | FileCheck %s --check-prefix=R7-RESERVED --check-prefix=R11-FREE +; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED +; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf 2>&1 | FileCheck %s --check-prefix=R7-RESERVED --check-prefix=R11-FREE +; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED +; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-FREE +; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-FREE +; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=reserved 2>&1 | FileCheck %s --check-prefix=R7-RESERVED --check-prefix=R11-FREE +; RUN: not llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=reserved -mattr=+aapcs-frame-chain 2>&1 | FileCheck %s --check-prefix=R7-FREE --check-prefix=R11-RESERVED declare void @leaf(i32 %input) define void @reserved_r7(i32 %input) { -; RESERVED-NONE-NOT: error: write to reserved register 'R7' -; RESERVED-R7: error: write to reserved register 'R7' -; RESERVED-R11-NOT: error: write to reserved register 'R7' +; R7-RESERVED: error: write to reserved register 'R7' +; R7-FREE-NOT: error: write to reserved register 'R7' %1 = call i32 asm sideeffect "mov $0, $1", "={r7},r"(i32 %input) ret void } define void @reserved_r11(i32 %input) { -; RESERVED-NONE-NOT: error: write to reserved register 'R11' -; RESERVED-R7-NOT: error: write to reserved register 'R11' -; RESERVED-R11: error: write to reserved register 'R11' +; R11-RESERVED: error: write to reserved register 'R11' +; R11-FREE-NOT: error: write to reserved register 'R11' %1 = call i32 asm sideeffect "mov $0, $1", "={r11},r"(i32 %input) ret void } diff --git a/llvm/test/CodeGen/Thumb/frame-chain.ll b/llvm/test/CodeGen/Thumb/frame-chain.ll index c92235e0f827..eb62ce09caf1 100644 --- a/llvm/test/CodeGen/Thumb/frame-chain.ll +++ b/llvm/test/CodeGen/Thumb/frame-chain.ll @@ -1,12 +1,9 @@ ; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all --verify-machineinstrs | FileCheck %s --check-prefixes=FP,LEAF-FP -; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain --verify-machineinstrs | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-FP -; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain-leaf --verify-machineinstrs | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-FP-AAPCS +; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=all -mattr=+aapcs-frame-chain --verify-machineinstrs | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-FP-AAPCS ; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf --verify-machineinstrs | FileCheck %s --check-prefixes=FP,LEAF-NOFP -; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain --verify-machineinstrs | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-NOFP -; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain-leaf --verify-machineinstrs | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-NOFP-AAPCS +; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=non-leaf -mattr=+aapcs-frame-chain --verify-machineinstrs | FileCheck %s --check-prefixes=FP-AAPCS,LEAF-NOFP-AAPCS ; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none --verify-machineinstrs | FileCheck %s --check-prefixes=NOFP,LEAF-NOFP -; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain --verify-machineinstrs | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP -; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain-leaf --verify-machineinstrs | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP-AAPCS +; RUN: llc -mtriple thumbv6m-arm-none-eabi -filetype asm -o - %s -frame-pointer=none -mattr=+aapcs-frame-chain --verify-machineinstrs | FileCheck %s --check-prefixes=NOFP-AAPCS,LEAF-NOFP-AAPCS define dso_local noundef i32 @leaf(i32 noundef %0) { ; LEAF-FP-LABEL: leaf: -- GitLab From af3ffff34fb39cfb1e19dce68bd1c3fefda336a4 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 7 Jun 2024 11:02:50 +0100 Subject: [PATCH 226/462] [DAG] Always allow folding XOR patterns to ABS pre-legalization (#94601) Removes residual ARM handling for vXi64 ABS nodes to prevent infinite loops. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 4 +- llvm/lib/Target/ARM/ARMISelLowering.cpp | 16 - llvm/lib/Target/ARM/ARMInstrNEON.td | 16 - ...amdgpu-codegenprepare-fold-binop-select.ll | 48 +- .../AMDGPU/amdgpu-codegenprepare-idiv.ll | 952 +++++++-------- llvm/test/CodeGen/AMDGPU/bypass-div.ll | 74 +- llvm/test/CodeGen/AMDGPU/div_i128.ll | 818 ++++++------- llvm/test/CodeGen/AMDGPU/div_v2i128.ll | 994 +++++++-------- llvm/test/CodeGen/AMDGPU/idiv-licm.ll | 100 +- llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll | 20 +- llvm/test/CodeGen/AMDGPU/itofp.i128.ll | 60 +- llvm/test/CodeGen/AMDGPU/rem_i128.ll | 898 +++++++------- llvm/test/CodeGen/AMDGPU/sdiv.ll | 1070 ++++++++--------- llvm/test/CodeGen/AMDGPU/sdiv64.ll | 294 +++-- llvm/test/CodeGen/AMDGPU/srem64.ll | 250 ++-- .../X86/div-rem-pair-recomposition-signed.ll | 298 ++--- llvm/test/CodeGen/X86/pr38539.ll | 2 +- 17 files changed, 2864 insertions(+), 3050 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 02cd125eeff0..a913472525cf 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -4042,7 +4042,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { } // fold B = sra (A, size(A)-1); sub (xor (A, B), B) -> (abs A) - if (hasOperation(ISD::ABS, VT) && + if ((!LegalOperations || hasOperation(ISD::ABS, VT)) && sd_match(N1, m_Sra(m_Value(A), m_SpecificInt(BitWidth - 1))) && sd_match(N0, m_Xor(m_Specific(A), m_Specific(N1)))) return DAG.getNode(ISD::ABS, DL, VT, A); @@ -9526,7 +9526,7 @@ SDValue DAGCombiner::visitXOR(SDNode *N) { } // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X) - if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { + if (!LegalOperations || hasOperation(ISD::ABS, VT)) { SDValue A = N0Opcode == ISD::ADD ? N0 : N1; SDValue S = N0Opcode == ISD::SRA ? N0 : N1; if (A.getOpcode() == ISD::ADD && S.getOpcode() == ISD::SRA) { diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 78aaaca4e185..e3270471981c 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -1623,9 +1623,6 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM, setPrefFunctionAlignment(Align(1ULL << Subtarget->getPrefLoopLogAlignment())); setMinFunctionAlignment(Subtarget->isThumb() ? Align(2) : Align(4)); - - if (Subtarget->isThumb() || Subtarget->isThumb2()) - setTargetDAGCombine(ISD::ABS); } bool ARMTargetLowering::useSoftFloat() const { @@ -13504,18 +13501,6 @@ static SDValue PerformVSetCCToVCTPCombine(SDNode *N, DCI.DAG.getZExtOrTrunc(Op1S, DL, MVT::i32)); } -static SDValue PerformABSCombine(SDNode *N, - TargetLowering::DAGCombinerInfo &DCI, - const ARMSubtarget *Subtarget) { - SelectionDAG &DAG = DCI.DAG; - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - - if (TLI.isOperationLegal(N->getOpcode(), N->getValueType(0))) - return SDValue(); - - return TLI.expandABS(N, DAG); -} - /// PerformADDECombine - Target-specific dag combine transform from /// ARMISD::ADDC, ARMISD::ADDE, and ISD::MUL_LOHI to MLAL or /// ARMISD::ADDC, ARMISD::ADDE and ARMISD::UMLAL to ARMISD::UMAAL @@ -18879,7 +18864,6 @@ SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N, case ISD::SELECT: return PerformSELECTCombine(N, DCI, Subtarget); case ISD::VSELECT: return PerformVSELECTCombine(N, DCI, Subtarget); case ISD::SETCC: return PerformVSetCCToVCTPCombine(N, DCI, Subtarget); - case ISD::ABS: return PerformABSCombine(N, DCI, Subtarget); case ARMISD::ADDE: return PerformADDECombine(N, DCI, Subtarget); case ARMISD::UMLAL: return PerformUMLALCombine(N, DCI.DAG, Subtarget); case ISD::ADD: return PerformADDCombine(N, DCI, Subtarget); diff --git a/llvm/lib/Target/ARM/ARMInstrNEON.td b/llvm/lib/Target/ARM/ARMInstrNEON.td index c600478b6640..fcabc9076e4d 100644 --- a/llvm/lib/Target/ARM/ARMInstrNEON.td +++ b/llvm/lib/Target/ARM/ARMInstrNEON.td @@ -5670,22 +5670,6 @@ def : Pat<(v2i64 (zext (abdu (v2i32 DPR:$opA), (v2i32 DPR:$opB)))), (VABDLuv2i64 DPR:$opA, DPR:$opB)>; } -// ISD::ABS is not legal for v2i64, so VABDL needs to be matched from the -// shift/xor pattern for ABS. -// TODO: Remove me. -def abd_shr : - PatFrag<(ops node:$in1, node:$in2, node:$shift), - (ARMvshrsImm (sub (zext node:$in1), - (zext node:$in2)), (i32 $shift))>; - -let Predicates = [HasNEON] in { -def : Pat<(xor (v2i64 (abd_shr (v2i32 DPR:$opA), (v2i32 DPR:$opB), 63)), - (v2i64 (add (sub (zext (v2i32 DPR:$opA)), - (zext (v2i32 DPR:$opB))), - (abd_shr (v2i32 DPR:$opA), (v2i32 DPR:$opB), 63)))), - (VABDLuv2i64 DPR:$opA, DPR:$opB)>; -} - // VABA : Vector Absolute Difference and Accumulate defm VABAs : N3VIntOp_QHS<0,0,0b0111,1, IIC_VABAD, IIC_VABAQ, "vaba", "s", abds, add>; diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll index 5c40a4ce13e3..cb59121d6970 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll @@ -135,31 +135,31 @@ define i32 @select_sdiv_lhs_opaque_const0_i32(i1 %cond) { ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: v_mov_b32_e32 v1, s4 ; GCN-NEXT: v_cndmask_b32_e32 v0, 5, v1, vcc -; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 -; GCN-NEXT: v_add_u32_e32 v0, vcc, v0, v1 -; GCN-NEXT: v_xor_b32_e32 v0, v0, v1 -; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 -; GCN-NEXT: v_sub_u32_e32 v3, vcc, 0, v0 +; GCN-NEXT: v_sub_u32_e32 v1, vcc, 0, v0 +; GCN-NEXT: v_max_i32_e32 v1, v0, v1 +; GCN-NEXT: v_cvt_f32_u32_e32 v2, v1 +; GCN-NEXT: v_sub_u32_e32 v3, vcc, 0, v1 ; GCN-NEXT: s_mov_b32 s4, 0xf4240 ; GCN-NEXT: v_rcp_iflag_f32_e32 v2, v2 +; GCN-NEXT: v_ashrrev_i32_e32 v0, 31, v0 ; GCN-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 ; GCN-NEXT: v_mul_lo_u32 v3, v3, v2 ; GCN-NEXT: v_mul_hi_u32 v3, v2, v3 ; GCN-NEXT: v_add_u32_e32 v2, vcc, v2, v3 ; GCN-NEXT: v_mul_hi_u32 v2, v2, s4 -; GCN-NEXT: v_mul_lo_u32 v3, v2, v0 +; GCN-NEXT: v_mul_lo_u32 v3, v2, v1 ; GCN-NEXT: v_add_u32_e32 v4, vcc, 1, v2 ; GCN-NEXT: v_sub_u32_e32 v3, vcc, 0xf4240, v3 -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v1 ; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc -; GCN-NEXT: v_sub_u32_e64 v4, s[4:5], v3, v0 +; GCN-NEXT: v_sub_u32_e64 v4, s[4:5], v3, v1 ; GCN-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GCN-NEXT: v_add_u32_e32 v4, vcc, 1, v2 -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 -; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc -; GCN-NEXT: v_xor_b32_e32 v0, v0, v1 -; GCN-NEXT: v_sub_u32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v1 +; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc +; GCN-NEXT: v_xor_b32_e32 v1, v1, v0 +; GCN-NEXT: v_sub_u32_e32 v0, vcc, v1, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] %select = select i1 %cond, i32 ptrtoint (ptr addrspace(1) @gv to i32), i32 5 %op = sdiv i32 1000000, %select @@ -217,31 +217,31 @@ define i32 @select_sdiv_lhs_opaque_const1_i32(i1 %cond) { ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: v_mov_b32_e32 v1, s4 ; GCN-NEXT: v_cndmask_b32_e64 v0, v1, 5, vcc -; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 -; GCN-NEXT: v_add_u32_e32 v0, vcc, v0, v1 -; GCN-NEXT: v_xor_b32_e32 v0, v0, v1 -; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 -; GCN-NEXT: v_sub_u32_e32 v3, vcc, 0, v0 +; GCN-NEXT: v_sub_u32_e32 v1, vcc, 0, v0 +; GCN-NEXT: v_max_i32_e32 v1, v0, v1 +; GCN-NEXT: v_cvt_f32_u32_e32 v2, v1 +; GCN-NEXT: v_sub_u32_e32 v3, vcc, 0, v1 ; GCN-NEXT: s_mov_b32 s4, 0xf4240 ; GCN-NEXT: v_rcp_iflag_f32_e32 v2, v2 +; GCN-NEXT: v_ashrrev_i32_e32 v0, 31, v0 ; GCN-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 ; GCN-NEXT: v_mul_lo_u32 v3, v3, v2 ; GCN-NEXT: v_mul_hi_u32 v3, v2, v3 ; GCN-NEXT: v_add_u32_e32 v2, vcc, v2, v3 ; GCN-NEXT: v_mul_hi_u32 v2, v2, s4 -; GCN-NEXT: v_mul_lo_u32 v3, v2, v0 +; GCN-NEXT: v_mul_lo_u32 v3, v2, v1 ; GCN-NEXT: v_add_u32_e32 v4, vcc, 1, v2 ; GCN-NEXT: v_sub_u32_e32 v3, vcc, 0xf4240, v3 -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v1 ; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc -; GCN-NEXT: v_sub_u32_e64 v4, s[4:5], v3, v0 +; GCN-NEXT: v_sub_u32_e64 v4, s[4:5], v3, v1 ; GCN-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GCN-NEXT: v_add_u32_e32 v4, vcc, 1, v2 -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 -; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc -; GCN-NEXT: v_xor_b32_e32 v0, v0, v1 -; GCN-NEXT: v_sub_u32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v1 +; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc +; GCN-NEXT: v_xor_b32_e32 v1, v1, v0 +; GCN-NEXT: v_sub_u32_e32 v0, vcc, v1, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] %select = select i1 %cond, i32 5, i32 ptrtoint (ptr addrspace(1) @gv to i32) %op = sdiv i32 1000000, %select diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll index 2ad28b8dd6ec..8144fb7a3b64 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll @@ -245,39 +245,36 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) { ; GFX6-NEXT: s_mov_b32 s7, 0xf000 ; GFX6-NEXT: s_mov_b32 s6, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_ashr_i32 s8, s3, 31 -; GFX6-NEXT: s_add_i32 s3, s3, s8 -; GFX6-NEXT: s_xor_b32 s3, s3, s8 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3 -; GFX6-NEXT: s_sub_i32 s4, 0, s3 -; GFX6-NEXT: s_ashr_i32 s9, s2, 31 -; GFX6-NEXT: s_add_i32 s2, s2, s9 -; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX6-NEXT: s_xor_b32 s2, s2, s9 +; GFX6-NEXT: s_abs_i32 s8, s3 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GFX6-NEXT: s_sub_i32 s4, 0, s8 ; GFX6-NEXT: s_mov_b32 s5, s1 +; GFX6-NEXT: s_xor_b32 s1, s2, s3 +; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 +; GFX6-NEXT: s_ashr_i32 s1, s1, 31 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_lo_u32 v1, s4, v0 ; GFX6-NEXT: s_mov_b32 s4, s0 -; GFX6-NEXT: s_xor_b32 s0, s9, s8 +; GFX6-NEXT: s_abs_i32 s0, s2 ; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0 -; GFX6-NEXT: v_readfirstlane_b32 s1, v0 -; GFX6-NEXT: s_mul_i32 s1, s1, s3 -; GFX6-NEXT: s_sub_i32 s1, s2, s1 -; GFX6-NEXT: s_sub_i32 s2, s1, s3 +; GFX6-NEXT: v_mul_hi_u32 v0, s0, v0 +; GFX6-NEXT: v_readfirstlane_b32 s2, v0 +; GFX6-NEXT: s_mul_i32 s2, s2, s8 +; GFX6-NEXT: s_sub_i32 s0, s0, s2 +; GFX6-NEXT: s_sub_i32 s2, s0, s8 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 -; GFX6-NEXT: s_cmp_ge_u32 s1, s3 +; GFX6-NEXT: s_cmp_ge_u32 s0, s8 ; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GFX6-NEXT: s_cselect_b32 s1, s2, s1 +; GFX6-NEXT: s_cselect_b32 s0, s2, s0 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 -; GFX6-NEXT: s_cmp_ge_u32 s1, s3 +; GFX6-NEXT: s_cmp_ge_u32 s0, s8 ; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GFX6-NEXT: v_xor_b32_e32 v0, s0, v0 -; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0 +; GFX6-NEXT: v_xor_b32_e32 v0, s1, v0 +; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s1, v0 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0 ; GFX6-NEXT: s_endpgm ; @@ -286,16 +283,13 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) { ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_ashr_i32 s4, s3, 31 -; GFX9-NEXT: s_add_i32 s3, s3, s4 -; GFX9-NEXT: s_xor_b32 s3, s3, s4 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 -; GFX9-NEXT: s_ashr_i32 s5, s2, 31 -; GFX9-NEXT: s_add_i32 s2, s2, s5 -; GFX9-NEXT: s_xor_b32 s4, s5, s4 +; GFX9-NEXT: s_abs_i32 s4, s3 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4 +; GFX9-NEXT: s_sub_i32 s5, 0, s4 +; GFX9-NEXT: s_xor_b32 s3, s2, s3 +; GFX9-NEXT: s_abs_i32 s2, s2 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s2, s2, s5 -; GFX9-NEXT: s_sub_i32 s5, 0, s3 +; GFX9-NEXT: s_ashr_i32 s3, s3, 31 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX9-NEXT: v_readfirstlane_b32 s6, v0 @@ -303,18 +297,18 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) { ; GFX9-NEXT: s_mul_hi_u32 s5, s6, s5 ; GFX9-NEXT: s_add_i32 s6, s6, s5 ; GFX9-NEXT: s_mul_hi_u32 s5, s2, s6 -; GFX9-NEXT: s_mul_i32 s6, s5, s3 +; GFX9-NEXT: s_mul_i32 s6, s5, s4 ; GFX9-NEXT: s_sub_i32 s2, s2, s6 ; GFX9-NEXT: s_add_i32 s7, s5, 1 -; GFX9-NEXT: s_sub_i32 s6, s2, s3 -; GFX9-NEXT: s_cmp_ge_u32 s2, s3 +; GFX9-NEXT: s_sub_i32 s6, s2, s4 +; GFX9-NEXT: s_cmp_ge_u32 s2, s4 ; GFX9-NEXT: s_cselect_b32 s5, s7, s5 ; GFX9-NEXT: s_cselect_b32 s2, s6, s2 ; GFX9-NEXT: s_add_i32 s6, s5, 1 -; GFX9-NEXT: s_cmp_ge_u32 s2, s3 +; GFX9-NEXT: s_cmp_ge_u32 s2, s4 ; GFX9-NEXT: s_cselect_b32 s2, s6, s5 -; GFX9-NEXT: s_xor_b32 s2, s2, s4 -; GFX9-NEXT: s_sub_i32 s2, s2, s4 +; GFX9-NEXT: s_xor_b32 s2, s2, s3 +; GFX9-NEXT: s_sub_i32 s2, s2, s3 ; GFX9-NEXT: v_mov_b32_e32 v0, s2 ; GFX9-NEXT: global_store_dword v1, v0, s[0:1] ; GFX9-NEXT: s_endpgm @@ -366,37 +360,36 @@ define amdgpu_kernel void @srem_i32(ptr addrspace(1) %out, i32 %x, i32 %y) { ; GFX6-LABEL: srem_i32: ; GFX6: ; %bb.0: ; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; GFX6-NEXT: s_mov_b32 s7, 0xf000 +; GFX6-NEXT: s_mov_b32 s6, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_ashr_i32 s4, s3, 31 -; GFX6-NEXT: s_add_i32 s3, s3, s4 -; GFX6-NEXT: s_xor_b32 s4, s3, s4 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s4 -; GFX6-NEXT: s_sub_i32 s3, 0, s4 -; GFX6-NEXT: s_ashr_i32 s5, s2, 31 -; GFX6-NEXT: s_add_i32 s2, s2, s5 +; GFX6-NEXT: s_abs_i32 s3, s3 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3 +; GFX6-NEXT: s_sub_i32 s4, 0, s3 +; GFX6-NEXT: s_abs_i32 s8, s2 +; GFX6-NEXT: s_mov_b32 s5, s1 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX6-NEXT: s_xor_b32 s6, s2, s5 -; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX6-NEXT: v_mul_lo_u32 v1, s3, v0 -; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: v_mul_lo_u32 v1, s4, v0 +; GFX6-NEXT: s_mov_b32 s4, s0 +; GFX6-NEXT: s_ashr_i32 s0, s2, 31 ; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_mul_hi_u32 v0, s6, v0 -; GFX6-NEXT: v_readfirstlane_b32 s7, v0 -; GFX6-NEXT: s_mul_i32 s7, s7, s4 -; GFX6-NEXT: s_sub_i32 s6, s6, s7 -; GFX6-NEXT: s_sub_i32 s7, s6, s4 -; GFX6-NEXT: s_cmp_ge_u32 s6, s4 -; GFX6-NEXT: s_cselect_b32 s6, s7, s6 -; GFX6-NEXT: s_sub_i32 s7, s6, s4 -; GFX6-NEXT: s_cmp_ge_u32 s6, s4 -; GFX6-NEXT: s_cselect_b32 s4, s7, s6 -; GFX6-NEXT: s_xor_b32 s4, s4, s5 -; GFX6-NEXT: s_sub_i32 s4, s4, s5 -; GFX6-NEXT: v_mov_b32_e32 v0, s4 -; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0 +; GFX6-NEXT: v_mul_hi_u32 v0, s8, v0 +; GFX6-NEXT: v_readfirstlane_b32 s1, v0 +; GFX6-NEXT: s_mul_i32 s1, s1, s3 +; GFX6-NEXT: s_sub_i32 s1, s8, s1 +; GFX6-NEXT: s_sub_i32 s2, s1, s3 +; GFX6-NEXT: s_cmp_ge_u32 s1, s3 +; GFX6-NEXT: s_cselect_b32 s1, s2, s1 +; GFX6-NEXT: s_sub_i32 s2, s1, s3 +; GFX6-NEXT: s_cmp_ge_u32 s1, s3 +; GFX6-NEXT: s_cselect_b32 s1, s2, s1 +; GFX6-NEXT: s_xor_b32 s1, s1, s0 +; GFX6-NEXT: s_sub_i32 s0, s1, s0 +; GFX6-NEXT: v_mov_b32_e32 v0, s0 +; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0 ; GFX6-NEXT: s_endpgm ; ; GFX9-LABEL: srem_i32: @@ -404,15 +397,12 @@ define amdgpu_kernel void @srem_i32(ptr addrspace(1) %out, i32 %x, i32 %y) { ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_ashr_i32 s4, s3, 31 -; GFX9-NEXT: s_add_i32 s3, s3, s4 -; GFX9-NEXT: s_xor_b32 s3, s3, s4 +; GFX9-NEXT: s_abs_i32 s3, s3 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 ; GFX9-NEXT: s_sub_i32 s5, 0, s3 ; GFX9-NEXT: s_ashr_i32 s4, s2, 31 -; GFX9-NEXT: s_add_i32 s2, s2, s4 +; GFX9-NEXT: s_abs_i32 s2, s2 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s2, s2, s4 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX9-NEXT: v_readfirstlane_b32 s6, v0 @@ -1857,126 +1847,114 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x ; GFX6-NEXT: s_mov_b32 s19, 0xf000 ; GFX6-NEXT: s_mov_b32 s18, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_ashr_i32 s2, s12, 31 -; GFX6-NEXT: s_add_i32 s3, s12, s2 -; GFX6-NEXT: s_xor_b32 s3, s3, s2 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3 -; GFX6-NEXT: s_sub_i32 s4, 0, s3 +; GFX6-NEXT: s_abs_i32 s2, s12 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GFX6-NEXT: s_sub_i32 s3, 0, s2 +; GFX6-NEXT: s_xor_b32 s4, s8, s12 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX6-NEXT: v_mul_lo_u32 v1, s4, v0 -; GFX6-NEXT: s_ashr_i32 s4, s8, 31 -; GFX6-NEXT: s_add_i32 s5, s8, s4 -; GFX6-NEXT: s_xor_b32 s5, s5, s4 +; GFX6-NEXT: v_mul_lo_u32 v1, s3, v0 +; GFX6-NEXT: s_abs_i32 s3, s8 +; GFX6-NEXT: s_ashr_i32 s8, s4, 31 ; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 -; GFX6-NEXT: s_xor_b32 s8, s4, s2 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_mul_hi_u32 v0, s5, v0 -; GFX6-NEXT: v_readfirstlane_b32 s2, v0 -; GFX6-NEXT: s_mul_i32 s2, s2, s3 -; GFX6-NEXT: s_sub_i32 s2, s5, s2 -; GFX6-NEXT: s_sub_i32 s4, s2, s3 -; GFX6-NEXT: s_cmp_ge_u32 s2, s3 +; GFX6-NEXT: v_mul_hi_u32 v0, s3, v0 +; GFX6-NEXT: v_readfirstlane_b32 s4, v0 +; GFX6-NEXT: s_mul_i32 s4, s4, s2 +; GFX6-NEXT: s_sub_i32 s3, s3, s4 +; GFX6-NEXT: s_sub_i32 s4, s3, s2 +; GFX6-NEXT: s_cmp_ge_u32 s3, s2 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 -; GFX6-NEXT: s_cselect_b32 s2, s4, s2 +; GFX6-NEXT: s_cselect_b32 s3, s4, s3 ; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 -; GFX6-NEXT: s_cmp_ge_u32 s2, s3 +; GFX6-NEXT: s_cmp_ge_u32 s3, s2 ; GFX6-NEXT: s_cselect_b64 s[2:3], -1, 0 -; GFX6-NEXT: s_ashr_i32 s4, s13, 31 -; GFX6-NEXT: s_add_i32 s5, s13, s4 -; GFX6-NEXT: s_xor_b32 s5, s5, s4 -; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5 -; GFX6-NEXT: s_sub_i32 s6, 0, s5 +; GFX6-NEXT: s_abs_i32 s4, s13 +; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s4 +; GFX6-NEXT: s_sub_i32 s5, 0, s4 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GFX6-NEXT: s_xor_b32 s6, s9, s13 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v2, v2 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[2:3] -; GFX6-NEXT: v_xor_b32_e32 v0, s8, v0 ; GFX6-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, s6, v2 -; GFX6-NEXT: s_ashr_i32 s6, s9, 31 -; GFX6-NEXT: s_add_i32 s7, s9, s6 -; GFX6-NEXT: s_xor_b32 s7, s7, s6 +; GFX6-NEXT: v_xor_b32_e32 v0, s8, v0 +; GFX6-NEXT: v_mul_lo_u32 v3, s5, v2 +; GFX6-NEXT: s_abs_i32 s5, s9 +; GFX6-NEXT: s_ashr_i32 s9, s6, 31 ; GFX6-NEXT: v_mul_hi_u32 v3, v2, v3 -; GFX6-NEXT: s_xor_b32 s9, s6, s4 ; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; GFX6-NEXT: v_mul_hi_u32 v2, s7, v2 -; GFX6-NEXT: v_readfirstlane_b32 s4, v2 -; GFX6-NEXT: s_mul_i32 s4, s4, s5 -; GFX6-NEXT: s_sub_i32 s4, s7, s4 -; GFX6-NEXT: s_sub_i32 s6, s4, s5 -; GFX6-NEXT: s_cmp_ge_u32 s4, s5 +; GFX6-NEXT: v_mul_hi_u32 v2, s5, v2 +; GFX6-NEXT: v_readfirstlane_b32 s6, v2 +; GFX6-NEXT: s_mul_i32 s6, s6, s4 +; GFX6-NEXT: s_sub_i32 s5, s5, s6 +; GFX6-NEXT: s_sub_i32 s6, s5, s4 +; GFX6-NEXT: s_cmp_ge_u32 s5, s4 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v2 -; GFX6-NEXT: s_cselect_b32 s4, s6, s4 +; GFX6-NEXT: s_cselect_b32 s5, s6, s5 ; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 -; GFX6-NEXT: s_cmp_ge_u32 s4, s5 +; GFX6-NEXT: s_cmp_ge_u32 s5, s4 ; GFX6-NEXT: s_cselect_b64 s[4:5], -1, 0 -; GFX6-NEXT: s_ashr_i32 s6, s14, 31 -; GFX6-NEXT: s_add_i32 s7, s14, s6 -; GFX6-NEXT: s_xor_b32 s7, s7, s6 -; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s7 -; GFX6-NEXT: s_sub_i32 s12, 0, s7 +; GFX6-NEXT: s_abs_i32 s6, s14 +; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s6 +; GFX6-NEXT: s_sub_i32 s7, 0, s6 ; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v2 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v4, v4 -; GFX6-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[4:5] -; GFX6-NEXT: v_xor_b32_e32 v2, s9, v2 ; GFX6-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 ; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v4 -; GFX6-NEXT: v_mul_lo_u32 v5, s12, v4 -; GFX6-NEXT: s_ashr_i32 s12, s10, 31 -; GFX6-NEXT: s_add_i32 s10, s10, s12 -; GFX6-NEXT: s_xor_b32 s10, s10, s12 +; GFX6-NEXT: v_mul_lo_u32 v5, s7, v4 +; GFX6-NEXT: s_abs_i32 s7, s10 +; GFX6-NEXT: s_xor_b32 s10, s10, s14 +; GFX6-NEXT: s_ashr_i32 s10, s10, 31 ; GFX6-NEXT: v_mul_hi_u32 v5, v4, v5 -; GFX6-NEXT: s_xor_b32 s12, s12, s6 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GFX6-NEXT: v_mul_hi_u32 v4, s10, v4 -; GFX6-NEXT: v_readfirstlane_b32 s6, v4 -; GFX6-NEXT: s_mul_i32 s6, s6, s7 -; GFX6-NEXT: s_sub_i32 s6, s10, s6 -; GFX6-NEXT: s_sub_i32 s10, s6, s7 -; GFX6-NEXT: s_cmp_ge_u32 s6, s7 +; GFX6-NEXT: v_mul_hi_u32 v4, s7, v4 +; GFX6-NEXT: v_readfirstlane_b32 s12, v4 +; GFX6-NEXT: s_mul_i32 s12, s12, s6 +; GFX6-NEXT: s_sub_i32 s7, s7, s12 +; GFX6-NEXT: s_sub_i32 s12, s7, s6 +; GFX6-NEXT: s_cmp_ge_u32 s7, s6 ; GFX6-NEXT: v_add_i32_e32 v5, vcc, 1, v4 -; GFX6-NEXT: s_cselect_b32 s6, s10, s6 +; GFX6-NEXT: s_cselect_b32 s7, s12, s7 ; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 -; GFX6-NEXT: s_cmp_ge_u32 s6, s7 +; GFX6-NEXT: s_cmp_ge_u32 s7, s6 ; GFX6-NEXT: s_cselect_b64 s[6:7], -1, 0 -; GFX6-NEXT: s_ashr_i32 s10, s15, 31 -; GFX6-NEXT: s_add_i32 s13, s15, s10 -; GFX6-NEXT: s_xor_b32 s13, s13, s10 -; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s13 -; GFX6-NEXT: s_sub_i32 s0, 0, s13 +; GFX6-NEXT: s_abs_i32 s12, s15 +; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s12 +; GFX6-NEXT: s_sub_i32 s0, 0, s12 ; GFX6-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc ; GFX6-NEXT: v_add_i32_e32 v5, vcc, 1, v4 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v1, v6 -; GFX6-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[6:7] -; GFX6-NEXT: v_xor_b32_e32 v4, s12, v4 +; GFX6-NEXT: s_abs_i32 s1, s11 ; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s8, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1 -; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, s9, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, s0, v3 -; GFX6-NEXT: s_ashr_i32 s0, s11, 31 -; GFX6-NEXT: s_add_i32 s1, s11, s0 -; GFX6-NEXT: s_xor_b32 s1, s1, s0 -; GFX6-NEXT: v_mul_hi_u32 v2, v3, v2 -; GFX6-NEXT: s_xor_b32 s0, s0, s10 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_hi_u32 v3, s1, v2 -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s12, v4 -; GFX6-NEXT: v_readfirstlane_b32 s2, v3 -; GFX6-NEXT: s_mul_i32 s2, s2, s13 +; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v1 +; GFX6-NEXT: v_cndmask_b32_e64 v1, v2, v3, s[4:5] +; GFX6-NEXT: v_cndmask_b32_e64 v3, v4, v5, s[6:7] +; GFX6-NEXT: v_xor_b32_e32 v1, s9, v1 +; GFX6-NEXT: v_mul_lo_u32 v2, s0, v6 +; GFX6-NEXT: s_xor_b32 s0, s11, s15 +; GFX6-NEXT: v_xor_b32_e32 v3, s10, v3 +; GFX6-NEXT: s_ashr_i32 s0, s0, 31 +; GFX6-NEXT: v_mul_hi_u32 v2, v6, v2 +; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, s9, v1 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v6, v2 +; GFX6-NEXT: v_mul_hi_u32 v4, s1, v2 +; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s10, v3 +; GFX6-NEXT: v_readfirstlane_b32 s2, v4 +; GFX6-NEXT: s_mul_i32 s2, s2, s12 ; GFX6-NEXT: s_sub_i32 s1, s1, s2 -; GFX6-NEXT: s_sub_i32 s2, s1, s13 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, 1, v3 -; GFX6-NEXT: s_cmp_ge_u32 s1, s13 +; GFX6-NEXT: s_sub_i32 s2, s1, s12 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v4 +; GFX6-NEXT: s_cmp_ge_u32 s1, s12 ; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 -; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc +; GFX6-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc ; GFX6-NEXT: s_cselect_b32 s1, s2, s1 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 1, v3 -; GFX6-NEXT: s_cmp_ge_u32 s1, s13 +; GFX6-NEXT: s_cmp_ge_u32 s1, s12 ; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 ; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX6-NEXT: v_xor_b32_e32 v3, s0, v3 @@ -1990,16 +1968,13 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x ; GFX9-NEXT: v_mov_b32_e32 v4, 0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_ashr_i32 s2, s8, 31 -; GFX9-NEXT: s_add_i32 s3, s8, s2 -; GFX9-NEXT: s_xor_b32 s3, s3, s2 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 -; GFX9-NEXT: s_ashr_i32 s8, s4, 31 -; GFX9-NEXT: s_add_i32 s4, s4, s8 -; GFX9-NEXT: s_xor_b32 s2, s8, s2 +; GFX9-NEXT: s_abs_i32 s2, s8 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GFX9-NEXT: s_xor_b32 s3, s4, s8 +; GFX9-NEXT: s_sub_i32 s8, 0, s2 +; GFX9-NEXT: s_abs_i32 s4, s4 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s4, s4, s8 -; GFX9-NEXT: s_sub_i32 s8, 0, s3 +; GFX9-NEXT: s_ashr_i32 s3, s3, 31 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX9-NEXT: v_readfirstlane_b32 s12, v0 @@ -2007,108 +1982,99 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x ; GFX9-NEXT: s_mul_hi_u32 s8, s12, s8 ; GFX9-NEXT: s_add_i32 s12, s12, s8 ; GFX9-NEXT: s_mul_hi_u32 s8, s4, s12 -; GFX9-NEXT: s_mul_i32 s12, s8, s3 +; GFX9-NEXT: s_mul_i32 s12, s8, s2 ; GFX9-NEXT: s_sub_i32 s4, s4, s12 ; GFX9-NEXT: s_add_i32 s13, s8, 1 -; GFX9-NEXT: s_sub_i32 s12, s4, s3 -; GFX9-NEXT: s_cmp_ge_u32 s4, s3 +; GFX9-NEXT: s_sub_i32 s12, s4, s2 +; GFX9-NEXT: s_cmp_ge_u32 s4, s2 ; GFX9-NEXT: s_cselect_b32 s8, s13, s8 ; GFX9-NEXT: s_cselect_b32 s4, s12, s4 ; GFX9-NEXT: s_add_i32 s12, s8, 1 -; GFX9-NEXT: s_cmp_ge_u32 s4, s3 -; GFX9-NEXT: s_cselect_b32 s3, s12, s8 -; GFX9-NEXT: s_ashr_i32 s4, s9, 31 -; GFX9-NEXT: s_add_i32 s8, s9, s4 -; GFX9-NEXT: s_xor_b32 s8, s8, s4 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s8 -; GFX9-NEXT: s_ashr_i32 s9, s5, 31 -; GFX9-NEXT: s_xor_b32 s3, s3, s2 -; GFX9-NEXT: s_add_i32 s5, s5, s9 +; GFX9-NEXT: s_cmp_ge_u32 s4, s2 +; GFX9-NEXT: s_cselect_b32 s2, s12, s8 +; GFX9-NEXT: s_abs_i32 s4, s9 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4 +; GFX9-NEXT: s_xor_b32 s2, s2, s3 +; GFX9-NEXT: s_xor_b32 s8, s5, s9 +; GFX9-NEXT: s_sub_i32 s9, 0, s4 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s4, s9, s4 -; GFX9-NEXT: s_sub_i32 s2, s3, s2 -; GFX9-NEXT: s_xor_b32 s3, s5, s9 +; GFX9-NEXT: s_sub_i32 s2, s2, s3 +; GFX9-NEXT: s_abs_i32 s5, s5 +; GFX9-NEXT: s_ashr_i32 s8, s8, 31 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: s_sub_i32 s5, 0, s8 -; GFX9-NEXT: v_readfirstlane_b32 s9, v0 -; GFX9-NEXT: s_mul_i32 s5, s5, s9 -; GFX9-NEXT: s_mul_hi_u32 s5, s9, s5 -; GFX9-NEXT: s_add_i32 s9, s9, s5 -; GFX9-NEXT: s_mul_hi_u32 s5, s3, s9 -; GFX9-NEXT: s_mul_i32 s9, s5, s8 -; GFX9-NEXT: s_sub_i32 s3, s3, s9 -; GFX9-NEXT: s_add_i32 s12, s5, 1 -; GFX9-NEXT: s_sub_i32 s9, s3, s8 -; GFX9-NEXT: s_cmp_ge_u32 s3, s8 -; GFX9-NEXT: s_cselect_b32 s5, s12, s5 +; GFX9-NEXT: v_readfirstlane_b32 s3, v0 +; GFX9-NEXT: s_mul_i32 s9, s9, s3 +; GFX9-NEXT: s_mul_hi_u32 s9, s3, s9 +; GFX9-NEXT: s_add_i32 s3, s3, s9 +; GFX9-NEXT: s_mul_hi_u32 s3, s5, s3 +; GFX9-NEXT: s_mul_i32 s9, s3, s4 +; GFX9-NEXT: s_sub_i32 s5, s5, s9 +; GFX9-NEXT: s_add_i32 s12, s3, 1 +; GFX9-NEXT: s_sub_i32 s9, s5, s4 +; GFX9-NEXT: s_cmp_ge_u32 s5, s4 +; GFX9-NEXT: s_cselect_b32 s3, s12, s3 +; GFX9-NEXT: s_cselect_b32 s5, s9, s5 +; GFX9-NEXT: s_add_i32 s9, s3, 1 +; GFX9-NEXT: s_cmp_ge_u32 s5, s4 ; GFX9-NEXT: s_cselect_b32 s3, s9, s3 -; GFX9-NEXT: s_add_i32 s9, s5, 1 -; GFX9-NEXT: s_cmp_ge_u32 s3, s8 -; GFX9-NEXT: s_cselect_b32 s3, s9, s5 -; GFX9-NEXT: s_ashr_i32 s5, s10, 31 -; GFX9-NEXT: s_add_i32 s8, s10, s5 -; GFX9-NEXT: s_xor_b32 s8, s8, s5 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s8 -; GFX9-NEXT: s_ashr_i32 s9, s6, 31 -; GFX9-NEXT: s_xor_b32 s3, s3, s4 -; GFX9-NEXT: s_add_i32 s6, s6, s9 +; GFX9-NEXT: s_abs_i32 s4, s10 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4 +; GFX9-NEXT: s_xor_b32 s3, s3, s8 +; GFX9-NEXT: s_sub_i32 s9, 0, s4 +; GFX9-NEXT: s_sub_i32 s3, s3, s8 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s5, s9, s5 -; GFX9-NEXT: s_sub_i32 s3, s3, s4 -; GFX9-NEXT: s_xor_b32 s4, s6, s9 +; GFX9-NEXT: s_xor_b32 s5, s6, s10 +; GFX9-NEXT: s_abs_i32 s6, s6 +; GFX9-NEXT: s_ashr_i32 s5, s5, 31 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: s_sub_i32 s6, 0, s8 -; GFX9-NEXT: v_readfirstlane_b32 s9, v0 -; GFX9-NEXT: s_mul_i32 s6, s6, s9 -; GFX9-NEXT: s_mul_hi_u32 s6, s9, s6 -; GFX9-NEXT: s_add_i32 s9, s9, s6 -; GFX9-NEXT: s_mul_hi_u32 s6, s4, s9 -; GFX9-NEXT: s_mul_i32 s9, s6, s8 -; GFX9-NEXT: s_sub_i32 s4, s4, s9 -; GFX9-NEXT: s_add_i32 s10, s6, 1 -; GFX9-NEXT: s_sub_i32 s9, s4, s8 -; GFX9-NEXT: s_cmp_ge_u32 s4, s8 -; GFX9-NEXT: s_cselect_b32 s6, s10, s6 -; GFX9-NEXT: s_cselect_b32 s4, s9, s4 -; GFX9-NEXT: s_add_i32 s9, s6, 1 -; GFX9-NEXT: s_cmp_ge_u32 s4, s8 -; GFX9-NEXT: s_cselect_b32 s4, s9, s6 -; GFX9-NEXT: s_ashr_i32 s6, s11, 31 -; GFX9-NEXT: s_add_i32 s8, s11, s6 -; GFX9-NEXT: s_xor_b32 s8, s8, s6 -; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s8 -; GFX9-NEXT: v_mov_b32_e32 v0, s2 -; GFX9-NEXT: s_ashr_i32 s2, s7, 31 +; GFX9-NEXT: v_mov_b32_e32 v1, s3 +; GFX9-NEXT: v_readfirstlane_b32 s8, v0 +; GFX9-NEXT: s_mul_i32 s9, s9, s8 +; GFX9-NEXT: s_mul_hi_u32 s9, s8, s9 +; GFX9-NEXT: s_add_i32 s8, s8, s9 +; GFX9-NEXT: s_mul_hi_u32 s8, s6, s8 +; GFX9-NEXT: s_mul_i32 s9, s8, s4 +; GFX9-NEXT: s_sub_i32 s6, s6, s9 +; GFX9-NEXT: s_add_i32 s10, s8, 1 +; GFX9-NEXT: s_sub_i32 s9, s6, s4 +; GFX9-NEXT: s_cmp_ge_u32 s6, s4 +; GFX9-NEXT: s_cselect_b32 s8, s10, s8 +; GFX9-NEXT: s_cselect_b32 s6, s9, s6 +; GFX9-NEXT: s_add_i32 s9, s8, 1 +; GFX9-NEXT: s_cmp_ge_u32 s6, s4 +; GFX9-NEXT: s_cselect_b32 s4, s9, s8 +; GFX9-NEXT: s_abs_i32 s6, s11 +; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s6 ; GFX9-NEXT: s_xor_b32 s4, s4, s5 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 -; GFX9-NEXT: s_add_i32 s7, s7, s2 -; GFX9-NEXT: s_xor_b32 s6, s2, s6 +; GFX9-NEXT: v_mov_b32_e32 v0, s2 +; GFX9-NEXT: s_xor_b32 s2, s7, s11 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v2, v2 +; GFX9-NEXT: s_abs_i32 s3, s7 +; GFX9-NEXT: s_sub_i32 s7, 0, s6 ; GFX9-NEXT: s_sub_i32 s4, s4, s5 -; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 -; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX9-NEXT: s_xor_b32 s2, s7, s2 -; GFX9-NEXT: s_sub_i32 s5, 0, s8 -; GFX9-NEXT: v_mov_b32_e32 v2, s4 -; GFX9-NEXT: v_readfirstlane_b32 s7, v1 -; GFX9-NEXT: s_mul_i32 s5, s5, s7 -; GFX9-NEXT: s_mul_hi_u32 s5, s7, s5 -; GFX9-NEXT: s_add_i32 s7, s7, s5 -; GFX9-NEXT: s_mul_hi_u32 s5, s2, s7 -; GFX9-NEXT: s_mul_i32 s7, s5, s8 -; GFX9-NEXT: s_sub_i32 s2, s2, s7 -; GFX9-NEXT: s_add_i32 s9, s5, 1 -; GFX9-NEXT: s_sub_i32 s7, s2, s8 -; GFX9-NEXT: s_cmp_ge_u32 s2, s8 -; GFX9-NEXT: s_cselect_b32 s5, s9, s5 -; GFX9-NEXT: s_cselect_b32 s2, s7, s2 +; GFX9-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 +; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GFX9-NEXT: s_ashr_i32 s2, s2, 31 +; GFX9-NEXT: v_readfirstlane_b32 s5, v2 +; GFX9-NEXT: s_mul_i32 s7, s7, s5 +; GFX9-NEXT: s_mul_hi_u32 s7, s5, s7 +; GFX9-NEXT: s_add_i32 s5, s5, s7 +; GFX9-NEXT: s_mul_hi_u32 s5, s3, s5 +; GFX9-NEXT: s_mul_i32 s7, s5, s6 +; GFX9-NEXT: s_sub_i32 s3, s3, s7 +; GFX9-NEXT: s_add_i32 s8, s5, 1 +; GFX9-NEXT: s_sub_i32 s7, s3, s6 +; GFX9-NEXT: s_cmp_ge_u32 s3, s6 +; GFX9-NEXT: s_cselect_b32 s5, s8, s5 +; GFX9-NEXT: s_cselect_b32 s3, s7, s3 ; GFX9-NEXT: s_add_i32 s7, s5, 1 -; GFX9-NEXT: s_cmp_ge_u32 s2, s8 -; GFX9-NEXT: s_cselect_b32 s2, s7, s5 -; GFX9-NEXT: s_xor_b32 s2, s2, s6 -; GFX9-NEXT: s_sub_i32 s2, s2, s6 -; GFX9-NEXT: v_mov_b32_e32 v1, s3 +; GFX9-NEXT: s_cmp_ge_u32 s3, s6 +; GFX9-NEXT: s_cselect_b32 s3, s7, s5 +; GFX9-NEXT: s_xor_b32 s3, s3, s2 +; GFX9-NEXT: s_sub_i32 s2, s3, s2 +; GFX9-NEXT: v_mov_b32_e32 v2, s4 ; GFX9-NEXT: v_mov_b32_e32 v3, s2 ; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; GFX9-NEXT: s_endpgm @@ -2279,116 +2245,104 @@ define amdgpu_kernel void @srem_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x ; GFX6-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd ; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_ashr_i32 s2, s8, 31 -; GFX6-NEXT: s_add_i32 s3, s8, s2 -; GFX6-NEXT: s_xor_b32 s2, s3, s2 +; GFX6-NEXT: s_abs_i32 s2, s8 ; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GFX6-NEXT: s_sub_i32 s3, 0, s2 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_lo_u32 v1, s3, v0 -; GFX6-NEXT: s_ashr_i32 s3, s4, 31 -; GFX6-NEXT: s_add_i32 s4, s4, s3 -; GFX6-NEXT: s_xor_b32 s4, s4, s3 +; GFX6-NEXT: s_abs_i32 s3, s4 +; GFX6-NEXT: s_ashr_i32 s4, s4, 31 ; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_mul_hi_u32 v0, s4, v0 +; GFX6-NEXT: v_mul_hi_u32 v0, s3, v0 ; GFX6-NEXT: v_readfirstlane_b32 s8, v0 ; GFX6-NEXT: s_mul_i32 s8, s8, s2 -; GFX6-NEXT: s_sub_i32 s4, s4, s8 -; GFX6-NEXT: s_sub_i32 s8, s4, s2 -; GFX6-NEXT: s_cmp_ge_u32 s4, s2 -; GFX6-NEXT: s_cselect_b32 s4, s8, s4 -; GFX6-NEXT: s_sub_i32 s8, s4, s2 -; GFX6-NEXT: s_cmp_ge_u32 s4, s2 -; GFX6-NEXT: s_cselect_b32 s2, s8, s4 -; GFX6-NEXT: s_ashr_i32 s4, s9, 31 -; GFX6-NEXT: s_add_i32 s8, s9, s4 -; GFX6-NEXT: s_xor_b32 s4, s8, s4 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s4 -; GFX6-NEXT: s_sub_i32 s8, 0, s4 -; GFX6-NEXT: s_xor_b32 s2, s2, s3 -; GFX6-NEXT: s_sub_i32 s9, s2, s3 +; GFX6-NEXT: s_sub_i32 s3, s3, s8 +; GFX6-NEXT: s_sub_i32 s8, s3, s2 +; GFX6-NEXT: s_cmp_ge_u32 s3, s2 +; GFX6-NEXT: s_cselect_b32 s3, s8, s3 +; GFX6-NEXT: s_sub_i32 s8, s3, s2 +; GFX6-NEXT: s_cmp_ge_u32 s3, s2 +; GFX6-NEXT: s_cselect_b32 s2, s8, s3 +; GFX6-NEXT: s_abs_i32 s3, s9 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3 +; GFX6-NEXT: s_sub_i32 s8, 0, s3 +; GFX6-NEXT: s_xor_b32 s2, s2, s4 +; GFX6-NEXT: s_sub_i32 s4, s2, s4 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_lo_u32 v1, s8, v0 -; GFX6-NEXT: s_ashr_i32 s8, s5, 31 -; GFX6-NEXT: s_add_i32 s5, s5, s8 -; GFX6-NEXT: s_xor_b32 s5, s5, s8 +; GFX6-NEXT: s_abs_i32 s8, s5 +; GFX6-NEXT: s_ashr_i32 s5, s5, 31 ; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_mul_hi_u32 v0, s5, v0 +; GFX6-NEXT: v_mul_hi_u32 v0, s8, v0 ; GFX6-NEXT: v_readfirstlane_b32 s2, v0 -; GFX6-NEXT: s_mul_i32 s2, s2, s4 -; GFX6-NEXT: s_sub_i32 s2, s5, s2 -; GFX6-NEXT: s_sub_i32 s3, s2, s4 -; GFX6-NEXT: s_cmp_ge_u32 s2, s4 -; GFX6-NEXT: s_cselect_b32 s2, s3, s2 -; GFX6-NEXT: s_sub_i32 s3, s2, s4 -; GFX6-NEXT: s_cmp_ge_u32 s2, s4 -; GFX6-NEXT: s_cselect_b32 s2, s3, s2 -; GFX6-NEXT: s_ashr_i32 s3, s10, 31 -; GFX6-NEXT: s_add_i32 s4, s10, s3 -; GFX6-NEXT: s_xor_b32 s3, s4, s3 +; GFX6-NEXT: s_mul_i32 s2, s2, s3 +; GFX6-NEXT: s_sub_i32 s2, s8, s2 +; GFX6-NEXT: s_sub_i32 s8, s2, s3 +; GFX6-NEXT: s_cmp_ge_u32 s2, s3 +; GFX6-NEXT: s_cselect_b32 s2, s8, s2 +; GFX6-NEXT: s_sub_i32 s8, s2, s3 +; GFX6-NEXT: s_cmp_ge_u32 s2, s3 +; GFX6-NEXT: s_cselect_b32 s2, s8, s2 +; GFX6-NEXT: s_abs_i32 s3, s10 ; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3 -; GFX6-NEXT: s_sub_i32 s4, 0, s3 -; GFX6-NEXT: s_xor_b32 s2, s2, s8 +; GFX6-NEXT: s_sub_i32 s8, 0, s3 +; GFX6-NEXT: s_xor_b32 s2, s2, s5 +; GFX6-NEXT: s_sub_i32 s5, s2, s5 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX6-NEXT: v_mul_lo_u32 v1, s4, v0 -; GFX6-NEXT: s_ashr_i32 s4, s6, 31 -; GFX6-NEXT: s_add_i32 s5, s6, s4 -; GFX6-NEXT: s_xor_b32 s5, s5, s4 +; GFX6-NEXT: v_mul_lo_u32 v1, s8, v0 +; GFX6-NEXT: s_abs_i32 s8, s6 +; GFX6-NEXT: s_ashr_i32 s6, s6, 31 ; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 -; GFX6-NEXT: s_sub_i32 s6, s2, s8 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_mul_hi_u32 v0, s5, v0 +; GFX6-NEXT: v_mul_hi_u32 v0, s8, v0 ; GFX6-NEXT: v_readfirstlane_b32 s2, v0 ; GFX6-NEXT: s_mul_i32 s2, s2, s3 -; GFX6-NEXT: s_sub_i32 s2, s5, s2 -; GFX6-NEXT: s_sub_i32 s5, s2, s3 +; GFX6-NEXT: s_sub_i32 s2, s8, s2 +; GFX6-NEXT: s_sub_i32 s8, s2, s3 ; GFX6-NEXT: s_cmp_ge_u32 s2, s3 -; GFX6-NEXT: s_cselect_b32 s2, s5, s2 -; GFX6-NEXT: s_sub_i32 s5, s2, s3 +; GFX6-NEXT: s_cselect_b32 s2, s8, s2 +; GFX6-NEXT: s_sub_i32 s8, s2, s3 ; GFX6-NEXT: s_cmp_ge_u32 s2, s3 -; GFX6-NEXT: s_cselect_b32 s5, s5, s2 -; GFX6-NEXT: s_ashr_i32 s2, s11, 31 -; GFX6-NEXT: s_add_i32 s3, s11, s2 -; GFX6-NEXT: s_xor_b32 s8, s3, s2 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s8 -; GFX6-NEXT: s_sub_i32 s10, 0, s8 -; GFX6-NEXT: s_xor_b32 s5, s5, s4 -; GFX6-NEXT: s_sub_i32 s4, s5, s4 -; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 +; GFX6-NEXT: s_cselect_b32 s8, s8, s2 +; GFX6-NEXT: s_abs_i32 s9, s11 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s9 +; GFX6-NEXT: s_sub_i32 s2, 0, s9 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 -; GFX6-NEXT: s_mov_b32 s2, -1 +; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 -; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v0 -; GFX6-NEXT: v_mov_b32_e32 v0, s9 -; GFX6-NEXT: s_ashr_i32 s9, s7, 31 -; GFX6-NEXT: s_add_i32 s7, s7, s9 -; GFX6-NEXT: v_mul_lo_u32 v2, s10, v1 -; GFX6-NEXT: s_xor_b32 s7, s7, s9 -; GFX6-NEXT: v_mul_hi_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2 -; GFX6-NEXT: v_mul_hi_u32 v2, s7, v1 -; GFX6-NEXT: v_mov_b32_e32 v1, s6 -; GFX6-NEXT: v_readfirstlane_b32 s5, v2 -; GFX6-NEXT: s_mul_i32 s5, s5, s8 -; GFX6-NEXT: s_sub_i32 s5, s7, s5 -; GFX6-NEXT: s_sub_i32 s6, s5, s8 -; GFX6-NEXT: s_cmp_ge_u32 s5, s8 -; GFX6-NEXT: s_cselect_b32 s5, s6, s5 -; GFX6-NEXT: s_sub_i32 s6, s5, s8 -; GFX6-NEXT: s_cmp_ge_u32 s5, s8 -; GFX6-NEXT: s_cselect_b32 s5, s6, s5 -; GFX6-NEXT: s_xor_b32 s5, s5, s9 -; GFX6-NEXT: s_sub_i32 s5, s5, s9 -; GFX6-NEXT: v_mov_b32_e32 v2, s4 -; GFX6-NEXT: v_mov_b32_e32 v3, s5 +; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v0 +; GFX6-NEXT: v_mov_b32_e32 v0, s4 +; GFX6-NEXT: s_abs_i32 s4, s7 +; GFX6-NEXT: v_mul_lo_u32 v1, s2, v2 +; GFX6-NEXT: s_mov_b32 s2, -1 +; GFX6-NEXT: v_mul_hi_u32 v3, v2, v1 +; GFX6-NEXT: v_mov_b32_e32 v1, s5 +; GFX6-NEXT: s_ashr_i32 s5, s7, 31 +; GFX6-NEXT: s_xor_b32 s7, s8, s6 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GFX6-NEXT: v_mul_hi_u32 v2, s4, v2 +; GFX6-NEXT: s_sub_i32 s6, s7, s6 +; GFX6-NEXT: v_readfirstlane_b32 s7, v2 +; GFX6-NEXT: s_mul_i32 s7, s7, s9 +; GFX6-NEXT: s_sub_i32 s4, s4, s7 +; GFX6-NEXT: s_sub_i32 s7, s4, s9 +; GFX6-NEXT: s_cmp_ge_u32 s4, s9 +; GFX6-NEXT: s_cselect_b32 s4, s7, s4 +; GFX6-NEXT: s_sub_i32 s7, s4, s9 +; GFX6-NEXT: s_cmp_ge_u32 s4, s9 +; GFX6-NEXT: s_cselect_b32 s4, s7, s4 +; GFX6-NEXT: s_xor_b32 s4, s4, s5 +; GFX6-NEXT: s_sub_i32 s4, s4, s5 +; GFX6-NEXT: v_mov_b32_e32 v2, s6 +; GFX6-NEXT: v_mov_b32_e32 v3, s4 ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; GFX6-NEXT: s_endpgm ; @@ -2398,15 +2352,12 @@ define amdgpu_kernel void @srem_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x ; GFX9-NEXT: v_mov_b32_e32 v4, 0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_ashr_i32 s2, s8, 31 -; GFX9-NEXT: s_add_i32 s3, s8, s2 -; GFX9-NEXT: s_xor_b32 s2, s3, s2 +; GFX9-NEXT: s_abs_i32 s2, s8 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GFX9-NEXT: s_sub_i32 s8, 0, s2 ; GFX9-NEXT: s_ashr_i32 s3, s4, 31 -; GFX9-NEXT: s_add_i32 s4, s4, s3 +; GFX9-NEXT: s_abs_i32 s4, s4 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s4, s4, s3 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX9-NEXT: v_readfirstlane_b32 s12, v0 @@ -2422,51 +2373,44 @@ define amdgpu_kernel void @srem_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x ; GFX9-NEXT: s_sub_i32 s8, s4, s2 ; GFX9-NEXT: s_cmp_ge_u32 s4, s2 ; GFX9-NEXT: s_cselect_b32 s2, s8, s4 -; GFX9-NEXT: s_ashr_i32 s4, s9, 31 -; GFX9-NEXT: s_add_i32 s8, s9, s4 -; GFX9-NEXT: s_xor_b32 s4, s8, s4 +; GFX9-NEXT: s_abs_i32 s4, s9 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4 -; GFX9-NEXT: s_ashr_i32 s8, s5, 31 ; GFX9-NEXT: s_xor_b32 s2, s2, s3 -; GFX9-NEXT: s_add_i32 s5, s5, s8 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 +; GFX9-NEXT: s_sub_i32 s9, 0, s4 ; GFX9-NEXT: s_sub_i32 s2, s2, s3 -; GFX9-NEXT: s_xor_b32 s3, s5, s8 -; GFX9-NEXT: s_sub_i32 s5, 0, s4 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 +; GFX9-NEXT: s_ashr_i32 s8, s5, 31 +; GFX9-NEXT: s_abs_i32 s5, s5 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: v_readfirstlane_b32 s9, v0 -; GFX9-NEXT: s_mul_i32 s5, s5, s9 -; GFX9-NEXT: s_mul_hi_u32 s5, s9, s5 -; GFX9-NEXT: s_add_i32 s9, s9, s5 -; GFX9-NEXT: s_mul_hi_u32 s5, s3, s9 -; GFX9-NEXT: s_mul_i32 s5, s5, s4 -; GFX9-NEXT: s_sub_i32 s3, s3, s5 +; GFX9-NEXT: v_readfirstlane_b32 s3, v0 +; GFX9-NEXT: s_mul_i32 s9, s9, s3 +; GFX9-NEXT: s_mul_hi_u32 s9, s3, s9 +; GFX9-NEXT: s_add_i32 s3, s3, s9 +; GFX9-NEXT: s_mul_hi_u32 s3, s5, s3 +; GFX9-NEXT: s_mul_i32 s3, s3, s4 +; GFX9-NEXT: s_sub_i32 s3, s5, s3 ; GFX9-NEXT: s_sub_i32 s5, s3, s4 ; GFX9-NEXT: s_cmp_ge_u32 s3, s4 ; GFX9-NEXT: s_cselect_b32 s3, s5, s3 ; GFX9-NEXT: s_sub_i32 s5, s3, s4 ; GFX9-NEXT: s_cmp_ge_u32 s3, s4 ; GFX9-NEXT: s_cselect_b32 s3, s5, s3 -; GFX9-NEXT: s_ashr_i32 s4, s10, 31 -; GFX9-NEXT: s_add_i32 s5, s10, s4 -; GFX9-NEXT: s_xor_b32 s4, s5, s4 +; GFX9-NEXT: s_abs_i32 s4, s10 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4 ; GFX9-NEXT: s_xor_b32 s3, s3, s8 +; GFX9-NEXT: s_sub_i32 s9, 0, s4 ; GFX9-NEXT: s_sub_i32 s3, s3, s8 -; GFX9-NEXT: s_sub_i32 s8, 0, s4 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX9-NEXT: s_ashr_i32 s5, s6, 31 -; GFX9-NEXT: s_add_i32 s6, s6, s5 -; GFX9-NEXT: s_xor_b32 s6, s6, s5 +; GFX9-NEXT: s_abs_i32 s6, s6 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: v_mov_b32_e32 v1, s3 -; GFX9-NEXT: v_readfirstlane_b32 s9, v0 -; GFX9-NEXT: s_mul_i32 s8, s8, s9 -; GFX9-NEXT: s_mul_hi_u32 s8, s9, s8 -; GFX9-NEXT: s_add_i32 s9, s9, s8 -; GFX9-NEXT: s_mul_hi_u32 s8, s6, s9 +; GFX9-NEXT: v_readfirstlane_b32 s8, v0 +; GFX9-NEXT: s_mul_i32 s9, s9, s8 +; GFX9-NEXT: s_mul_hi_u32 s9, s8, s9 +; GFX9-NEXT: s_add_i32 s8, s8, s9 +; GFX9-NEXT: s_mul_hi_u32 s8, s6, s8 ; GFX9-NEXT: s_mul_i32 s8, s8, s4 ; GFX9-NEXT: s_sub_i32 s6, s6, s8 ; GFX9-NEXT: s_sub_i32 s8, s6, s4 @@ -2475,36 +2419,34 @@ define amdgpu_kernel void @srem_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x ; GFX9-NEXT: s_sub_i32 s8, s6, s4 ; GFX9-NEXT: s_cmp_ge_u32 s6, s4 ; GFX9-NEXT: s_cselect_b32 s4, s8, s6 -; GFX9-NEXT: s_ashr_i32 s6, s11, 31 -; GFX9-NEXT: s_add_i32 s8, s11, s6 -; GFX9-NEXT: s_xor_b32 s6, s8, s6 -; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s6 +; GFX9-NEXT: s_abs_i32 s6, s11 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s6 +; GFX9-NEXT: s_xor_b32 s4, s4, s5 ; GFX9-NEXT: v_mov_b32_e32 v0, s2 ; GFX9-NEXT: s_ashr_i32 s2, s7, 31 -; GFX9-NEXT: s_xor_b32 s3, s4, s5 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v2, v2 -; GFX9-NEXT: s_add_i32 s4, s7, s2 -; GFX9-NEXT: s_sub_i32 s3, s3, s5 -; GFX9-NEXT: s_sub_i32 s5, 0, s6 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v2, v1 +; GFX9-NEXT: v_mov_b32_e32 v1, s3 +; GFX9-NEXT: s_abs_i32 s3, s7 +; GFX9-NEXT: s_sub_i32 s7, 0, s6 ; GFX9-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GFX9-NEXT: s_xor_b32 s4, s4, s2 -; GFX9-NEXT: v_readfirstlane_b32 s7, v2 -; GFX9-NEXT: s_mul_i32 s5, s5, s7 -; GFX9-NEXT: s_mul_hi_u32 s5, s7, s5 -; GFX9-NEXT: s_add_i32 s7, s7, s5 -; GFX9-NEXT: s_mul_hi_u32 s5, s4, s7 -; GFX9-NEXT: s_mul_i32 s5, s5, s6 ; GFX9-NEXT: s_sub_i32 s4, s4, s5 -; GFX9-NEXT: s_sub_i32 s5, s4, s6 -; GFX9-NEXT: s_cmp_ge_u32 s4, s6 -; GFX9-NEXT: s_cselect_b32 s4, s5, s4 -; GFX9-NEXT: s_sub_i32 s5, s4, s6 -; GFX9-NEXT: s_cmp_ge_u32 s4, s6 -; GFX9-NEXT: s_cselect_b32 s4, s5, s4 -; GFX9-NEXT: s_xor_b32 s4, s4, s2 -; GFX9-NEXT: s_sub_i32 s2, s4, s2 -; GFX9-NEXT: v_mov_b32_e32 v2, s3 +; GFX9-NEXT: v_readfirstlane_b32 s5, v2 +; GFX9-NEXT: s_mul_i32 s7, s7, s5 +; GFX9-NEXT: s_mul_hi_u32 s7, s5, s7 +; GFX9-NEXT: s_add_i32 s5, s5, s7 +; GFX9-NEXT: s_mul_hi_u32 s5, s3, s5 +; GFX9-NEXT: s_mul_i32 s5, s5, s6 +; GFX9-NEXT: s_sub_i32 s3, s3, s5 +; GFX9-NEXT: s_sub_i32 s5, s3, s6 +; GFX9-NEXT: s_cmp_ge_u32 s3, s6 +; GFX9-NEXT: s_cselect_b32 s3, s5, s3 +; GFX9-NEXT: s_sub_i32 s5, s3, s6 +; GFX9-NEXT: s_cmp_ge_u32 s3, s6 +; GFX9-NEXT: s_cselect_b32 s3, s5, s3 +; GFX9-NEXT: s_xor_b32 s3, s3, s2 +; GFX9-NEXT: s_sub_i32 s2, s3, s2 +; GFX9-NEXT: v_mov_b32_e32 v2, s4 ; GFX9-NEXT: v_mov_b32_e32 v3, s2 ; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; GFX9-NEXT: s_endpgm @@ -6538,71 +6480,65 @@ define amdgpu_kernel void @sdiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x ; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: s_lshl_b32 s2, 0x1000, s6 -; GFX6-NEXT: s_ashr_i32 s3, s2, 31 -; GFX6-NEXT: s_add_i32 s2, s2, s3 -; GFX6-NEXT: s_xor_b32 s2, s2, s3 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2 -; GFX6-NEXT: s_sub_i32 s6, 0, s2 +; GFX6-NEXT: s_abs_i32 s3, s2 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3 +; GFX6-NEXT: s_sub_i32 s6, 0, s3 +; GFX6-NEXT: s_xor_b32 s2, s4, s2 ; GFX6-NEXT: s_lshl_b32 s7, 0x1000, s7 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_lo_u32 v1, s6, v0 -; GFX6-NEXT: s_ashr_i32 s6, s4, 31 -; GFX6-NEXT: s_add_i32 s4, s4, s6 -; GFX6-NEXT: s_xor_b32 s4, s4, s6 +; GFX6-NEXT: s_abs_i32 s6, s4 +; GFX6-NEXT: s_ashr_i32 s4, s2, 31 ; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 -; GFX6-NEXT: s_xor_b32 s6, s6, s3 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_mul_hi_u32 v0, s4, v0 -; GFX6-NEXT: v_readfirstlane_b32 s3, v0 -; GFX6-NEXT: s_mul_i32 s3, s3, s2 -; GFX6-NEXT: s_sub_i32 s3, s4, s3 -; GFX6-NEXT: s_sub_i32 s4, s3, s2 +; GFX6-NEXT: v_mul_hi_u32 v0, s6, v0 +; GFX6-NEXT: v_readfirstlane_b32 s2, v0 +; GFX6-NEXT: s_mul_i32 s2, s2, s3 +; GFX6-NEXT: s_sub_i32 s2, s6, s2 +; GFX6-NEXT: s_sub_i32 s6, s2, s3 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 -; GFX6-NEXT: s_cmp_ge_u32 s3, s2 +; GFX6-NEXT: s_cmp_ge_u32 s2, s3 ; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GFX6-NEXT: s_cselect_b32 s3, s4, s3 +; GFX6-NEXT: s_cselect_b32 s2, s6, s2 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 -; GFX6-NEXT: s_cmp_ge_u32 s3, s2 +; GFX6-NEXT: s_cmp_ge_u32 s2, s3 ; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 -; GFX6-NEXT: s_ashr_i32 s4, s7, 31 -; GFX6-NEXT: s_add_i32 s7, s7, s4 -; GFX6-NEXT: s_xor_b32 s7, s7, s4 -; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s7 -; GFX6-NEXT: s_sub_i32 s8, 0, s7 +; GFX6-NEXT: s_abs_i32 s6, s7 +; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s6 +; GFX6-NEXT: s_sub_i32 s2, 0, s6 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GFX6-NEXT: v_xor_b32_e32 v0, s6, v0 +; GFX6-NEXT: s_xor_b32 s7, s5, s7 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v2, v2 -; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s6, v0 -; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: s_abs_i32 s5, s5 +; GFX6-NEXT: v_xor_b32_e32 v0, s4, v0 +; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s4, v0 ; GFX6-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GFX6-NEXT: s_ashr_i32 s7, s7, 31 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: v_mul_lo_u32 v3, s2, v2 ; GFX6-NEXT: s_mov_b32 s2, -1 -; GFX6-NEXT: v_mul_lo_u32 v3, s8, v2 -; GFX6-NEXT: s_ashr_i32 s8, s5, 31 -; GFX6-NEXT: s_add_i32 s5, s5, s8 -; GFX6-NEXT: s_xor_b32 s5, s5, s8 ; GFX6-NEXT: v_mul_hi_u32 v1, v2, v3 -; GFX6-NEXT: s_xor_b32 s4, s8, s4 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1 ; GFX6-NEXT: v_mul_hi_u32 v1, s5, v1 -; GFX6-NEXT: v_readfirstlane_b32 s6, v1 -; GFX6-NEXT: s_mul_i32 s6, s6, s7 -; GFX6-NEXT: s_sub_i32 s5, s5, s6 -; GFX6-NEXT: s_sub_i32 s6, s5, s7 +; GFX6-NEXT: v_readfirstlane_b32 s4, v1 +; GFX6-NEXT: s_mul_i32 s4, s4, s6 +; GFX6-NEXT: s_sub_i32 s4, s5, s4 +; GFX6-NEXT: s_sub_i32 s5, s4, s6 ; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v1 -; GFX6-NEXT: s_cmp_ge_u32 s5, s7 +; GFX6-NEXT: s_cmp_ge_u32 s4, s6 ; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 ; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc -; GFX6-NEXT: s_cselect_b32 s5, s6, s5 +; GFX6-NEXT: s_cselect_b32 s4, s5, s4 ; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v1 -; GFX6-NEXT: s_cmp_ge_u32 s5, s7 +; GFX6-NEXT: s_cmp_ge_u32 s4, s6 ; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 ; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc -; GFX6-NEXT: v_xor_b32_e32 v1, s4, v1 -; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, s4, v1 +; GFX6-NEXT: v_xor_b32_e32 v1, s7, v1 +; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, s7, v1 ; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; GFX6-NEXT: s_endpgm ; @@ -6613,65 +6549,59 @@ define amdgpu_kernel void @sdiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_lshl_b32 s2, 0x1000, s6 -; GFX9-NEXT: s_ashr_i32 s3, s2, 31 -; GFX9-NEXT: s_add_i32 s2, s2, s3 -; GFX9-NEXT: s_xor_b32 s2, s2, s3 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GFX9-NEXT: s_abs_i32 s3, s2 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 ; GFX9-NEXT: s_lshl_b32 s6, 0x1000, s7 -; GFX9-NEXT: s_ashr_i32 s7, s4, 31 -; GFX9-NEXT: s_add_i32 s4, s4, s7 +; GFX9-NEXT: s_abs_i32 s7, s4 +; GFX9-NEXT: s_xor_b32 s2, s4, s2 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s3, s7, s3 -; GFX9-NEXT: s_xor_b32 s4, s4, s7 -; GFX9-NEXT: s_sub_i32 s7, 0, s2 +; GFX9-NEXT: s_sub_i32 s4, 0, s3 +; GFX9-NEXT: s_ashr_i32 s2, s2, 31 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX9-NEXT: v_readfirstlane_b32 s8, v0 -; GFX9-NEXT: s_mul_i32 s7, s7, s8 -; GFX9-NEXT: s_mul_hi_u32 s7, s8, s7 -; GFX9-NEXT: s_add_i32 s8, s8, s7 -; GFX9-NEXT: s_mul_hi_u32 s7, s4, s8 -; GFX9-NEXT: s_mul_i32 s8, s7, s2 -; GFX9-NEXT: s_sub_i32 s4, s4, s8 -; GFX9-NEXT: s_add_i32 s9, s7, 1 -; GFX9-NEXT: s_sub_i32 s8, s4, s2 -; GFX9-NEXT: s_cmp_ge_u32 s4, s2 -; GFX9-NEXT: s_cselect_b32 s7, s9, s7 -; GFX9-NEXT: s_cselect_b32 s4, s8, s4 -; GFX9-NEXT: s_add_i32 s8, s7, 1 -; GFX9-NEXT: s_cmp_ge_u32 s4, s2 -; GFX9-NEXT: s_cselect_b32 s2, s8, s7 -; GFX9-NEXT: s_ashr_i32 s4, s6, 31 -; GFX9-NEXT: s_add_i32 s6, s6, s4 -; GFX9-NEXT: s_xor_b32 s6, s6, s4 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6 -; GFX9-NEXT: s_ashr_i32 s7, s5, 31 -; GFX9-NEXT: s_xor_b32 s2, s2, s3 -; GFX9-NEXT: s_add_i32 s5, s5, s7 +; GFX9-NEXT: s_mul_i32 s4, s4, s8 +; GFX9-NEXT: s_mul_hi_u32 s4, s8, s4 +; GFX9-NEXT: s_add_i32 s8, s8, s4 +; GFX9-NEXT: s_mul_hi_u32 s4, s7, s8 +; GFX9-NEXT: s_mul_i32 s8, s4, s3 +; GFX9-NEXT: s_sub_i32 s7, s7, s8 +; GFX9-NEXT: s_add_i32 s9, s4, 1 +; GFX9-NEXT: s_sub_i32 s8, s7, s3 +; GFX9-NEXT: s_cmp_ge_u32 s7, s3 +; GFX9-NEXT: s_cselect_b32 s4, s9, s4 +; GFX9-NEXT: s_cselect_b32 s7, s8, s7 +; GFX9-NEXT: s_add_i32 s8, s4, 1 +; GFX9-NEXT: s_cmp_ge_u32 s7, s3 +; GFX9-NEXT: s_cselect_b32 s3, s8, s4 +; GFX9-NEXT: s_abs_i32 s4, s6 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4 +; GFX9-NEXT: s_xor_b32 s3, s3, s2 +; GFX9-NEXT: s_sub_i32 s7, 0, s4 +; GFX9-NEXT: s_sub_i32 s2, s3, s2 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s4, s7, s4 -; GFX9-NEXT: s_sub_i32 s2, s2, s3 -; GFX9-NEXT: s_xor_b32 s3, s5, s7 +; GFX9-NEXT: s_xor_b32 s6, s5, s6 +; GFX9-NEXT: s_abs_i32 s5, s5 +; GFX9-NEXT: s_ashr_i32 s6, s6, 31 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: s_sub_i32 s5, 0, s6 -; GFX9-NEXT: v_readfirstlane_b32 s7, v0 -; GFX9-NEXT: s_mul_i32 s5, s5, s7 -; GFX9-NEXT: s_mul_hi_u32 s5, s7, s5 -; GFX9-NEXT: s_add_i32 s7, s7, s5 -; GFX9-NEXT: s_mul_hi_u32 s5, s3, s7 -; GFX9-NEXT: s_mul_i32 s7, s5, s6 -; GFX9-NEXT: s_sub_i32 s3, s3, s7 -; GFX9-NEXT: s_add_i32 s8, s5, 1 -; GFX9-NEXT: s_sub_i32 s7, s3, s6 -; GFX9-NEXT: s_cmp_ge_u32 s3, s6 -; GFX9-NEXT: s_cselect_b32 s5, s8, s5 +; GFX9-NEXT: v_readfirstlane_b32 s3, v0 +; GFX9-NEXT: s_mul_i32 s7, s7, s3 +; GFX9-NEXT: s_mul_hi_u32 s7, s3, s7 +; GFX9-NEXT: s_add_i32 s3, s3, s7 +; GFX9-NEXT: s_mul_hi_u32 s3, s5, s3 +; GFX9-NEXT: s_mul_i32 s7, s3, s4 +; GFX9-NEXT: s_sub_i32 s5, s5, s7 +; GFX9-NEXT: s_add_i32 s8, s3, 1 +; GFX9-NEXT: s_sub_i32 s7, s5, s4 +; GFX9-NEXT: s_cmp_ge_u32 s5, s4 +; GFX9-NEXT: s_cselect_b32 s3, s8, s3 +; GFX9-NEXT: s_cselect_b32 s5, s7, s5 +; GFX9-NEXT: s_add_i32 s7, s3, 1 +; GFX9-NEXT: s_cmp_ge_u32 s5, s4 ; GFX9-NEXT: s_cselect_b32 s3, s7, s3 -; GFX9-NEXT: s_add_i32 s7, s5, 1 -; GFX9-NEXT: s_cmp_ge_u32 s3, s6 -; GFX9-NEXT: s_cselect_b32 s3, s7, s5 -; GFX9-NEXT: s_xor_b32 s3, s3, s4 -; GFX9-NEXT: s_sub_i32 s3, s3, s4 +; GFX9-NEXT: s_xor_b32 s3, s3, s6 +; GFX9-NEXT: s_sub_i32 s3, s3, s6 ; GFX9-NEXT: v_mov_b32_e32 v0, s2 ; GFX9-NEXT: v_mov_b32_e32 v1, s3 ; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] @@ -7001,19 +6931,16 @@ define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x ; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: s_lshl_b32 s2, 0x1000, s6 -; GFX6-NEXT: s_ashr_i32 s3, s2, 31 -; GFX6-NEXT: s_add_i32 s2, s2, s3 -; GFX6-NEXT: s_xor_b32 s2, s2, s3 +; GFX6-NEXT: s_abs_i32 s2, s2 ; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GFX6-NEXT: s_sub_i32 s3, 0, s2 -; GFX6-NEXT: s_ashr_i32 s6, s4, 31 +; GFX6-NEXT: s_lshl_b32 s6, 0x1000, s7 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_lo_u32 v1, s3, v0 -; GFX6-NEXT: s_add_i32 s3, s4, s6 -; GFX6-NEXT: s_xor_b32 s3, s3, s6 -; GFX6-NEXT: s_lshl_b32 s4, 0x1000, s7 +; GFX6-NEXT: s_abs_i32 s3, s4 +; GFX6-NEXT: s_ashr_i32 s4, s4, 31 ; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ; GFX6-NEXT: v_mul_hi_u32 v0, s3, v0 @@ -7026,38 +6953,35 @@ define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x ; GFX6-NEXT: s_sub_i32 s7, s3, s2 ; GFX6-NEXT: s_cmp_ge_u32 s3, s2 ; GFX6-NEXT: s_cselect_b32 s7, s7, s3 -; GFX6-NEXT: s_ashr_i32 s2, s4, 31 -; GFX6-NEXT: s_add_i32 s4, s4, s2 -; GFX6-NEXT: s_xor_b32 s4, s4, s2 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s4 -; GFX6-NEXT: s_sub_i32 s2, 0, s4 -; GFX6-NEXT: s_ashr_i32 s8, s5, 31 -; GFX6-NEXT: s_xor_b32 s7, s7, s6 +; GFX6-NEXT: s_abs_i32 s6, s6 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6 +; GFX6-NEXT: s_sub_i32 s2, 0, s6 +; GFX6-NEXT: s_abs_i32 s8, s5 +; GFX6-NEXT: s_xor_b32 s7, s7, s4 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX6-NEXT: s_sub_i32 s6, s7, s6 +; GFX6-NEXT: s_sub_i32 s4, s7, s4 +; GFX6-NEXT: s_ashr_i32 s5, s5, 31 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_lo_u32 v1, s2, v0 -; GFX6-NEXT: s_add_i32 s2, s5, s8 -; GFX6-NEXT: s_xor_b32 s5, s2, s8 ; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_mul_hi_u32 v0, s5, v0 +; GFX6-NEXT: v_mul_hi_u32 v0, s8, v0 ; GFX6-NEXT: v_readfirstlane_b32 s7, v0 -; GFX6-NEXT: s_mul_i32 s7, s7, s4 -; GFX6-NEXT: s_sub_i32 s5, s5, s7 -; GFX6-NEXT: s_sub_i32 s7, s5, s4 -; GFX6-NEXT: s_cmp_ge_u32 s5, s4 -; GFX6-NEXT: s_cselect_b32 s5, s7, s5 -; GFX6-NEXT: s_sub_i32 s7, s5, s4 -; GFX6-NEXT: s_cmp_ge_u32 s5, s4 -; GFX6-NEXT: s_cselect_b32 s4, s7, s5 -; GFX6-NEXT: s_xor_b32 s4, s4, s8 -; GFX6-NEXT: s_sub_i32 s4, s4, s8 -; GFX6-NEXT: v_mov_b32_e32 v0, s6 -; GFX6-NEXT: v_mov_b32_e32 v1, s4 +; GFX6-NEXT: s_mul_i32 s7, s7, s6 +; GFX6-NEXT: s_sub_i32 s7, s8, s7 +; GFX6-NEXT: s_sub_i32 s8, s7, s6 +; GFX6-NEXT: s_cmp_ge_u32 s7, s6 +; GFX6-NEXT: s_cselect_b32 s7, s8, s7 +; GFX6-NEXT: s_sub_i32 s8, s7, s6 +; GFX6-NEXT: s_cmp_ge_u32 s7, s6 +; GFX6-NEXT: s_cselect_b32 s6, s8, s7 +; GFX6-NEXT: s_xor_b32 s6, s6, s5 +; GFX6-NEXT: s_sub_i32 s5, s6, s5 +; GFX6-NEXT: v_mov_b32_e32 v0, s4 +; GFX6-NEXT: v_mov_b32_e32 v1, s5 ; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; GFX6-NEXT: s_endpgm ; @@ -7068,16 +6992,13 @@ define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_lshl_b32 s2, 0x1000, s6 -; GFX9-NEXT: s_ashr_i32 s3, s2, 31 -; GFX9-NEXT: s_add_i32 s2, s2, s3 -; GFX9-NEXT: s_xor_b32 s2, s2, s3 +; GFX9-NEXT: s_abs_i32 s2, s2 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GFX9-NEXT: s_lshl_b32 s3, 0x1000, s7 ; GFX9-NEXT: s_sub_i32 s7, 0, s2 ; GFX9-NEXT: s_ashr_i32 s6, s4, 31 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_add_i32 s4, s4, s6 -; GFX9-NEXT: s_xor_b32 s4, s4, s6 +; GFX9-NEXT: s_abs_i32 s4, s4 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX9-NEXT: v_readfirstlane_b32 s8, v0 @@ -7093,24 +7014,21 @@ define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x ; GFX9-NEXT: s_sub_i32 s7, s4, s2 ; GFX9-NEXT: s_cmp_ge_u32 s4, s2 ; GFX9-NEXT: s_cselect_b32 s2, s7, s4 -; GFX9-NEXT: s_ashr_i32 s4, s3, 31 -; GFX9-NEXT: s_add_i32 s3, s3, s4 -; GFX9-NEXT: s_xor_b32 s3, s3, s4 +; GFX9-NEXT: s_abs_i32 s3, s3 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 ; GFX9-NEXT: s_xor_b32 s2, s2, s6 +; GFX9-NEXT: s_sub_i32 s7, 0, s3 ; GFX9-NEXT: s_sub_i32 s2, s2, s6 -; GFX9-NEXT: s_sub_i32 s6, 0, s3 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX9-NEXT: s_ashr_i32 s4, s5, 31 -; GFX9-NEXT: s_add_i32 s5, s5, s4 -; GFX9-NEXT: s_xor_b32 s5, s5, s4 +; GFX9-NEXT: s_abs_i32 s5, s5 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: v_readfirstlane_b32 s7, v0 -; GFX9-NEXT: s_mul_i32 s6, s6, s7 -; GFX9-NEXT: s_mul_hi_u32 s6, s7, s6 -; GFX9-NEXT: s_add_i32 s7, s7, s6 -; GFX9-NEXT: s_mul_hi_u32 s6, s5, s7 +; GFX9-NEXT: v_readfirstlane_b32 s6, v0 +; GFX9-NEXT: s_mul_i32 s7, s7, s6 +; GFX9-NEXT: s_mul_hi_u32 s7, s6, s7 +; GFX9-NEXT: s_add_i32 s6, s6, s7 +; GFX9-NEXT: s_mul_hi_u32 s6, s5, s6 ; GFX9-NEXT: s_mul_i32 s6, s6, s3 ; GFX9-NEXT: s_sub_i32 s5, s5, s6 ; GFX9-NEXT: s_sub_i32 s6, s5, s3 diff --git a/llvm/test/CodeGen/AMDGPU/bypass-div.ll b/llvm/test/CodeGen/AMDGPU/bypass-div.ll index 4d8687b141a7..5bbea7ecf3f2 100644 --- a/llvm/test/CodeGen/AMDGPU/bypass-div.ll +++ b/llvm/test/CodeGen/AMDGPU/bypass-div.ll @@ -575,34 +575,33 @@ define i32 @sdiv32(i32 %a, i32 %b) { ; GFX9-LABEL: sdiv32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v1 -; GFX9-NEXT: v_add_u32_e32 v1, v1, v2 -; GFX9-NEXT: v_xor_b32_e32 v1, v1, v2 -; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v1 -; GFX9-NEXT: v_sub_u32_e32 v4, 0, v1 -; GFX9-NEXT: v_ashrrev_i32_e32 v5, 31, v0 -; GFX9-NEXT: v_add_u32_e32 v0, v0, v5 +; GFX9-NEXT: v_sub_u32_e32 v2, 0, v1 +; GFX9-NEXT: v_max_i32_e32 v2, v1, v2 +; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v2 +; GFX9-NEXT: v_sub_u32_e32 v4, 0, v2 +; GFX9-NEXT: v_sub_u32_e32 v5, 0, v0 +; GFX9-NEXT: v_max_i32_e32 v5, v0, v5 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v3, v3 -; GFX9-NEXT: v_xor_b32_e32 v0, v0, v5 -; GFX9-NEXT: v_xor_b32_e32 v2, v5, v2 +; GFX9-NEXT: v_xor_b32_e32 v0, v0, v1 +; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v0 ; GFX9-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3 ; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3 ; GFX9-NEXT: v_mul_lo_u32 v4, v4, v3 ; GFX9-NEXT: v_mul_hi_u32 v4, v3, v4 ; GFX9-NEXT: v_add_u32_e32 v3, v3, v4 -; GFX9-NEXT: v_mul_hi_u32 v3, v0, v3 -; GFX9-NEXT: v_mul_lo_u32 v4, v3, v1 -; GFX9-NEXT: v_add_u32_e32 v5, 1, v3 -; GFX9-NEXT: v_sub_u32_e32 v0, v0, v4 -; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 -; GFX9-NEXT: v_sub_u32_e32 v4, v0, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX9-NEXT: v_add_u32_e32 v4, 1, v3 -; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc -; GFX9-NEXT: v_xor_b32_e32 v0, v0, v2 -; GFX9-NEXT: v_sub_u32_e32 v0, v0, v2 +; GFX9-NEXT: v_mul_hi_u32 v3, v5, v3 +; GFX9-NEXT: v_mul_lo_u32 v4, v3, v2 +; GFX9-NEXT: v_add_u32_e32 v1, 1, v3 +; GFX9-NEXT: v_sub_u32_e32 v4, v5, v4 +; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v4, v2 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX9-NEXT: v_sub_u32_e32 v3, v4, v2 +; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc +; GFX9-NEXT: v_add_u32_e32 v4, 1, v1 +; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v3, v2 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc +; GFX9-NEXT: v_xor_b32_e32 v1, v1, v0 +; GFX9-NEXT: v_sub_u32_e32 v0, v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] %d = sdiv i32 %a, %b ret i32 %d @@ -640,31 +639,30 @@ define i32 @srem32(i32 %a, i32 %b) { ; GFX9-LABEL: srem32: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v1 -; GFX9-NEXT: v_add_u32_e32 v1, v1, v2 -; GFX9-NEXT: v_xor_b32_e32 v1, v1, v2 +; GFX9-NEXT: v_sub_u32_e32 v2, 0, v1 +; GFX9-NEXT: v_max_i32_e32 v1, v1, v2 ; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v1 ; GFX9-NEXT: v_sub_u32_e32 v3, 0, v1 -; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v0 -; GFX9-NEXT: v_add_u32_e32 v0, v0, v4 +; GFX9-NEXT: v_sub_u32_e32 v4, 0, v0 +; GFX9-NEXT: v_max_i32_e32 v4, v0, v4 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v2, v2 -; GFX9-NEXT: v_xor_b32_e32 v0, v0, v4 +; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v0 ; GFX9-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2 ; GFX9-NEXT: v_mul_lo_u32 v3, v3, v2 ; GFX9-NEXT: v_mul_hi_u32 v3, v2, v3 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v3 -; GFX9-NEXT: v_mul_hi_u32 v2, v0, v2 +; GFX9-NEXT: v_mul_hi_u32 v2, v4, v2 ; GFX9-NEXT: v_mul_lo_u32 v2, v2, v1 -; GFX9-NEXT: v_sub_u32_e32 v0, v0, v2 -; GFX9-NEXT: v_sub_u32_e32 v2, v0, v1 -; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_sub_u32_e32 v2, v0, v1 -; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_xor_b32_e32 v0, v0, v4 -; GFX9-NEXT: v_sub_u32_e32 v0, v0, v4 +; GFX9-NEXT: v_sub_u32_e32 v2, v4, v2 +; GFX9-NEXT: v_sub_u32_e32 v3, v2, v1 +; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v2, v1 +; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; GFX9-NEXT: v_sub_u32_e32 v3, v2, v1 +; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v2, v1 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc +; GFX9-NEXT: v_xor_b32_e32 v1, v1, v0 +; GFX9-NEXT: v_sub_u32_e32 v0, v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] %d = srem i32 %a, %b ret i32 %d diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll index 7d8eba1e8708..f0ab3a5342e0 100644 --- a/llvm/test/CodeGen/AMDGPU/div_i128.ll +++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll @@ -9,26 +9,28 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-LABEL: v_sdiv_i128_vv: ; GFX9: ; %bb.0: ; %_udiv-special-cases ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, 0, v0 +; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, 0, v1, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, 0, v2, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, 0, v3, vcc +; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[2:3] ; GFX9-NEXT: v_ashrrev_i32_e32 v16, 31, v3 -; GFX9-NEXT: v_xor_b32_e32 v0, v16, v0 -; GFX9-NEXT: v_xor_b32_e32 v1, v16, v1 -; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v0, v16 -; GFX9-NEXT: v_xor_b32_e32 v2, v16, v2 -; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v1, v16, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v9, v1, v9, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v8, v0, v8, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v11, v3, v11, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v10, v2, v10, vcc +; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, 0, v4 +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, 0, v5, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, 0, v6, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, 0, v7, vcc +; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[6:7] ; GFX9-NEXT: v_ashrrev_i32_e32 v17, 31, v7 -; GFX9-NEXT: v_xor_b32_e32 v3, v16, v3 -; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, v2, v16, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, v3, v16, vcc -; GFX9-NEXT: v_xor_b32_e32 v3, v17, v4 -; GFX9-NEXT: v_xor_b32_e32 v2, v17, v5 -; GFX9-NEXT: v_sub_co_u32_e32 v20, vcc, v3, v17 -; GFX9-NEXT: v_xor_b32_e32 v0, v17, v6 -; GFX9-NEXT: v_subb_co_u32_e32 v21, vcc, v2, v17, vcc -; GFX9-NEXT: v_xor_b32_e32 v1, v17, v7 -; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v17, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v17, vcc -; GFX9-NEXT: v_or_b32_e32 v3, v21, v1 -; GFX9-NEXT: v_or_b32_e32 v2, v20, v0 +; GFX9-NEXT: v_cndmask_b32_e32 v20, v5, v1, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v21, v4, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc +; GFX9-NEXT: v_or_b32_e32 v3, v20, v1 +; GFX9-NEXT: v_or_b32_e32 v2, v21, v0 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3] ; GFX9-NEXT: v_or_b32_e32 v3, v9, v11 ; GFX9-NEXT: v_or_b32_e32 v2, v8, v10 @@ -37,35 +39,35 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-NEXT: v_add_u32_e32 v2, 32, v2 ; GFX9-NEXT: v_ffbh_u32_e32 v3, v1 ; GFX9-NEXT: v_min_u32_e32 v2, v2, v3 -; GFX9-NEXT: v_ffbh_u32_e32 v3, v20 +; GFX9-NEXT: v_ffbh_u32_e32 v3, v21 ; GFX9-NEXT: v_add_u32_e32 v3, 32, v3 -; GFX9-NEXT: v_ffbh_u32_e32 v4, v21 +; GFX9-NEXT: v_ffbh_u32_e32 v4, v20 ; GFX9-NEXT: v_min_u32_e32 v3, v3, v4 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, 64, v3 ; GFX9-NEXT: v_addc_co_u32_e64 v4, s[6:7], 0, 0, vcc ; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] -; GFX9-NEXT: v_ffbh_u32_e32 v5, v11 +; GFX9-NEXT: v_ffbh_u32_e32 v6, v11 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc ; GFX9-NEXT: v_ffbh_u32_e32 v3, v10 ; GFX9-NEXT: v_add_u32_e32 v3, 32, v3 -; GFX9-NEXT: v_min_u32_e32 v3, v3, v5 -; GFX9-NEXT: v_ffbh_u32_e32 v5, v8 -; GFX9-NEXT: v_add_u32_e32 v5, 32, v5 -; GFX9-NEXT: v_ffbh_u32_e32 v6, v9 -; GFX9-NEXT: v_min_u32_e32 v5, v5, v6 +; GFX9-NEXT: v_min_u32_e32 v3, v3, v6 +; GFX9-NEXT: v_ffbh_u32_e32 v6, v8 +; GFX9-NEXT: v_add_u32_e32 v6, 32, v6 +; GFX9-NEXT: v_ffbh_u32_e32 v7, v9 +; GFX9-NEXT: v_min_u32_e32 v6, v6, v7 ; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, 0, vcc -; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, 64, v5 -; GFX9-NEXT: v_addc_co_u32_e64 v6, s[6:7], 0, 0, vcc +; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, 64, v6 +; GFX9-NEXT: v_addc_co_u32_e64 v7, s[6:7], 0, 0, vcc ; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11] -; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f -; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, 0, vcc -; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v3 -; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v6, vcc ; GFX9-NEXT: v_mov_b32_e32 v5, 0 +; GFX9-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, 0, vcc +; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v3 +; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v7, vcc ; GFX9-NEXT: v_subbrev_co_u32_e32 v4, vcc, 0, v5, vcc ; GFX9-NEXT: v_subbrev_co_u32_e32 v5, vcc, 0, v5, vcc +; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f ; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3] ; GFX9-NEXT: v_mov_b32_e32 v18, v16 ; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc @@ -138,8 +140,8 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-NEXT: v_cndmask_b32_e64 v8, v12, v8, s[4:5] ; GFX9-NEXT: v_cndmask_b32_e32 v11, 0, v7, vcc ; GFX9-NEXT: v_cndmask_b32_e32 v10, 0, v6, vcc -; GFX9-NEXT: v_add_co_u32_e32 v26, vcc, -1, v20 -; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, -1, v21, vcc +; GFX9-NEXT: v_add_co_u32_e32 v26, vcc, -1, v21 +; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, -1, v20, vcc ; GFX9-NEXT: v_addc_co_u32_e32 v28, vcc, -1, v0, vcc ; GFX9-NEXT: v_mov_b32_e32 v14, 0 ; GFX9-NEXT: v_mov_b32_e32 v12, 0 @@ -164,9 +166,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-NEXT: v_subb_co_u32_e32 v14, vcc, v28, v10, vcc ; GFX9-NEXT: v_subb_co_u32_e32 v14, vcc, v29, v11, vcc ; GFX9-NEXT: v_ashrrev_i32_e32 v30, 31, v14 -; GFX9-NEXT: v_and_b32_e32 v14, v30, v20 -; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v8, v14 ; GFX9-NEXT: v_and_b32_e32 v14, v30, v21 +; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v8, v14 +; GFX9-NEXT: v_and_b32_e32 v14, v30, v20 ; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v9, v14, vcc ; GFX9-NEXT: v_and_b32_e32 v14, v30, v0 ; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, v10, v14, vcc @@ -218,238 +220,244 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0: ; %bb.0: ; %_udiv-special-cases ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-O0-NEXT: ; implicit-def: $vgpr8 : SGPR spill to VGPR lane -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5 -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v4 +; GFX9-O0-NEXT: v_mov_b32_e32 v21, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v4 ; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v0 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v2 +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v0 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v22, v7 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v5 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: s_waitcnt vmcnt(4) -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 +; GFX9-O0-NEXT: ; kill: def $vgpr10 killed $vgpr10 def $vgpr10_vgpr11 killed $exec +; GFX9-O0-NEXT: s_waitcnt vmcnt(1) +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v2 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: s_waitcnt vmcnt(1) -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 +; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v1 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 -; GFX9-O0-NEXT: s_mov_b32 s4, 63 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v4 -; GFX9-O0-NEXT: v_ashrrev_i64 v[13:14], s4, v[11:12] -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v2 -; GFX9-O0-NEXT: v_ashrrev_i64 v[11:12], s4, v[11:12] -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v14 -; GFX9-O0-NEXT: v_xor_b32_e64 v1, v6, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v13 -; GFX9-O0-NEXT: v_xor_b32_e64 v13, v4, v5 -; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v10 -; GFX9-O0-NEXT: v_xor_b32_e64 v1, v6, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 -; GFX9-O0-NEXT: v_xor_b32_e64 v15, v4, v5 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v15 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v14 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v9, vcc, v9, v4 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v5, vcc, v5, v6, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v13, vcc, v10, v4, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v6, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v4 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v10 +; GFX9-O0-NEXT: v_mov_b32_e32 v20, v11 +; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 0 +; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 1 +; GFX9-O0-NEXT: s_mov_b32 s10, s6 +; GFX9-O0-NEXT: v_writelane_b32 v0, s10, 2 +; GFX9-O0-NEXT: s_mov_b32 s11, s7 +; GFX9-O0-NEXT: v_writelane_b32 v0, s11, 3 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v6, vcc, s10, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, s11 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v4, vcc, v1, v3, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v1, s10 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v5, vcc, v1, v14, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v1, s11 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v20, vcc ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v4 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v7 +; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7] +; GFX9-O0-NEXT: v_cmp_lt_i64_e64 s[4:5], v[10:11], s[4:5] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v3, v4, s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v2, v3, s[4:5] +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v6 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v20, v1, s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v1, v14, v1, s[4:5] ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 -; GFX9-O0-NEXT: v_xor_b32_e64 v1, v5, v1 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_xor_b32_e64 v11, v3, v2 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v8 -; GFX9-O0-NEXT: v_xor_b32_e64 v1, v5, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v7 -; GFX9-O0-NEXT: v_xor_b32_e64 v7, v3, v2 -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 -; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 killed $vgpr7_vgpr8 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v12 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v3 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v8, vcc, v8, v5, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v7, v3, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v2, v5, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v8 +; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 killed $vgpr8_vgpr9 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v21 +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v22 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v18, vcc, s10, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, s11 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v10, vcc, v8, v9, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v8, s10 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v8, v13, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v8, s11 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v8, vcc, v8, v15, vcc ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v8 +; GFX9-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v10 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v19 +; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7] +; GFX9-O0-NEXT: v_cmp_lt_i64_e64 s[4:5], v[21:22], s[4:5] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v10, v9, v10, s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v18 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, v7, v9, s[4:5] +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v10 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v12 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, v15, v8, s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v11 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v8, v13, v8, s[4:5] ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v7 -; GFX9-O0-NEXT: v_xor_b32_e64 v5, v5, v6 -; GFX9-O0-NEXT: v_xor_b32_e64 v3, v3, v4 -; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v4 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v3 -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v9 +; GFX9-O0-NEXT: v_xor_b32_e64 v15, v15, v20 +; GFX9-O0-NEXT: v_xor_b32_e64 v13, v13, v14 +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v15 +; GFX9-O0-NEXT: s_mov_b32 s4, 63 +; GFX9-O0-NEXT: v_ashrrev_i64 v[13:14], s4, v[13:14] +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v12 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v18 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v19 +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v5 +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v16 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v17 +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v2 -; GFX9-O0-NEXT: v_or_b32_e64 v3, v8, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 -; GFX9-O0-NEXT: v_or_b32_e64 v1, v5, v6 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 -; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 0 -; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 1 -; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[1:2], s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v14 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: v_or_b32_e64 v15, v4, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 -; GFX9-O0-NEXT: v_or_b32_e64 v9, v3, v1 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v15 -; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[9:10], s[6:7] +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v19 +; GFX9-O0-NEXT: v_or_b32_e64 v15, v13, v14 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v18 +; GFX9-O0-NEXT: v_or_b32_e64 v13, v13, v14 +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v15 +; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[13:14], s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v17 +; GFX9-O0-NEXT: v_or_b32_e64 v15, v13, v14 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v16 +; GFX9-O0-NEXT: v_or_b32_e64 v13, v13, v14 +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v15 +; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[13:14], s[6:7] ; GFX9-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9] -; GFX9-O0-NEXT: v_ffbh_u32_e64 v6, v6 -; GFX9-O0-NEXT: s_mov_b32 s9, 32 -; GFX9-O0-NEXT: v_add_u32_e64 v6, v6, s9 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v7, v7 -; GFX9-O0-NEXT: v_min_u32_e64 v6, v6, v7 -; GFX9-O0-NEXT: s_mov_b32 s8, 0 -; GFX9-O0-NEXT: ; implicit-def: $sgpr10 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v7 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v5 -; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 +; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[8:9], v[11:12], s[8:9] ; GFX9-O0-NEXT: v_ffbh_u32_e64 v8, v8 -; GFX9-O0-NEXT: v_min_u32_e64 v15, v5, v8 -; GFX9-O0-NEXT: ; implicit-def: $sgpr10 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v5 -; GFX9-O0-NEXT: s_mov_b64 s[10:11], 64 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v15 -; GFX9-O0-NEXT: s_mov_b32 s12, s10 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16 -; GFX9-O0-NEXT: s_mov_b32 s14, s11 -; GFX9-O0-NEXT: v_add_co_u32_e64 v8, s[12:13], v8, s12 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, s14 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[12:13], v5, v9, s[12:13] +; GFX9-O0-NEXT: s_mov_b32 s13, 32 +; GFX9-O0-NEXT: v_add_u32_e64 v8, v8, s13 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v9, v9 +; GFX9-O0-NEXT: v_min_u32_e64 v8, v8, v9 +; GFX9-O0-NEXT: s_mov_b32 s12, 0 +; GFX9-O0-NEXT: ; implicit-def: $sgpr14 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s12 ; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 -; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[6:7] -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[12:13], v[11:12], s[12:13] -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v10, s[12:13] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, v6, v7, s[12:13] -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v9 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v7, v7 +; GFX9-O0-NEXT: v_add_u32_e64 v7, v7, s13 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v10, v10 +; GFX9-O0-NEXT: v_min_u32_e64 v13, v7, v10 +; GFX9-O0-NEXT: ; implicit-def: $sgpr14 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, s12 +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v7 +; GFX9-O0-NEXT: s_mov_b64 s[14:15], 64 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v13 +; GFX9-O0-NEXT: s_mov_b32 s16, s14 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v14 +; GFX9-O0-NEXT: s_mov_b32 s18, s15 +; GFX9-O0-NEXT: v_add_co_u32_e64 v10, s[16:17], v10, s16 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s18 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v7, s[16:17], v7, v11, s[16:17] +; GFX9-O0-NEXT: ; kill: def $vgpr10 killed $vgpr10 def $vgpr10_vgpr11 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v11 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, v7, v12, s[8:9] +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v10 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, v8, v9, s[8:9] +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 ; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v7 +; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[8:9], v[5:6], s[8:9] ; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v1 -; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 +; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s13 ; GFX9-O0-NEXT: v_ffbh_u32_e64 v6, v2 ; GFX9-O0-NEXT: v_min_u32_e64 v6, v5, v6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr16 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s12 ; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 ; GFX9-O0-NEXT: v_mov_b32_e32 v8, v7 ; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v3 -; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 +; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s13 ; GFX9-O0-NEXT: v_ffbh_u32_e64 v11, v4 -; GFX9-O0-NEXT: v_min_u32_e64 v15, v5, v11 -; GFX9-O0-NEXT: ; implicit-def: $sgpr9 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v15 -; GFX9-O0-NEXT: s_mov_b32 s8, s10 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16 -; GFX9-O0-NEXT: s_mov_b32 s10, s11 -; GFX9-O0-NEXT: v_add_co_u32_e64 v11, s[8:9], v11, s8 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, s10 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[8:9], v5, v12, s[8:9] +; GFX9-O0-NEXT: v_min_u32_e64 v12, v5, v11 +; GFX9-O0-NEXT: ; implicit-def: $sgpr13 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s12 +; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v12 +; GFX9-O0-NEXT: s_mov_b32 s12, s14 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v13 +; GFX9-O0-NEXT: s_mov_b32 s14, s15 +; GFX9-O0-NEXT: v_add_co_u32_e64 v11, s[12:13], v11, s12 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, s14 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[12:13], v5, v12, s[12:13] ; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v12, v5 ; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 -; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[8:9], v[13:14], s[8:9] ; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 ; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 @@ -462,8 +470,6 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 ; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr6_vgpr7 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 -; GFX9-O0-NEXT: s_mov_b32 s10, s6 -; GFX9-O0-NEXT: s_mov_b32 s11, s7 ; GFX9-O0-NEXT: v_sub_co_u32_e32 v5, vcc, v5, v8 ; GFX9-O0-NEXT: v_subb_co_u32_e32 v9, vcc, v6, v7, vcc ; GFX9-O0-NEXT: v_mov_b32_e32 v7, s10 @@ -546,75 +552,75 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) ; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_mov_b64 s[4:5], exec -; GFX9-O0-NEXT: v_writelane_b32 v0, s4, 2 -; GFX9-O0-NEXT: v_writelane_b32 v0, s5, 3 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: v_writelane_b32 v0, s4, 4 +; GFX9-O0-NEXT: v_writelane_b32 v0, s5, 5 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7] ; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-O0-NEXT: s_cbranch_execz .LBB0_3 ; GFX9-O0-NEXT: s_branch .LBB0_8 ; GFX9-O0-NEXT: .LBB0_1: ; %Flow -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s4, v0, 4 -; GFX9-O0-NEXT: v_readlane_b32 s5, v0, 5 +; GFX9-O0-NEXT: v_readlane_b32 s4, v0, 6 +; GFX9-O0-NEXT: v_readlane_b32 s5, v0, 7 ; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX9-O0-NEXT: ; %bb.2: ; %Flow -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_waitcnt vmcnt(6) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_branch .LBB0_5 ; GFX9-O0-NEXT: .LBB0_3: ; %Flow2 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s4, v4, 2 -; GFX9-O0-NEXT: v_readlane_b32 s5, v4, 3 +; GFX9-O0-NEXT: v_readlane_b32 s4, v4, 4 +; GFX9-O0-NEXT: v_readlane_b32 s5, v4, 5 ; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_branch .LBB0_9 ; GFX9-O0-NEXT: .LBB0_4: ; %udiv-loop-exit -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_mov_b32 s4, 1 ; GFX9-O0-NEXT: s_waitcnt vmcnt(2) ; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[0:1] @@ -647,67 +653,67 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_branch .LBB0_3 ; GFX9-O0-NEXT: .LBB0_5: ; %Flow1 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s4, v8, 6 -; GFX9-O0-NEXT: v_readlane_b32 s5, v8, 7 +; GFX9-O0-NEXT: v_readlane_b32 s4, v8, 8 +; GFX9-O0-NEXT: v_readlane_b32 s5, v8, 9 ; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_branch .LBB0_4 ; GFX9-O0-NEXT: .LBB0_6: ; %udiv-do-while ; GFX9-O0-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s6, v16, 8 -; GFX9-O0-NEXT: v_readlane_b32 s7, v16, 9 -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload +; GFX9-O0-NEXT: v_readlane_b32 s6, v16, 10 +; GFX9-O0-NEXT: v_readlane_b32 s7, v16, 11 +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_mov_b32 s4, 63 ; GFX9-O0-NEXT: s_waitcnt vmcnt(16) ; GFX9-O0-NEXT: v_lshrrev_b64 v[29:30], s4, v[2:3] @@ -847,72 +853,72 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7] ; GFX9-O0-NEXT: v_mov_b32_e32 v18, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v17, v2 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill ; GFX9-O0-NEXT: v_mov_b32_e32 v18, v1 ; GFX9-O0-NEXT: v_mov_b32_e32 v17, v0 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill ; GFX9-O0-NEXT: v_mov_b32_e32 v18, v15 ; GFX9-O0-NEXT: v_mov_b32_e32 v17, v14 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill ; GFX9-O0-NEXT: v_mov_b32_e32 v18, v13 ; GFX9-O0-NEXT: v_mov_b32_e32 v17, v12 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5] -; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 4 -; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 5 +; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 6 +; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 7 ; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5] -; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 8 -; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 9 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 10 +; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 11 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GFX9-O0-NEXT: s_cbranch_execnz .LBB0_6 ; GFX9-O0-NEXT: s_branch .LBB0_1 ; GFX9-O0-NEXT: .LBB0_7: ; %udiv-preheader -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload @@ -993,51 +999,51 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v13, v17 ; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7] ; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9 ; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8 ; GFX9-O0-NEXT: v_mov_b32_e32 v13, s7 ; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6 -; GFX9-O0-NEXT: v_writelane_b32 v16, s4, 8 -; GFX9-O0-NEXT: v_writelane_b32 v16, s5, 9 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: v_writelane_b32 v16, s4, 10 +; GFX9-O0-NEXT: v_writelane_b32 v16, s5, 11 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_branch .LBB0_6 ; GFX9-O0-NEXT: .LBB0_8: ; %udiv-bb1 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload @@ -1074,14 +1080,14 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 ; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill ; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 ; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_mov_b32 s4, 0x7f ; GFX9-O0-NEXT: v_sub_u32_e64 v3, s4, v4 ; GFX9-O0-NEXT: v_lshlrev_b64 v[5:6], v3, v[11:12] @@ -1127,12 +1133,12 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v8, v3 -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill ; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 ; GFX9-O0-NEXT: v_mov_b32_e32 v3, v10 ; GFX9-O0-NEXT: v_or_b32_e64 v3, v3, v4 @@ -1147,39 +1153,39 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: v_mov_b32_e32 v4, s9 ; GFX9-O0-NEXT: v_mov_b32_e32 v1, s6 ; GFX9-O0-NEXT: v_mov_b32_e32 v2, s7 -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_mov_b64 s[6:7], exec ; GFX9-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5] ; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7] -; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 6 -; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 7 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 8 +; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 9 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-O0-NEXT: s_cbranch_execz .LBB0_5 ; GFX9-O0-NEXT: s_branch .LBB0_7 ; GFX9-O0-NEXT: .LBB0_9: ; %udiv-end -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) @@ -1224,11 +1230,11 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 ; GFX9-O0-NEXT: ; kill: killed $vgpr4 ; GFX9-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_nop 0 -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) ; GFX9-O0-NEXT: s_setpc_b64 s[30:31] diff --git a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll index 16a03badcb13..e04cd7112560 100644 --- a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll +++ b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll @@ -6,185 +6,187 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) { ; SDAG-LABEL: v_sdiv_v2i128_vv: ; SDAG: ; %bb.0: ; %_udiv-special-cases_udiv-special-cases ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SDAG-NEXT: v_ashrrev_i32_e32 v24, 31, v3 -; SDAG-NEXT: v_ashrrev_i32_e32 v25, 31, v11 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 +; SDAG-NEXT: v_ashrrev_i32_e32 v26, 31, v3 +; SDAG-NEXT: v_ashrrev_i32_e32 v27, 31, v11 +; SDAG-NEXT: v_sub_i32_e32 v16, vcc, 0, v0 +; SDAG-NEXT: v_mov_b32_e32 v19, 0 ; SDAG-NEXT: s_mov_b64 s[10:11], 0x7f -; SDAG-NEXT: v_mov_b32_e32 v26, v24 -; SDAG-NEXT: v_mov_b32_e32 v27, v25 -; SDAG-NEXT: v_xor_b32_e32 v17, v24, v3 -; SDAG-NEXT: v_xor_b32_e32 v18, v24, v2 -; SDAG-NEXT: v_xor_b32_e32 v1, v24, v1 -; SDAG-NEXT: v_xor_b32_e32 v0, v24, v0 -; SDAG-NEXT: v_xor_b32_e32 v19, v25, v11 -; SDAG-NEXT: v_xor_b32_e32 v20, v25, v10 -; SDAG-NEXT: v_xor_b32_e32 v9, v25, v9 -; SDAG-NEXT: v_xor_b32_e32 v8, v25, v8 -; SDAG-NEXT: v_sub_i32_e32 v2, vcc, v0, v24 -; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v1, v24, vcc -; SDAG-NEXT: v_ffbh_u32_e32 v0, v2 -; SDAG-NEXT: v_subb_u32_e32 v10, vcc, v18, v24, vcc -; SDAG-NEXT: v_add_i32_e64 v1, s[4:5], 32, v0 -; SDAG-NEXT: v_ffbh_u32_e32 v18, v3 -; SDAG-NEXT: v_subb_u32_e32 v11, vcc, v17, v24, vcc -; SDAG-NEXT: v_or_b32_e32 v0, v2, v10 -; SDAG-NEXT: v_ffbh_u32_e32 v17, v10 -; SDAG-NEXT: v_min_u32_e32 v18, v1, v18 -; SDAG-NEXT: v_sub_i32_e32 v28, vcc, v8, v25 -; SDAG-NEXT: v_or_b32_e32 v1, v3, v11 -; SDAG-NEXT: v_add_i32_e64 v8, s[4:5], 32, v17 -; SDAG-NEXT: v_ffbh_u32_e32 v17, v11 -; SDAG-NEXT: v_add_i32_e64 v18, s[4:5], 64, v18 -; SDAG-NEXT: v_addc_u32_e64 v21, s[4:5], 0, 0, s[4:5] -; SDAG-NEXT: v_subb_u32_e32 v29, vcc, v9, v25, vcc -; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[0:1] -; SDAG-NEXT: v_ffbh_u32_e32 v1, v28 -; SDAG-NEXT: v_min_u32_e32 v8, v8, v17 -; SDAG-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[10:11] -; SDAG-NEXT: v_cndmask_b32_e64 v17, v21, 0, s[6:7] -; SDAG-NEXT: v_subb_u32_e32 v0, vcc, v20, v25, vcc -; SDAG-NEXT: v_add_i32_e64 v9, s[8:9], 32, v1 -; SDAG-NEXT: v_ffbh_u32_e32 v20, v29 -; SDAG-NEXT: v_cndmask_b32_e64 v18, v18, v8, s[6:7] -; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v19, v25, vcc -; SDAG-NEXT: v_or_b32_e32 v8, v28, v0 -; SDAG-NEXT: v_ffbh_u32_e32 v19, v0 -; SDAG-NEXT: v_min_u32_e32 v20, v9, v20 -; SDAG-NEXT: v_or_b32_e32 v9, v29, v1 -; SDAG-NEXT: v_add_i32_e32 v19, vcc, 32, v19 +; SDAG-NEXT: v_mov_b32_e32 v28, v26 +; SDAG-NEXT: v_mov_b32_e32 v29, v27 +; SDAG-NEXT: v_subb_u32_e32 v17, vcc, 0, v1, vcc +; SDAG-NEXT: v_subb_u32_e32 v18, vcc, 0, v2, vcc +; SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[2:3] +; SDAG-NEXT: v_cndmask_b32_e64 v17, v1, v17, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v16, v0, v16, s[4:5] +; SDAG-NEXT: v_subb_u32_e32 v0, vcc, 0, v3, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v2, v2, v18, s[4:5] +; SDAG-NEXT: v_ffbh_u32_e32 v1, v16 +; SDAG-NEXT: v_ffbh_u32_e32 v18, v17 +; SDAG-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[4:5] +; SDAG-NEXT: v_sub_i32_e32 v20, vcc, 0, v8 +; SDAG-NEXT: v_or_b32_e32 v0, v16, v2 +; SDAG-NEXT: v_ffbh_u32_e32 v21, v2 +; SDAG-NEXT: v_add_i32_e64 v22, s[4:5], 32, v1 +; SDAG-NEXT: v_subb_u32_e32 v23, vcc, 0, v9, vcc +; SDAG-NEXT: v_or_b32_e32 v1, v17, v3 +; SDAG-NEXT: v_add_i32_e64 v21, s[4:5], 32, v21 +; SDAG-NEXT: v_min_u32_e32 v18, v22, v18 +; SDAG-NEXT: v_ffbh_u32_e32 v22, v3 +; SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[10:11] +; SDAG-NEXT: v_cndmask_b32_e64 v30, v9, v23, s[4:5] +; SDAG-NEXT: v_subb_u32_e32 v9, vcc, 0, v10, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v31, v8, v20, s[4:5] +; SDAG-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[0:1] +; SDAG-NEXT: v_min_u32_e32 v1, v21, v22 +; SDAG-NEXT: v_add_i32_e64 v8, s[8:9], 64, v18 +; SDAG-NEXT: v_addc_u32_e64 v18, s[8:9], 0, 0, s[8:9] +; SDAG-NEXT: v_subb_u32_e32 v20, vcc, 0, v11, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v0, v10, v9, s[4:5] +; SDAG-NEXT: v_ffbh_u32_e32 v9, v31 +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] +; SDAG-NEXT: v_cndmask_b32_e64 v10, v18, 0, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v18, v8, v1, vcc +; SDAG-NEXT: v_ffbh_u32_e32 v21, v30 +; SDAG-NEXT: v_cndmask_b32_e64 v1, v11, v20, s[4:5] +; SDAG-NEXT: v_or_b32_e32 v8, v31, v0 +; SDAG-NEXT: v_ffbh_u32_e32 v11, v0 +; SDAG-NEXT: v_add_i32_e32 v20, vcc, 32, v9 +; SDAG-NEXT: v_or_b32_e32 v9, v30, v1 +; SDAG-NEXT: v_add_i32_e32 v11, vcc, 32, v11 +; SDAG-NEXT: v_min_u32_e32 v20, v20, v21 ; SDAG-NEXT: v_ffbh_u32_e32 v21, v1 -; SDAG-NEXT: v_add_i32_e32 v20, vcc, 64, v20 -; SDAG-NEXT: v_addc_u32_e64 v22, s[6:7], 0, 0, vcc ; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] -; SDAG-NEXT: v_min_u32_e32 v8, v19, v21 -; SDAG-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[0:1] -; SDAG-NEXT: v_cndmask_b32_e64 v9, v22, 0, s[6:7] -; SDAG-NEXT: s_or_b64 s[8:9], vcc, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v8, v20, v8, s[6:7] +; SDAG-NEXT: v_min_u32_e32 v8, v11, v21 +; SDAG-NEXT: v_add_i32_e64 v9, s[4:5], 64, v20 +; SDAG-NEXT: v_addc_u32_e64 v11, s[4:5], 0, 0, s[4:5] +; SDAG-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e64 v11, v11, 0, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc ; SDAG-NEXT: v_sub_i32_e32 v8, vcc, v8, v18 -; SDAG-NEXT: v_subb_u32_e32 v9, vcc, v9, v17, vcc -; SDAG-NEXT: v_xor_b32_e32 v17, 0x7f, v8 -; SDAG-NEXT: v_subbrev_u32_e32 v18, vcc, 0, v16, vcc +; SDAG-NEXT: v_subb_u32_e32 v9, vcc, v11, v10, vcc +; SDAG-NEXT: v_xor_b32_e32 v10, 0x7f, v8 +; SDAG-NEXT: v_subbrev_u32_e32 v18, vcc, 0, v19, vcc ; SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], s[10:11], v[8:9] ; SDAG-NEXT: v_cndmask_b32_e64 v20, 0, 1, s[4:5] -; SDAG-NEXT: v_subbrev_u32_e32 v19, vcc, 0, v16, vcc -; SDAG-NEXT: v_or_b32_e32 v16, v17, v18 +; SDAG-NEXT: v_subbrev_u32_e32 v19, vcc, 0, v19, vcc +; SDAG-NEXT: v_or_b32_e32 v10, v10, v18 ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[18:19] ; SDAG-NEXT: v_cndmask_b32_e64 v21, 0, 1, vcc -; SDAG-NEXT: v_or_b32_e32 v17, v9, v19 +; SDAG-NEXT: v_or_b32_e32 v11, v9, v19 ; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[18:19] ; SDAG-NEXT: v_cndmask_b32_e32 v20, v21, v20, vcc -; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[16:17] -; SDAG-NEXT: v_and_b32_e32 v16, 1, v20 -; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v16 -; SDAG-NEXT: s_or_b64 s[4:5], s[8:9], s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v20, v11, 0, s[4:5] +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11] +; SDAG-NEXT: v_and_b32_e32 v10, 1, v20 +; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v10 +; SDAG-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v20, v3, 0, s[4:5] ; SDAG-NEXT: s_xor_b64 s[6:7], s[4:5], -1 -; SDAG-NEXT: v_cndmask_b32_e64 v17, v10, 0, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v21, v3, 0, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v16, v2, 0, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v21, v2, 0, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v22, v17, 0, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v23, v16, 0, s[4:5] ; SDAG-NEXT: s_and_b64 s[4:5], s[6:7], vcc ; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] ; SDAG-NEXT: s_cbranch_execz .LBB0_6 ; SDAG-NEXT: ; %bb.1: ; %udiv-bb15 -; SDAG-NEXT: v_add_i32_e32 v30, vcc, 1, v8 +; SDAG-NEXT: v_add_i32_e32 v32, vcc, 1, v8 ; SDAG-NEXT: v_sub_i32_e64 v20, s[4:5], 63, v8 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 -; SDAG-NEXT: v_mov_b32_e32 v17, 0 -; SDAG-NEXT: v_addc_u32_e32 v31, vcc, 0, v9, vcc -; SDAG-NEXT: v_lshl_b64 v[20:21], v[2:3], v20 -; SDAG-NEXT: v_addc_u32_e32 v32, vcc, 0, v18, vcc -; SDAG-NEXT: v_addc_u32_e32 v33, vcc, 0, v19, vcc -; SDAG-NEXT: v_or_b32_e32 v18, v30, v32 -; SDAG-NEXT: v_sub_i32_e32 v34, vcc, 0x7f, v8 -; SDAG-NEXT: v_or_b32_e32 v19, v31, v33 -; SDAG-NEXT: v_lshl_b64 v[8:9], v[10:11], v34 -; SDAG-NEXT: v_sub_i32_e32 v35, vcc, 64, v34 -; SDAG-NEXT: v_lshl_b64 v[22:23], v[2:3], v34 +; SDAG-NEXT: v_mov_b32_e32 v10, 0 +; SDAG-NEXT: v_mov_b32_e32 v11, 0 +; SDAG-NEXT: v_addc_u32_e32 v33, vcc, 0, v9, vcc +; SDAG-NEXT: v_lshl_b64 v[20:21], v[16:17], v20 +; SDAG-NEXT: v_addc_u32_e32 v34, vcc, 0, v18, vcc +; SDAG-NEXT: v_addc_u32_e32 v35, vcc, 0, v19, vcc +; SDAG-NEXT: v_or_b32_e32 v18, v32, v34 +; SDAG-NEXT: v_sub_i32_e32 v24, vcc, 0x7f, v8 +; SDAG-NEXT: v_or_b32_e32 v19, v33, v35 +; SDAG-NEXT: v_lshl_b64 v[8:9], v[2:3], v24 +; SDAG-NEXT: v_sub_i32_e32 v25, vcc, 64, v24 +; SDAG-NEXT: v_lshl_b64 v[22:23], v[16:17], v24 ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[18:19] -; SDAG-NEXT: v_lshr_b64 v[18:19], v[2:3], v35 +; SDAG-NEXT: v_lshr_b64 v[18:19], v[16:17], v25 ; SDAG-NEXT: v_or_b32_e32 v9, v9, v19 ; SDAG-NEXT: v_or_b32_e32 v8, v8, v18 -; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v34 +; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v24 ; SDAG-NEXT: v_cndmask_b32_e64 v9, v21, v9, s[4:5] ; SDAG-NEXT: v_cndmask_b32_e64 v8, v20, v8, s[4:5] ; SDAG-NEXT: v_cndmask_b32_e64 v21, 0, v23, s[4:5] ; SDAG-NEXT: v_cndmask_b32_e64 v20, 0, v22, s[4:5] -; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v34 -; SDAG-NEXT: v_cndmask_b32_e64 v9, v9, v11, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v8, v8, v10, s[4:5] +; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v24 +; SDAG-NEXT: v_cndmask_b32_e64 v9, v9, v3, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v8, v8, v2, s[4:5] ; SDAG-NEXT: v_mov_b32_e32 v18, 0 ; SDAG-NEXT: v_mov_b32_e32 v19, 0 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc ; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; SDAG-NEXT: s_cbranch_execz .LBB0_5 ; SDAG-NEXT: ; %bb.2: ; %udiv-preheader4 -; SDAG-NEXT: v_lshr_b64 v[16:17], v[2:3], v30 -; SDAG-NEXT: v_sub_i32_e32 v35, vcc, 64, v30 -; SDAG-NEXT: v_subrev_i32_e32 v36, vcc, 64, v30 -; SDAG-NEXT: v_lshr_b64 v[37:38], v[10:11], v30 -; SDAG-NEXT: v_add_i32_e32 v34, vcc, -1, v28 +; SDAG-NEXT: v_lshr_b64 v[10:11], v[16:17], v32 +; SDAG-NEXT: v_sub_i32_e32 v37, vcc, 64, v32 +; SDAG-NEXT: v_subrev_i32_e32 v48, vcc, 64, v32 +; SDAG-NEXT: v_lshr_b64 v[24:25], v[2:3], v32 +; SDAG-NEXT: v_add_i32_e32 v36, vcc, -1, v31 ; SDAG-NEXT: s_mov_b64 s[10:11], 0 ; SDAG-NEXT: v_mov_b32_e32 v22, 0 ; SDAG-NEXT: v_mov_b32_e32 v23, 0 ; SDAG-NEXT: v_mov_b32_e32 v18, 0 ; SDAG-NEXT: v_mov_b32_e32 v19, 0 -; SDAG-NEXT: v_lshl_b64 v[48:49], v[10:11], v35 -; SDAG-NEXT: v_lshr_b64 v[10:11], v[10:11], v36 -; SDAG-NEXT: v_addc_u32_e32 v35, vcc, -1, v29, vcc -; SDAG-NEXT: v_or_b32_e32 v17, v17, v49 -; SDAG-NEXT: v_or_b32_e32 v16, v16, v48 -; SDAG-NEXT: v_addc_u32_e32 v36, vcc, -1, v0, vcc -; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v30 -; SDAG-NEXT: v_cndmask_b32_e64 v17, v11, v17, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v16, v10, v16, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v11, 0, v38, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v10, 0, v37, s[4:5] -; SDAG-NEXT: v_addc_u32_e32 v37, vcc, -1, v1, vcc -; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v30 -; SDAG-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; SDAG-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc -; SDAG-NEXT: v_mov_b32_e32 v17, 0 +; SDAG-NEXT: v_lshl_b64 v[38:39], v[2:3], v37 +; SDAG-NEXT: v_lshr_b64 v[2:3], v[2:3], v48 +; SDAG-NEXT: v_addc_u32_e32 v37, vcc, -1, v30, vcc +; SDAG-NEXT: v_or_b32_e32 v11, v11, v39 +; SDAG-NEXT: v_or_b32_e32 v10, v10, v38 +; SDAG-NEXT: v_addc_u32_e32 v38, vcc, -1, v0, vcc +; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v32 +; SDAG-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v25, 0, v25, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v24, 0, v24, s[4:5] +; SDAG-NEXT: v_addc_u32_e32 v39, vcc, -1, v1, vcc +; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v32 +; SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v17, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v16, vcc +; SDAG-NEXT: v_mov_b32_e32 v11, 0 ; SDAG-NEXT: .LBB0_3: ; %udiv-do-while3 ; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1 -; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1 -; SDAG-NEXT: v_lshrrev_b32_e32 v16, 31, v3 +; SDAG-NEXT: v_lshl_b64 v[16:17], v[24:25], 1 +; SDAG-NEXT: v_lshrrev_b32_e32 v10, 31, v3 ; SDAG-NEXT: v_lshl_b64 v[2:3], v[2:3], 1 -; SDAG-NEXT: v_lshrrev_b32_e32 v38, 31, v9 +; SDAG-NEXT: v_lshrrev_b32_e32 v24, 31, v9 ; SDAG-NEXT: v_lshl_b64 v[8:9], v[8:9], 1 -; SDAG-NEXT: v_lshrrev_b32_e32 v39, 31, v21 +; SDAG-NEXT: v_lshrrev_b32_e32 v25, 31, v21 ; SDAG-NEXT: v_lshl_b64 v[20:21], v[20:21], 1 -; SDAG-NEXT: v_or_b32_e32 v10, v10, v16 -; SDAG-NEXT: v_or_b32_e32 v2, v2, v38 -; SDAG-NEXT: v_or_b32_e32 v8, v8, v39 +; SDAG-NEXT: v_or_b32_e32 v16, v16, v10 +; SDAG-NEXT: v_or_b32_e32 v2, v2, v24 +; SDAG-NEXT: v_or_b32_e32 v8, v8, v25 ; SDAG-NEXT: v_or_b32_e32 v9, v19, v9 -; SDAG-NEXT: v_sub_i32_e32 v16, vcc, v34, v2 +; SDAG-NEXT: v_sub_i32_e32 v10, vcc, v36, v2 ; SDAG-NEXT: v_or_b32_e32 v8, v18, v8 -; SDAG-NEXT: v_subb_u32_e32 v16, vcc, v35, v3, vcc -; SDAG-NEXT: v_subb_u32_e32 v16, vcc, v36, v10, vcc -; SDAG-NEXT: v_subb_u32_e32 v16, vcc, v37, v11, vcc -; SDAG-NEXT: v_ashrrev_i32_e32 v38, 31, v16 -; SDAG-NEXT: v_and_b32_e32 v39, v38, v28 -; SDAG-NEXT: v_and_b32_e32 v48, v38, v29 -; SDAG-NEXT: v_and_b32_e32 v49, v38, v0 -; SDAG-NEXT: v_and_b32_e32 v16, 1, v38 -; SDAG-NEXT: v_and_b32_e32 v38, v38, v1 -; SDAG-NEXT: v_sub_i32_e32 v2, vcc, v2, v39 +; SDAG-NEXT: v_subb_u32_e32 v10, vcc, v37, v3, vcc +; SDAG-NEXT: v_subb_u32_e32 v10, vcc, v38, v16, vcc +; SDAG-NEXT: v_subb_u32_e32 v10, vcc, v39, v17, vcc +; SDAG-NEXT: v_ashrrev_i32_e32 v24, 31, v10 +; SDAG-NEXT: v_and_b32_e32 v25, v24, v31 +; SDAG-NEXT: v_and_b32_e32 v48, v24, v30 +; SDAG-NEXT: v_and_b32_e32 v49, v24, v0 +; SDAG-NEXT: v_and_b32_e32 v10, 1, v24 +; SDAG-NEXT: v_and_b32_e32 v50, v24, v1 +; SDAG-NEXT: v_sub_i32_e32 v2, vcc, v2, v25 ; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v3, v48, vcc -; SDAG-NEXT: v_subb_u32_e32 v10, vcc, v10, v49, vcc -; SDAG-NEXT: v_subb_u32_e32 v11, vcc, v11, v38, vcc -; SDAG-NEXT: v_add_i32_e32 v30, vcc, -1, v30 -; SDAG-NEXT: v_addc_u32_e32 v31, vcc, -1, v31, vcc -; SDAG-NEXT: v_addc_u32_e32 v32, vcc, -1, v32, vcc +; SDAG-NEXT: v_subb_u32_e32 v24, vcc, v16, v49, vcc +; SDAG-NEXT: v_subb_u32_e32 v25, vcc, v17, v50, vcc +; SDAG-NEXT: v_add_i32_e32 v32, vcc, -1, v32 ; SDAG-NEXT: v_addc_u32_e32 v33, vcc, -1, v33, vcc -; SDAG-NEXT: v_or_b32_e32 v38, v30, v32 -; SDAG-NEXT: v_or_b32_e32 v39, v31, v33 -; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[38:39] +; SDAG-NEXT: v_addc_u32_e32 v34, vcc, -1, v34, vcc +; SDAG-NEXT: v_addc_u32_e32 v35, vcc, -1, v35, vcc +; SDAG-NEXT: v_or_b32_e32 v16, v32, v34 +; SDAG-NEXT: v_or_b32_e32 v17, v33, v35 +; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[16:17] ; SDAG-NEXT: v_or_b32_e32 v21, v23, v21 ; SDAG-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; SDAG-NEXT: v_or_b32_e32 v20, v22, v20 -; SDAG-NEXT: v_mov_b32_e32 v23, v17 -; SDAG-NEXT: v_mov_b32_e32 v22, v16 +; SDAG-NEXT: v_mov_b32_e32 v23, v11 +; SDAG-NEXT: v_mov_b32_e32 v22, v10 ; SDAG-NEXT: s_andn2_b64 exec, exec, s[10:11] ; SDAG-NEXT: s_cbranch_execnz .LBB0_3 ; SDAG-NEXT: ; %bb.4: ; %Flow13 @@ -196,68 +198,70 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) { ; SDAG-NEXT: v_lshl_b64 v[2:3], v[20:21], 1 ; SDAG-NEXT: v_or_b32_e32 v0, v0, v8 ; SDAG-NEXT: v_or_b32_e32 v20, v19, v1 -; SDAG-NEXT: v_or_b32_e32 v21, v17, v3 -; SDAG-NEXT: v_or_b32_e32 v17, v18, v0 -; SDAG-NEXT: v_or_b32_e32 v16, v16, v2 +; SDAG-NEXT: v_or_b32_e32 v22, v11, v3 +; SDAG-NEXT: v_or_b32_e32 v21, v18, v0 +; SDAG-NEXT: v_or_b32_e32 v23, v10, v2 ; SDAG-NEXT: .LBB0_6: ; %Flow16 ; SDAG-NEXT: s_or_b64 exec, exec, s[6:7] -; SDAG-NEXT: v_ashrrev_i32_e32 v18, 31, v7 -; SDAG-NEXT: v_ashrrev_i32_e32 v19, 31, v15 +; SDAG-NEXT: v_ashrrev_i32_e32 v16, 31, v7 +; SDAG-NEXT: v_ashrrev_i32_e32 v17, 31, v15 +; SDAG-NEXT: v_sub_i32_e32 v0, vcc, 0, v4 ; SDAG-NEXT: v_mov_b32_e32 v9, 0 ; SDAG-NEXT: s_mov_b64 s[10:11], 0x7f -; SDAG-NEXT: v_mov_b32_e32 v22, v18 -; SDAG-NEXT: v_mov_b32_e32 v23, v19 -; SDAG-NEXT: v_xor_b32_e32 v0, v18, v7 -; SDAG-NEXT: v_xor_b32_e32 v1, v18, v6 -; SDAG-NEXT: v_xor_b32_e32 v3, v18, v5 -; SDAG-NEXT: v_xor_b32_e32 v2, v18, v4 -; SDAG-NEXT: v_xor_b32_e32 v6, v19, v15 -; SDAG-NEXT: v_xor_b32_e32 v7, v19, v14 -; SDAG-NEXT: v_xor_b32_e32 v8, v19, v13 -; SDAG-NEXT: v_xor_b32_e32 v10, v19, v12 -; SDAG-NEXT: v_sub_i32_e32 v2, vcc, v2, v18 -; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v3, v18, vcc -; SDAG-NEXT: v_ffbh_u32_e32 v5, v2 -; SDAG-NEXT: v_subb_u32_e32 v4, vcc, v1, v18, vcc -; SDAG-NEXT: v_add_i32_e64 v1, s[4:5], 32, v5 -; SDAG-NEXT: v_ffbh_u32_e32 v11, v3 -; SDAG-NEXT: v_subb_u32_e32 v5, vcc, v0, v18, vcc +; SDAG-NEXT: v_mov_b32_e32 v18, v16 +; SDAG-NEXT: v_mov_b32_e32 v19, v17 +; SDAG-NEXT: v_subb_u32_e32 v1, vcc, 0, v5, vcc +; SDAG-NEXT: v_subb_u32_e32 v8, vcc, 0, v6, vcc +; SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[6:7] +; SDAG-NEXT: v_cndmask_b32_e64 v3, v5, v1, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v2, v4, v0, s[4:5] +; SDAG-NEXT: v_subb_u32_e32 v0, vcc, 0, v7, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v4, v6, v8, s[4:5] +; SDAG-NEXT: v_ffbh_u32_e32 v1, v2 +; SDAG-NEXT: v_ffbh_u32_e32 v6, v3 +; SDAG-NEXT: v_cndmask_b32_e64 v5, v7, v0, s[4:5] +; SDAG-NEXT: v_sub_i32_e32 v7, vcc, 0, v12 ; SDAG-NEXT: v_or_b32_e32 v0, v2, v4 -; SDAG-NEXT: v_ffbh_u32_e32 v12, v4 -; SDAG-NEXT: v_min_u32_e32 v11, v1, v11 -; SDAG-NEXT: v_sub_i32_e32 v28, vcc, v10, v19 +; SDAG-NEXT: v_ffbh_u32_e32 v8, v4 +; SDAG-NEXT: v_add_i32_e64 v10, s[4:5], 32, v1 +; SDAG-NEXT: v_subb_u32_e32 v11, vcc, 0, v13, vcc ; SDAG-NEXT: v_or_b32_e32 v1, v3, v5 -; SDAG-NEXT: v_add_i32_e64 v10, s[4:5], 32, v12 -; SDAG-NEXT: v_ffbh_u32_e32 v12, v5 -; SDAG-NEXT: v_add_i32_e64 v11, s[4:5], 64, v11 -; SDAG-NEXT: v_addc_u32_e64 v13, s[4:5], 0, 0, s[4:5] -; SDAG-NEXT: v_subb_u32_e32 v29, vcc, v8, v19, vcc -; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[0:1] -; SDAG-NEXT: v_ffbh_u32_e32 v1, v28 -; SDAG-NEXT: v_min_u32_e32 v8, v10, v12 -; SDAG-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v10, v13, 0, s[6:7] -; SDAG-NEXT: v_subb_u32_e32 v0, vcc, v7, v19, vcc -; SDAG-NEXT: v_add_i32_e64 v7, s[8:9], 32, v1 -; SDAG-NEXT: v_ffbh_u32_e32 v12, v29 -; SDAG-NEXT: v_cndmask_b32_e64 v8, v11, v8, s[6:7] -; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v6, v19, vcc -; SDAG-NEXT: v_or_b32_e32 v6, v28, v0 -; SDAG-NEXT: v_ffbh_u32_e32 v11, v0 -; SDAG-NEXT: v_min_u32_e32 v12, v7, v12 -; SDAG-NEXT: v_or_b32_e32 v7, v29, v1 -; SDAG-NEXT: v_add_i32_e32 v11, vcc, 32, v11 -; SDAG-NEXT: v_ffbh_u32_e32 v13, v1 -; SDAG-NEXT: v_add_i32_e32 v12, vcc, 64, v12 -; SDAG-NEXT: v_addc_u32_e64 v14, s[6:7], 0, 0, vcc +; SDAG-NEXT: v_add_i32_e64 v8, s[4:5], 32, v8 +; SDAG-NEXT: v_ffbh_u32_e32 v30, v5 +; SDAG-NEXT: v_min_u32_e32 v6, v10, v6 +; SDAG-NEXT: v_subb_u32_e32 v10, vcc, 0, v14, vcc +; SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[14:15] +; SDAG-NEXT: v_cndmask_b32_e64 v24, v13, v11, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v25, v12, v7, s[4:5] +; SDAG-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[0:1] +; SDAG-NEXT: v_min_u32_e32 v1, v8, v30 +; SDAG-NEXT: v_add_i32_e64 v6, s[8:9], 64, v6 +; SDAG-NEXT: v_addc_u32_e64 v7, s[8:9], 0, 0, s[8:9] +; SDAG-NEXT: v_subb_u32_e32 v8, vcc, 0, v15, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v0, v14, v10, s[4:5] +; SDAG-NEXT: v_ffbh_u32_e32 v10, v25 +; SDAG-NEXT: v_ffbh_u32_e32 v11, v24 +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v12, v7, 0, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v13, v6, v1, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v1, v15, v8, s[4:5] +; SDAG-NEXT: v_or_b32_e32 v6, v25, v0 +; SDAG-NEXT: v_ffbh_u32_e32 v8, v0 +; SDAG-NEXT: v_add_i32_e32 v10, vcc, 32, v10 +; SDAG-NEXT: v_or_b32_e32 v7, v24, v1 +; SDAG-NEXT: v_add_i32_e32 v8, vcc, 32, v8 +; SDAG-NEXT: v_ffbh_u32_e32 v14, v1 +; SDAG-NEXT: v_min_u32_e32 v10, v10, v11 ; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] -; SDAG-NEXT: v_min_u32_e32 v6, v11, v13 -; SDAG-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[0:1] -; SDAG-NEXT: v_cndmask_b32_e64 v7, v14, 0, s[6:7] -; SDAG-NEXT: s_or_b64 s[8:9], vcc, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v6, v12, v6, s[6:7] -; SDAG-NEXT: v_sub_i32_e32 v6, vcc, v6, v8 -; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v7, v10, vcc +; SDAG-NEXT: v_min_u32_e32 v6, v8, v14 +; SDAG-NEXT: v_add_i32_e64 v7, s[4:5], 64, v10 +; SDAG-NEXT: v_addc_u32_e64 v8, s[4:5], 0, 0, s[4:5] +; SDAG-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e64 v8, v8, 0, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc +; SDAG-NEXT: v_sub_i32_e32 v6, vcc, v6, v13 +; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v8, v12, vcc ; SDAG-NEXT: v_xor_b32_e32 v10, 0x7f, v6 ; SDAG-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v9, vcc ; SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], s[10:11], v[6:7] @@ -272,7 +276,7 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) { ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11] ; SDAG-NEXT: v_and_b32_e32 v10, 1, v12 ; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v10 -; SDAG-NEXT: s_or_b64 s[4:5], s[8:9], s[4:5] +; SDAG-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5] ; SDAG-NEXT: v_cndmask_b32_e64 v13, v5, 0, s[4:5] ; SDAG-NEXT: s_xor_b64 s[6:7], s[4:5], -1 ; SDAG-NEXT: v_cndmask_b32_e64 v11, v4, 0, s[4:5] @@ -318,7 +322,7 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) { ; SDAG-NEXT: v_sub_i32_e32 v35, vcc, 64, v30 ; SDAG-NEXT: v_subrev_i32_e32 v36, vcc, 64, v30 ; SDAG-NEXT: v_lshr_b64 v[37:38], v[4:5], v30 -; SDAG-NEXT: v_add_i32_e32 v34, vcc, -1, v28 +; SDAG-NEXT: v_add_i32_e32 v34, vcc, -1, v25 ; SDAG-NEXT: s_mov_b64 s[10:11], 0 ; SDAG-NEXT: v_mov_b32_e32 v14, 0 ; SDAG-NEXT: v_mov_b32_e32 v15, 0 @@ -326,7 +330,7 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) { ; SDAG-NEXT: v_mov_b32_e32 v13, 0 ; SDAG-NEXT: v_lshl_b64 v[48:49], v[4:5], v35 ; SDAG-NEXT: v_lshr_b64 v[4:5], v[4:5], v36 -; SDAG-NEXT: v_addc_u32_e32 v35, vcc, -1, v29, vcc +; SDAG-NEXT: v_addc_u32_e32 v35, vcc, -1, v24, vcc ; SDAG-NEXT: v_or_b32_e32 v11, v11, v49 ; SDAG-NEXT: v_or_b32_e32 v10, v10, v48 ; SDAG-NEXT: v_addc_u32_e32 v36, vcc, -1, v0, vcc @@ -363,8 +367,8 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) { ; SDAG-NEXT: v_and_b32_e32 v10, 1, v15 ; SDAG-NEXT: v_and_b32_e32 v38, v15, v1 ; SDAG-NEXT: v_and_b32_e32 v39, v15, v0 -; SDAG-NEXT: v_and_b32_e32 v48, v15, v29 -; SDAG-NEXT: v_and_b32_e32 v15, v15, v28 +; SDAG-NEXT: v_and_b32_e32 v48, v15, v24 +; SDAG-NEXT: v_and_b32_e32 v15, v15, v25 ; SDAG-NEXT: v_sub_i32_e32 v2, vcc, v2, v15 ; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v3, v48, vcc ; SDAG-NEXT: v_subb_u32_e32 v4, vcc, v4, v39, vcc @@ -396,14 +400,14 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) { ; SDAG-NEXT: v_or_b32_e32 v10, v10, v2 ; SDAG-NEXT: .LBB0_12: ; %Flow12 ; SDAG-NEXT: s_or_b64 exec, exec, s[6:7] -; SDAG-NEXT: v_xor_b32_e32 v3, v27, v26 -; SDAG-NEXT: v_xor_b32_e32 v2, v25, v24 -; SDAG-NEXT: v_xor_b32_e32 v7, v23, v22 -; SDAG-NEXT: v_xor_b32_e32 v6, v19, v18 +; SDAG-NEXT: v_xor_b32_e32 v3, v29, v28 +; SDAG-NEXT: v_xor_b32_e32 v2, v27, v26 +; SDAG-NEXT: v_xor_b32_e32 v7, v19, v18 +; SDAG-NEXT: v_xor_b32_e32 v6, v17, v16 ; SDAG-NEXT: v_xor_b32_e32 v4, v20, v3 -; SDAG-NEXT: v_xor_b32_e32 v5, v17, v2 -; SDAG-NEXT: v_xor_b32_e32 v1, v21, v3 -; SDAG-NEXT: v_xor_b32_e32 v0, v16, v2 +; SDAG-NEXT: v_xor_b32_e32 v5, v21, v2 +; SDAG-NEXT: v_xor_b32_e32 v1, v22, v3 +; SDAG-NEXT: v_xor_b32_e32 v0, v23, v2 ; SDAG-NEXT: v_xor_b32_e32 v8, v13, v7 ; SDAG-NEXT: v_xor_b32_e32 v9, v11, v6 ; SDAG-NEXT: v_xor_b32_e32 v11, v14, v7 @@ -1553,118 +1557,119 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) { ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SDAG-NEXT: buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill ; SDAG-NEXT: v_ashrrev_i32_e32 v28, 31, v3 -; SDAG-NEXT: v_ashrrev_i32_e32 v16, 31, v11 -; SDAG-NEXT: v_mov_b32_e32 v17, 0 +; SDAG-NEXT: v_sub_i32_e32 v16, vcc, 0, v0 +; SDAG-NEXT: v_mov_b32_e32 v19, 0 ; SDAG-NEXT: s_mov_b64 s[10:11], 0x7f ; SDAG-NEXT: v_mov_b32_e32 v29, v28 -; SDAG-NEXT: v_xor_b32_e32 v18, v3, v28 -; SDAG-NEXT: v_xor_b32_e32 v19, v2, v28 -; SDAG-NEXT: v_xor_b32_e32 v1, v1, v28 -; SDAG-NEXT: v_xor_b32_e32 v0, v0, v28 -; SDAG-NEXT: v_xor_b32_e32 v11, v11, v16 -; SDAG-NEXT: v_xor_b32_e32 v10, v10, v16 -; SDAG-NEXT: v_xor_b32_e32 v20, v9, v16 -; SDAG-NEXT: v_xor_b32_e32 v9, v8, v16 -; SDAG-NEXT: v_sub_i32_e32 v2, vcc, v0, v28 -; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v1, v28, vcc -; SDAG-NEXT: v_ffbh_u32_e32 v1, v2 -; SDAG-NEXT: v_subb_u32_e32 v0, vcc, v19, v28, vcc -; SDAG-NEXT: v_add_i32_e64 v19, s[4:5], 32, v1 -; SDAG-NEXT: v_ffbh_u32_e32 v21, v3 -; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v18, v28, vcc -; SDAG-NEXT: v_or_b32_e32 v8, v2, v0 -; SDAG-NEXT: v_ffbh_u32_e32 v18, v0 -; SDAG-NEXT: v_min_u32_e32 v19, v19, v21 -; SDAG-NEXT: v_sub_i32_e32 v31, vcc, v9, v16 -; SDAG-NEXT: v_or_b32_e32 v9, v3, v1 +; SDAG-NEXT: v_subb_u32_e32 v17, vcc, 0, v1, vcc +; SDAG-NEXT: v_subb_u32_e32 v18, vcc, 0, v2, vcc +; SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[2:3] +; SDAG-NEXT: v_cndmask_b32_e64 v17, v1, v17, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v16, v0, v16, s[4:5] +; SDAG-NEXT: v_subb_u32_e32 v1, vcc, 0, v3, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v0, v2, v18, s[4:5] +; SDAG-NEXT: v_ffbh_u32_e32 v18, v16 +; SDAG-NEXT: v_ffbh_u32_e32 v20, v17 +; SDAG-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5] +; SDAG-NEXT: v_sub_i32_e32 v21, vcc, 0, v8 +; SDAG-NEXT: v_or_b32_e32 v2, v16, v0 +; SDAG-NEXT: v_ffbh_u32_e32 v22, v0 ; SDAG-NEXT: v_add_i32_e64 v18, s[4:5], 32, v18 -; SDAG-NEXT: v_ffbh_u32_e32 v21, v1 -; SDAG-NEXT: v_add_i32_e64 v19, s[4:5], 64, v19 -; SDAG-NEXT: v_addc_u32_e64 v22, s[4:5], 0, 0, s[4:5] -; SDAG-NEXT: v_subb_u32_e32 v30, vcc, v20, v16, vcc -; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[8:9] -; SDAG-NEXT: v_ffbh_u32_e32 v9, v31 -; SDAG-NEXT: v_min_u32_e32 v18, v18, v21 -; SDAG-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[0:1] -; SDAG-NEXT: v_cndmask_b32_e64 v20, v22, 0, s[6:7] -; SDAG-NEXT: v_subb_u32_e32 v8, vcc, v10, v16, vcc -; SDAG-NEXT: v_add_i32_e64 v21, s[8:9], 32, v9 -; SDAG-NEXT: v_ffbh_u32_e32 v22, v30 -; SDAG-NEXT: v_cndmask_b32_e64 v18, v19, v18, s[6:7] -; SDAG-NEXT: v_subb_u32_e32 v9, vcc, v11, v16, vcc -; SDAG-NEXT: v_or_b32_e32 v10, v31, v8 -; SDAG-NEXT: v_ffbh_u32_e32 v16, v8 -; SDAG-NEXT: v_min_u32_e32 v19, v21, v22 -; SDAG-NEXT: v_or_b32_e32 v11, v30, v9 -; SDAG-NEXT: v_add_i32_e32 v16, vcc, 32, v16 -; SDAG-NEXT: v_ffbh_u32_e32 v21, v9 -; SDAG-NEXT: v_add_i32_e32 v19, vcc, 64, v19 -; SDAG-NEXT: v_addc_u32_e64 v22, s[6:7], 0, 0, vcc -; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11] -; SDAG-NEXT: v_min_u32_e32 v10, v16, v21 -; SDAG-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[8:9] -; SDAG-NEXT: v_cndmask_b32_e64 v11, v22, 0, s[6:7] -; SDAG-NEXT: s_or_b64 s[8:9], vcc, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v10, v19, v10, s[6:7] -; SDAG-NEXT: v_sub_i32_e32 v10, vcc, v10, v18 -; SDAG-NEXT: v_subb_u32_e32 v11, vcc, v11, v20, vcc -; SDAG-NEXT: v_xor_b32_e32 v16, 0x7f, v10 -; SDAG-NEXT: v_subbrev_u32_e32 v18, vcc, 0, v17, vcc -; SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], s[10:11], v[10:11] +; SDAG-NEXT: v_subb_u32_e32 v23, vcc, 0, v9, vcc +; SDAG-NEXT: v_or_b32_e32 v3, v17, v1 +; SDAG-NEXT: v_add_i32_e64 v22, s[4:5], 32, v22 +; SDAG-NEXT: v_ffbh_u32_e32 v24, v1 +; SDAG-NEXT: v_min_u32_e32 v18, v18, v20 +; SDAG-NEXT: v_subb_u32_e32 v20, vcc, 0, v10, vcc +; SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[10:11] +; SDAG-NEXT: v_cndmask_b32_e64 v30, v9, v23, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v31, v8, v21, s[4:5] +; SDAG-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[2:3] +; SDAG-NEXT: v_min_u32_e32 v3, v22, v24 +; SDAG-NEXT: v_add_i32_e64 v8, s[8:9], 64, v18 +; SDAG-NEXT: v_addc_u32_e64 v9, s[8:9], 0, 0, s[8:9] +; SDAG-NEXT: v_subb_u32_e32 v18, vcc, 0, v11, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v20, s[4:5] +; SDAG-NEXT: v_ffbh_u32_e32 v10, v31 +; SDAG-NEXT: v_ffbh_u32_e32 v20, v30 +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e64 v21, v9, 0, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v22, v8, v3, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v18, s[4:5] +; SDAG-NEXT: v_or_b32_e32 v8, v31, v2 +; SDAG-NEXT: v_ffbh_u32_e32 v11, v2 +; SDAG-NEXT: v_add_i32_e32 v10, vcc, 32, v10 +; SDAG-NEXT: v_or_b32_e32 v9, v30, v3 +; SDAG-NEXT: v_add_i32_e32 v11, vcc, 32, v11 +; SDAG-NEXT: v_ffbh_u32_e32 v18, v3 +; SDAG-NEXT: v_min_u32_e32 v10, v10, v20 +; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] +; SDAG-NEXT: v_min_u32_e32 v8, v11, v18 +; SDAG-NEXT: v_add_i32_e64 v9, s[4:5], 64, v10 +; SDAG-NEXT: v_addc_u32_e64 v10, s[4:5], 0, 0, s[4:5] +; SDAG-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] +; SDAG-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc +; SDAG-NEXT: v_sub_i32_e32 v8, vcc, v8, v22 +; SDAG-NEXT: v_subb_u32_e32 v9, vcc, v10, v21, vcc +; SDAG-NEXT: v_xor_b32_e32 v10, 0x7f, v8 +; SDAG-NEXT: v_subbrev_u32_e32 v18, vcc, 0, v19, vcc +; SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], s[10:11], v[8:9] ; SDAG-NEXT: v_cndmask_b32_e64 v20, 0, 1, s[4:5] -; SDAG-NEXT: v_subbrev_u32_e32 v19, vcc, 0, v17, vcc -; SDAG-NEXT: v_or_b32_e32 v16, v16, v18 +; SDAG-NEXT: v_subbrev_u32_e32 v19, vcc, 0, v19, vcc +; SDAG-NEXT: v_or_b32_e32 v10, v10, v18 ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[18:19] ; SDAG-NEXT: v_cndmask_b32_e64 v21, 0, 1, vcc -; SDAG-NEXT: v_or_b32_e32 v17, v11, v19 +; SDAG-NEXT: v_or_b32_e32 v11, v9, v19 ; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[18:19] ; SDAG-NEXT: v_cndmask_b32_e32 v20, v21, v20, vcc -; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[16:17] -; SDAG-NEXT: v_and_b32_e32 v16, 1, v20 -; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v16 -; SDAG-NEXT: s_or_b64 s[4:5], s[8:9], s[4:5] +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11] +; SDAG-NEXT: v_and_b32_e32 v10, 1, v20 +; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v10 +; SDAG-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5] ; SDAG-NEXT: v_cndmask_b32_e64 v35, v1, 0, s[4:5] ; SDAG-NEXT: s_xor_b64 s[6:7], s[4:5], -1 ; SDAG-NEXT: v_cndmask_b32_e64 v32, v0, 0, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v27, v3, 0, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v33, v2, 0, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v27, v17, 0, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v33, v16, 0, s[4:5] ; SDAG-NEXT: s_and_b64 s[4:5], s[6:7], vcc ; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] ; SDAG-NEXT: s_cbranch_execz .LBB2_6 ; SDAG-NEXT: ; %bb.1: ; %udiv-bb15 -; SDAG-NEXT: v_add_i32_e32 v32, vcc, 1, v10 -; SDAG-NEXT: v_sub_i32_e64 v20, s[4:5], 63, v10 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 -; SDAG-NEXT: v_mov_b32_e32 v17, 0 -; SDAG-NEXT: v_addc_u32_e32 v33, vcc, 0, v11, vcc -; SDAG-NEXT: v_lshl_b64 v[20:21], v[2:3], v20 +; SDAG-NEXT: v_add_i32_e32 v32, vcc, 1, v8 +; SDAG-NEXT: v_sub_i32_e64 v20, s[4:5], 63, v8 +; SDAG-NEXT: v_mov_b32_e32 v10, 0 +; SDAG-NEXT: v_mov_b32_e32 v11, 0 +; SDAG-NEXT: v_addc_u32_e32 v33, vcc, 0, v9, vcc +; SDAG-NEXT: v_lshl_b64 v[20:21], v[16:17], v20 ; SDAG-NEXT: v_addc_u32_e32 v34, vcc, 0, v18, vcc ; SDAG-NEXT: v_addc_u32_e32 v35, vcc, 0, v19, vcc ; SDAG-NEXT: v_or_b32_e32 v18, v32, v34 -; SDAG-NEXT: v_sub_i32_e32 v24, vcc, 0x7f, v10 +; SDAG-NEXT: v_sub_i32_e32 v24, vcc, 0x7f, v8 ; SDAG-NEXT: v_or_b32_e32 v19, v33, v35 -; SDAG-NEXT: v_lshl_b64 v[10:11], v[0:1], v24 +; SDAG-NEXT: v_lshl_b64 v[8:9], v[0:1], v24 ; SDAG-NEXT: v_sub_i32_e32 v25, vcc, 64, v24 -; SDAG-NEXT: v_lshl_b64 v[22:23], v[2:3], v24 +; SDAG-NEXT: v_lshl_b64 v[22:23], v[16:17], v24 ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[18:19] -; SDAG-NEXT: v_lshr_b64 v[18:19], v[2:3], v25 -; SDAG-NEXT: v_or_b32_e32 v11, v11, v19 -; SDAG-NEXT: v_or_b32_e32 v10, v10, v18 +; SDAG-NEXT: v_lshr_b64 v[18:19], v[16:17], v25 +; SDAG-NEXT: v_or_b32_e32 v9, v9, v19 +; SDAG-NEXT: v_or_b32_e32 v8, v8, v18 ; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v24 -; SDAG-NEXT: v_cndmask_b32_e64 v11, v21, v11, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v10, v20, v10, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v9, v21, v9, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v8, v20, v8, s[4:5] ; SDAG-NEXT: v_cndmask_b32_e64 v21, 0, v23, s[4:5] ; SDAG-NEXT: v_cndmask_b32_e64 v20, 0, v22, s[4:5] ; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v24 -; SDAG-NEXT: v_cndmask_b32_e64 v11, v11, v1, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v10, v10, v0, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v9, v9, v1, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v8, v8, v0, s[4:5] ; SDAG-NEXT: v_mov_b32_e32 v18, 0 ; SDAG-NEXT: v_mov_b32_e32 v19, 0 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc ; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; SDAG-NEXT: s_cbranch_execz .LBB2_5 ; SDAG-NEXT: ; %bb.2: ; %udiv-preheader4 -; SDAG-NEXT: v_lshr_b64 v[16:17], v[2:3], v32 +; SDAG-NEXT: v_lshr_b64 v[10:11], v[16:17], v32 ; SDAG-NEXT: v_sub_i32_e32 v26, vcc, 64, v32 ; SDAG-NEXT: v_subrev_i32_e32 v37, vcc, 64, v32 ; SDAG-NEXT: v_lshr_b64 v[24:25], v[0:1], v32 @@ -1677,43 +1682,43 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) { ; SDAG-NEXT: v_lshl_b64 v[26:27], v[0:1], v26 ; SDAG-NEXT: v_lshr_b64 v[48:49], v[0:1], v37 ; SDAG-NEXT: v_addc_u32_e32 v37, vcc, -1, v30, vcc -; SDAG-NEXT: v_or_b32_e32 v17, v17, v27 -; SDAG-NEXT: v_or_b32_e32 v16, v16, v26 -; SDAG-NEXT: v_addc_u32_e32 v38, vcc, -1, v8, vcc +; SDAG-NEXT: v_or_b32_e32 v11, v11, v27 +; SDAG-NEXT: v_or_b32_e32 v10, v10, v26 +; SDAG-NEXT: v_addc_u32_e32 v38, vcc, -1, v2, vcc ; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v32 -; SDAG-NEXT: v_cndmask_b32_e64 v17, v49, v17, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v16, v48, v16, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v11, v49, v11, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v10, v48, v10, s[4:5] ; SDAG-NEXT: v_cndmask_b32_e64 v27, 0, v25, s[4:5] ; SDAG-NEXT: v_cndmask_b32_e64 v26, 0, v24, s[4:5] -; SDAG-NEXT: v_addc_u32_e32 v39, vcc, -1, v9, vcc +; SDAG-NEXT: v_addc_u32_e32 v39, vcc, -1, v3, vcc ; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v32 -; SDAG-NEXT: v_cndmask_b32_e32 v25, v17, v3, vcc -; SDAG-NEXT: v_cndmask_b32_e32 v24, v16, v2, vcc -; SDAG-NEXT: v_mov_b32_e32 v17, 0 +; SDAG-NEXT: v_cndmask_b32_e32 v25, v11, v17, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v24, v10, v16, vcc +; SDAG-NEXT: v_mov_b32_e32 v11, 0 ; SDAG-NEXT: .LBB2_3: ; %udiv-do-while3 ; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1 ; SDAG-NEXT: v_lshl_b64 v[26:27], v[26:27], 1 -; SDAG-NEXT: v_lshrrev_b32_e32 v16, 31, v25 +; SDAG-NEXT: v_lshrrev_b32_e32 v10, 31, v25 ; SDAG-NEXT: v_lshl_b64 v[24:25], v[24:25], 1 -; SDAG-NEXT: v_lshrrev_b32_e32 v48, 31, v11 -; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1 +; SDAG-NEXT: v_lshrrev_b32_e32 v48, 31, v9 +; SDAG-NEXT: v_lshl_b64 v[8:9], v[8:9], 1 ; SDAG-NEXT: v_lshrrev_b32_e32 v49, 31, v21 ; SDAG-NEXT: v_lshl_b64 v[20:21], v[20:21], 1 -; SDAG-NEXT: v_or_b32_e32 v26, v26, v16 +; SDAG-NEXT: v_or_b32_e32 v26, v26, v10 ; SDAG-NEXT: v_or_b32_e32 v24, v24, v48 -; SDAG-NEXT: v_or_b32_e32 v10, v10, v49 -; SDAG-NEXT: v_or_b32_e32 v11, v19, v11 -; SDAG-NEXT: v_sub_i32_e32 v16, vcc, v36, v24 -; SDAG-NEXT: v_or_b32_e32 v10, v18, v10 -; SDAG-NEXT: v_subb_u32_e32 v16, vcc, v37, v25, vcc -; SDAG-NEXT: v_subb_u32_e32 v16, vcc, v38, v26, vcc -; SDAG-NEXT: v_subb_u32_e32 v16, vcc, v39, v27, vcc -; SDAG-NEXT: v_ashrrev_i32_e32 v16, 31, v16 -; SDAG-NEXT: v_and_b32_e32 v48, v16, v31 -; SDAG-NEXT: v_and_b32_e32 v49, v16, v30 -; SDAG-NEXT: v_and_b32_e32 v50, v16, v8 -; SDAG-NEXT: v_and_b32_e32 v51, v16, v9 -; SDAG-NEXT: v_and_b32_e32 v16, 1, v16 +; SDAG-NEXT: v_or_b32_e32 v8, v8, v49 +; SDAG-NEXT: v_or_b32_e32 v9, v19, v9 +; SDAG-NEXT: v_sub_i32_e32 v10, vcc, v36, v24 +; SDAG-NEXT: v_or_b32_e32 v8, v18, v8 +; SDAG-NEXT: v_subb_u32_e32 v10, vcc, v37, v25, vcc +; SDAG-NEXT: v_subb_u32_e32 v10, vcc, v38, v26, vcc +; SDAG-NEXT: v_subb_u32_e32 v10, vcc, v39, v27, vcc +; SDAG-NEXT: v_ashrrev_i32_e32 v10, 31, v10 +; SDAG-NEXT: v_and_b32_e32 v48, v10, v31 +; SDAG-NEXT: v_and_b32_e32 v49, v10, v30 +; SDAG-NEXT: v_and_b32_e32 v50, v10, v2 +; SDAG-NEXT: v_and_b32_e32 v51, v10, v3 +; SDAG-NEXT: v_and_b32_e32 v10, 1, v10 ; SDAG-NEXT: v_sub_i32_e32 v24, vcc, v24, v48 ; SDAG-NEXT: v_subb_u32_e32 v25, vcc, v25, v49, vcc ; SDAG-NEXT: v_subb_u32_e32 v26, vcc, v26, v50, vcc @@ -1728,137 +1733,138 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) { ; SDAG-NEXT: v_or_b32_e32 v21, v23, v21 ; SDAG-NEXT: s_or_b64 s[10:11], vcc, s[10:11] ; SDAG-NEXT: v_or_b32_e32 v20, v22, v20 -; SDAG-NEXT: v_mov_b32_e32 v23, v17 -; SDAG-NEXT: v_mov_b32_e32 v22, v16 +; SDAG-NEXT: v_mov_b32_e32 v23, v11 +; SDAG-NEXT: v_mov_b32_e32 v22, v10 ; SDAG-NEXT: s_andn2_b64 exec, exec, s[10:11] ; SDAG-NEXT: s_cbranch_execnz .LBB2_3 ; SDAG-NEXT: ; %bb.4: ; %Flow13 ; SDAG-NEXT: s_or_b64 exec, exec, s[10:11] ; SDAG-NEXT: .LBB2_5: ; %Flow14 ; SDAG-NEXT: s_or_b64 exec, exec, s[8:9] -; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1 +; SDAG-NEXT: v_lshl_b64 v[8:9], v[8:9], 1 ; SDAG-NEXT: v_lshrrev_b32_e32 v22, 31, v21 ; SDAG-NEXT: v_lshl_b64 v[20:21], v[20:21], 1 -; SDAG-NEXT: v_or_b32_e32 v10, v10, v22 -; SDAG-NEXT: v_or_b32_e32 v35, v19, v11 -; SDAG-NEXT: v_or_b32_e32 v27, v17, v21 -; SDAG-NEXT: v_or_b32_e32 v32, v18, v10 -; SDAG-NEXT: v_or_b32_e32 v33, v16, v20 +; SDAG-NEXT: v_or_b32_e32 v8, v8, v22 +; SDAG-NEXT: v_or_b32_e32 v35, v19, v9 +; SDAG-NEXT: v_or_b32_e32 v27, v11, v21 +; SDAG-NEXT: v_or_b32_e32 v32, v18, v8 +; SDAG-NEXT: v_or_b32_e32 v33, v10, v20 ; SDAG-NEXT: .LBB2_6: ; %Flow16 ; SDAG-NEXT: s_or_b64 exec, exec, s[6:7] ; SDAG-NEXT: v_ashrrev_i32_e32 v26, 31, v7 -; SDAG-NEXT: v_ashrrev_i32_e32 v16, 31, v15 -; SDAG-NEXT: v_mov_b32_e32 v17, 0 +; SDAG-NEXT: v_sub_i32_e32 v8, vcc, 0, v4 +; SDAG-NEXT: v_mov_b32_e32 v18, 0 ; SDAG-NEXT: s_mov_b64 s[10:11], 0x7f ; SDAG-NEXT: v_mov_b32_e32 v34, v26 -; SDAG-NEXT: v_xor_b32_e32 v10, v7, v26 -; SDAG-NEXT: v_xor_b32_e32 v11, v6, v26 -; SDAG-NEXT: v_xor_b32_e32 v5, v5, v26 -; SDAG-NEXT: v_xor_b32_e32 v4, v4, v26 -; SDAG-NEXT: v_xor_b32_e32 v15, v15, v16 -; SDAG-NEXT: v_xor_b32_e32 v14, v14, v16 -; SDAG-NEXT: v_xor_b32_e32 v13, v13, v16 -; SDAG-NEXT: v_xor_b32_e32 v12, v12, v16 -; SDAG-NEXT: v_sub_i32_e32 v6, vcc, v4, v26 -; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v5, v26, vcc -; SDAG-NEXT: v_ffbh_u32_e32 v5, v6 -; SDAG-NEXT: v_subb_u32_e32 v4, vcc, v11, v26, vcc -; SDAG-NEXT: v_add_i32_e64 v11, s[4:5], 32, v5 -; SDAG-NEXT: v_ffbh_u32_e32 v18, v7 -; SDAG-NEXT: v_subb_u32_e32 v5, vcc, v10, v26, vcc -; SDAG-NEXT: v_or_b32_e32 v10, v6, v4 -; SDAG-NEXT: v_ffbh_u32_e32 v19, v4 -; SDAG-NEXT: v_min_u32_e32 v18, v11, v18 -; SDAG-NEXT: v_sub_i32_e32 v37, vcc, v12, v16 -; SDAG-NEXT: v_or_b32_e32 v11, v7, v5 -; SDAG-NEXT: v_add_i32_e64 v12, s[4:5], 32, v19 -; SDAG-NEXT: v_ffbh_u32_e32 v19, v5 -; SDAG-NEXT: v_add_i32_e64 v18, s[4:5], 64, v18 -; SDAG-NEXT: v_addc_u32_e64 v20, s[4:5], 0, 0, s[4:5] -; SDAG-NEXT: v_subb_u32_e32 v36, vcc, v13, v16, vcc -; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[10:11] +; SDAG-NEXT: v_subb_u32_e32 v9, vcc, 0, v5, vcc +; SDAG-NEXT: v_subb_u32_e32 v10, vcc, 0, v6, vcc +; SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[6:7] +; SDAG-NEXT: v_cndmask_b32_e64 v9, v5, v9, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v8, v4, v8, s[4:5] +; SDAG-NEXT: v_subb_u32_e32 v5, vcc, 0, v7, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v4, v6, v10, s[4:5] +; SDAG-NEXT: v_ffbh_u32_e32 v10, v8 +; SDAG-NEXT: v_ffbh_u32_e32 v11, v9 +; SDAG-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[4:5] +; SDAG-NEXT: v_sub_i32_e32 v19, vcc, 0, v12 +; SDAG-NEXT: v_or_b32_e32 v6, v8, v4 +; SDAG-NEXT: v_ffbh_u32_e32 v20, v4 +; SDAG-NEXT: v_add_i32_e64 v10, s[4:5], 32, v10 +; SDAG-NEXT: v_subb_u32_e32 v21, vcc, 0, v13, vcc +; SDAG-NEXT: v_or_b32_e32 v7, v9, v5 +; SDAG-NEXT: v_add_i32_e64 v20, s[4:5], 32, v20 +; SDAG-NEXT: v_ffbh_u32_e32 v22, v5 +; SDAG-NEXT: v_min_u32_e32 v10, v10, v11 +; SDAG-NEXT: v_subb_u32_e32 v11, vcc, 0, v14, vcc +; SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], 0, v[14:15] +; SDAG-NEXT: v_cndmask_b32_e64 v36, v13, v21, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v37, v12, v19, s[4:5] +; SDAG-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[6:7] +; SDAG-NEXT: v_min_u32_e32 v7, v20, v22 +; SDAG-NEXT: v_add_i32_e64 v10, s[8:9], 64, v10 +; SDAG-NEXT: v_addc_u32_e64 v12, s[8:9], 0, 0, s[8:9] +; SDAG-NEXT: v_subb_u32_e32 v13, vcc, 0, v15, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v11, s[4:5] ; SDAG-NEXT: v_ffbh_u32_e32 v11, v37 -; SDAG-NEXT: v_min_u32_e32 v12, v12, v19 -; SDAG-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v19, v20, 0, s[6:7] -; SDAG-NEXT: v_subb_u32_e32 v10, vcc, v14, v16, vcc -; SDAG-NEXT: v_add_i32_e64 v13, s[8:9], 32, v11 ; SDAG-NEXT: v_ffbh_u32_e32 v14, v36 -; SDAG-NEXT: v_cndmask_b32_e64 v18, v18, v12, s[6:7] -; SDAG-NEXT: v_subb_u32_e32 v11, vcc, v15, v16, vcc -; SDAG-NEXT: v_or_b32_e32 v12, v37, v10 -; SDAG-NEXT: v_ffbh_u32_e32 v15, v10 -; SDAG-NEXT: v_min_u32_e32 v14, v13, v14 -; SDAG-NEXT: v_or_b32_e32 v13, v36, v11 -; SDAG-NEXT: v_add_i32_e32 v15, vcc, 32, v15 -; SDAG-NEXT: v_ffbh_u32_e32 v16, v11 -; SDAG-NEXT: v_add_i32_e32 v14, vcc, 64, v14 -; SDAG-NEXT: v_addc_u32_e64 v20, s[6:7], 0, 0, vcc +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v12, v12, 0, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v19, v10, v7, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v13, s[4:5] +; SDAG-NEXT: v_or_b32_e32 v10, v37, v6 +; SDAG-NEXT: v_ffbh_u32_e32 v13, v6 +; SDAG-NEXT: v_add_i32_e32 v15, vcc, 32, v11 +; SDAG-NEXT: v_or_b32_e32 v11, v36, v7 +; SDAG-NEXT: v_add_i32_e32 v13, vcc, 32, v13 +; SDAG-NEXT: v_ffbh_u32_e32 v20, v7 +; SDAG-NEXT: v_min_u32_e32 v14, v15, v14 +; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11] +; SDAG-NEXT: v_min_u32_e32 v10, v13, v20 +; SDAG-NEXT: v_add_i32_e64 v11, s[4:5], 64, v14 +; SDAG-NEXT: v_addc_u32_e64 v13, s[4:5], 0, 0, s[4:5] +; SDAG-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7] +; SDAG-NEXT: v_cndmask_b32_e64 v13, v13, 0, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v10, v11, v10, vcc +; SDAG-NEXT: v_sub_i32_e32 v10, vcc, v10, v19 +; SDAG-NEXT: v_subb_u32_e32 v11, vcc, v13, v12, vcc +; SDAG-NEXT: v_xor_b32_e32 v14, 0x7f, v10 +; SDAG-NEXT: v_subbrev_u32_e32 v12, vcc, 0, v18, vcc +; SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], s[10:11], v[10:11] +; SDAG-NEXT: v_cndmask_b32_e64 v19, 0, 1, s[4:5] +; SDAG-NEXT: v_subbrev_u32_e32 v13, vcc, 0, v18, vcc +; SDAG-NEXT: v_or_b32_e32 v14, v14, v12 +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13] +; SDAG-NEXT: v_cndmask_b32_e64 v18, 0, 1, vcc +; SDAG-NEXT: v_or_b32_e32 v15, v11, v13 ; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13] -; SDAG-NEXT: v_min_u32_e32 v12, v15, v16 -; SDAG-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[10:11] -; SDAG-NEXT: v_cndmask_b32_e64 v13, v20, 0, s[6:7] -; SDAG-NEXT: s_or_b64 s[8:9], vcc, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v12, v14, v12, s[6:7] -; SDAG-NEXT: v_sub_i32_e32 v12, vcc, v12, v18 -; SDAG-NEXT: v_subb_u32_e32 v13, vcc, v13, v19, vcc -; SDAG-NEXT: v_xor_b32_e32 v16, 0x7f, v12 -; SDAG-NEXT: v_subbrev_u32_e32 v14, vcc, 0, v17, vcc -; SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], s[10:11], v[12:13] -; SDAG-NEXT: v_cndmask_b32_e64 v18, 0, 1, s[4:5] -; SDAG-NEXT: v_subbrev_u32_e32 v15, vcc, 0, v17, vcc -; SDAG-NEXT: v_or_b32_e32 v16, v16, v14 +; SDAG-NEXT: v_cndmask_b32_e32 v18, v18, v19, vcc ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[14:15] -; SDAG-NEXT: v_cndmask_b32_e64 v19, 0, 1, vcc -; SDAG-NEXT: v_or_b32_e32 v17, v13, v15 -; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15] -; SDAG-NEXT: v_cndmask_b32_e32 v18, v19, v18, vcc -; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[16:17] -; SDAG-NEXT: v_and_b32_e32 v16, 1, v18 -; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v16 -; SDAG-NEXT: s_or_b64 s[4:5], s[8:9], s[4:5] +; SDAG-NEXT: v_and_b32_e32 v14, 1, v18 +; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v14 +; SDAG-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5] ; SDAG-NEXT: v_cndmask_b32_e64 v19, v5, 0, s[4:5] ; SDAG-NEXT: s_xor_b64 s[6:7], s[4:5], -1 ; SDAG-NEXT: v_cndmask_b32_e64 v18, v4, 0, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v17, v7, 0, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v16, v6, 0, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v15, v9, 0, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v14, v8, 0, s[4:5] ; SDAG-NEXT: s_and_b64 s[4:5], s[6:7], vcc ; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] ; SDAG-NEXT: s_cbranch_execz .LBB2_12 ; SDAG-NEXT: ; %bb.7: ; %udiv-bb1 -; SDAG-NEXT: v_add_i32_e32 v38, vcc, 1, v12 -; SDAG-NEXT: v_sub_i32_e64 v18, s[4:5], 63, v12 -; SDAG-NEXT: v_mov_b32_e32 v16, 0 -; SDAG-NEXT: v_mov_b32_e32 v17, 0 -; SDAG-NEXT: v_addc_u32_e32 v39, vcc, 0, v13, vcc -; SDAG-NEXT: v_lshl_b64 v[18:19], v[6:7], v18 -; SDAG-NEXT: v_addc_u32_e32 v48, vcc, 0, v14, vcc -; SDAG-NEXT: v_addc_u32_e32 v49, vcc, 0, v15, vcc -; SDAG-NEXT: v_or_b32_e32 v13, v38, v48 -; SDAG-NEXT: v_sub_i32_e32 v15, vcc, 0x7f, v12 -; SDAG-NEXT: v_or_b32_e32 v14, v39, v49 -; SDAG-NEXT: v_lshl_b64 v[20:21], v[4:5], v15 -; SDAG-NEXT: v_sub_i32_e32 v12, vcc, 64, v15 -; SDAG-NEXT: v_lshl_b64 v[22:23], v[6:7], v15 -; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[13:14] -; SDAG-NEXT: v_lshr_b64 v[12:13], v[6:7], v12 -; SDAG-NEXT: v_or_b32_e32 v13, v21, v13 -; SDAG-NEXT: v_or_b32_e32 v12, v20, v12 -; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v15 -; SDAG-NEXT: v_cndmask_b32_e64 v14, v19, v13, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v18, v18, v12, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v13, 0, v23, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v12, 0, v22, s[4:5] -; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v15 -; SDAG-NEXT: v_cndmask_b32_e64 v15, v14, v5, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v14, v18, v4, s[4:5] +; SDAG-NEXT: v_add_i32_e32 v38, vcc, 1, v10 +; SDAG-NEXT: v_sub_i32_e64 v18, s[4:5], 63, v10 +; SDAG-NEXT: v_mov_b32_e32 v14, 0 +; SDAG-NEXT: v_mov_b32_e32 v15, 0 +; SDAG-NEXT: v_addc_u32_e32 v39, vcc, 0, v11, vcc +; SDAG-NEXT: v_lshl_b64 v[18:19], v[8:9], v18 +; SDAG-NEXT: v_addc_u32_e32 v48, vcc, 0, v12, vcc +; SDAG-NEXT: v_addc_u32_e32 v49, vcc, 0, v13, vcc +; SDAG-NEXT: v_or_b32_e32 v11, v38, v48 +; SDAG-NEXT: v_sub_i32_e32 v13, vcc, 0x7f, v10 +; SDAG-NEXT: v_or_b32_e32 v12, v39, v49 +; SDAG-NEXT: v_lshl_b64 v[20:21], v[4:5], v13 +; SDAG-NEXT: v_sub_i32_e32 v10, vcc, 64, v13 +; SDAG-NEXT: v_lshl_b64 v[22:23], v[8:9], v13 +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[11:12] +; SDAG-NEXT: v_lshr_b64 v[10:11], v[8:9], v10 +; SDAG-NEXT: v_or_b32_e32 v11, v21, v11 +; SDAG-NEXT: v_or_b32_e32 v10, v20, v10 +; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v13 +; SDAG-NEXT: v_cndmask_b32_e64 v12, v19, v11, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v18, v18, v10, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v11, 0, v23, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v10, 0, v22, s[4:5] +; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13 +; SDAG-NEXT: v_cndmask_b32_e64 v13, v12, v5, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v12, v18, v4, s[4:5] ; SDAG-NEXT: v_mov_b32_e32 v18, 0 ; SDAG-NEXT: v_mov_b32_e32 v19, 0 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc ; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; SDAG-NEXT: s_cbranch_execz .LBB2_11 ; SDAG-NEXT: ; %bb.8: ; %udiv-preheader -; SDAG-NEXT: v_lshr_b64 v[16:17], v[6:7], v38 +; SDAG-NEXT: v_lshr_b64 v[14:15], v[8:9], v38 ; SDAG-NEXT: v_sub_i32_e32 v24, vcc, 64, v38 ; SDAG-NEXT: v_subrev_i32_e32 v51, vcc, 64, v38 ; SDAG-NEXT: v_lshr_b64 v[22:23], v[4:5], v38 @@ -1871,42 +1877,42 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) { ; SDAG-NEXT: v_lshl_b64 v[24:25], v[4:5], v24 ; SDAG-NEXT: v_lshr_b64 v[53:54], v[4:5], v51 ; SDAG-NEXT: v_addc_u32_e32 v51, vcc, -1, v36, vcc -; SDAG-NEXT: v_or_b32_e32 v17, v17, v25 -; SDAG-NEXT: v_or_b32_e32 v16, v16, v24 -; SDAG-NEXT: v_addc_u32_e32 v52, vcc, -1, v10, vcc +; SDAG-NEXT: v_or_b32_e32 v15, v15, v25 +; SDAG-NEXT: v_or_b32_e32 v14, v14, v24 +; SDAG-NEXT: v_addc_u32_e32 v52, vcc, -1, v6, vcc ; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v38 -; SDAG-NEXT: v_cndmask_b32_e64 v17, v54, v17, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v16, v53, v16, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v15, v54, v15, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v14, v53, v14, s[4:5] ; SDAG-NEXT: v_cndmask_b32_e64 v25, 0, v23, s[4:5] ; SDAG-NEXT: v_cndmask_b32_e64 v24, 0, v22, s[4:5] -; SDAG-NEXT: v_addc_u32_e32 v53, vcc, -1, v11, vcc +; SDAG-NEXT: v_addc_u32_e32 v53, vcc, -1, v7, vcc ; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v38 -; SDAG-NEXT: v_cndmask_b32_e32 v23, v17, v7, vcc -; SDAG-NEXT: v_cndmask_b32_e32 v22, v16, v6, vcc -; SDAG-NEXT: v_mov_b32_e32 v17, 0 +; SDAG-NEXT: v_cndmask_b32_e32 v23, v15, v9, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v22, v14, v8, vcc +; SDAG-NEXT: v_mov_b32_e32 v15, 0 ; SDAG-NEXT: .LBB2_9: ; %udiv-do-while ; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1 ; SDAG-NEXT: v_lshl_b64 v[24:25], v[24:25], 1 -; SDAG-NEXT: v_lshrrev_b32_e32 v16, 31, v23 +; SDAG-NEXT: v_lshrrev_b32_e32 v14, 31, v23 ; SDAG-NEXT: v_lshl_b64 v[22:23], v[22:23], 1 -; SDAG-NEXT: v_lshrrev_b32_e32 v54, 31, v15 -; SDAG-NEXT: v_lshl_b64 v[14:15], v[14:15], 1 -; SDAG-NEXT: v_lshrrev_b32_e32 v55, 31, v13 +; SDAG-NEXT: v_lshrrev_b32_e32 v54, 31, v13 ; SDAG-NEXT: v_lshl_b64 v[12:13], v[12:13], 1 -; SDAG-NEXT: v_or_b32_e32 v24, v24, v16 +; SDAG-NEXT: v_lshrrev_b32_e32 v55, 31, v11 +; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1 +; SDAG-NEXT: v_or_b32_e32 v24, v24, v14 ; SDAG-NEXT: v_or_b32_e32 v22, v22, v54 -; SDAG-NEXT: v_or_b32_e32 v14, v14, v55 -; SDAG-NEXT: v_or_b32_e32 v15, v19, v15 -; SDAG-NEXT: v_or_b32_e32 v13, v21, v13 -; SDAG-NEXT: v_or_b32_e32 v14, v18, v14 -; SDAG-NEXT: v_sub_i32_e32 v16, vcc, v50, v22 -; SDAG-NEXT: v_subb_u32_e32 v16, vcc, v51, v23, vcc -; SDAG-NEXT: v_subb_u32_e32 v16, vcc, v52, v24, vcc -; SDAG-NEXT: v_subb_u32_e32 v16, vcc, v53, v25, vcc -; SDAG-NEXT: v_ashrrev_i32_e32 v21, 31, v16 -; SDAG-NEXT: v_and_b32_e32 v16, 1, v21 -; SDAG-NEXT: v_and_b32_e32 v54, v21, v11 -; SDAG-NEXT: v_and_b32_e32 v55, v21, v10 +; SDAG-NEXT: v_or_b32_e32 v12, v12, v55 +; SDAG-NEXT: v_or_b32_e32 v13, v19, v13 +; SDAG-NEXT: v_or_b32_e32 v11, v21, v11 +; SDAG-NEXT: v_or_b32_e32 v12, v18, v12 +; SDAG-NEXT: v_sub_i32_e32 v14, vcc, v50, v22 +; SDAG-NEXT: v_subb_u32_e32 v14, vcc, v51, v23, vcc +; SDAG-NEXT: v_subb_u32_e32 v14, vcc, v52, v24, vcc +; SDAG-NEXT: v_subb_u32_e32 v14, vcc, v53, v25, vcc +; SDAG-NEXT: v_ashrrev_i32_e32 v21, 31, v14 +; SDAG-NEXT: v_and_b32_e32 v14, 1, v21 +; SDAG-NEXT: v_and_b32_e32 v54, v21, v7 +; SDAG-NEXT: v_and_b32_e32 v55, v21, v6 ; SDAG-NEXT: v_and_b32_e32 v40, v21, v36 ; SDAG-NEXT: v_and_b32_e32 v21, v21, v37 ; SDAG-NEXT: v_sub_i32_e32 v22, vcc, v22, v21 @@ -1921,89 +1927,87 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) { ; SDAG-NEXT: v_or_b32_e32 v54, v38, v48 ; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[54:55] ; SDAG-NEXT: s_or_b64 s[10:11], vcc, s[10:11] -; SDAG-NEXT: v_or_b32_e32 v12, v20, v12 -; SDAG-NEXT: v_mov_b32_e32 v21, v17 -; SDAG-NEXT: v_mov_b32_e32 v20, v16 +; SDAG-NEXT: v_or_b32_e32 v10, v20, v10 +; SDAG-NEXT: v_mov_b32_e32 v21, v15 +; SDAG-NEXT: v_mov_b32_e32 v20, v14 ; SDAG-NEXT: s_andn2_b64 exec, exec, s[10:11] ; SDAG-NEXT: s_cbranch_execnz .LBB2_9 ; SDAG-NEXT: ; %bb.10: ; %Flow ; SDAG-NEXT: s_or_b64 exec, exec, s[10:11] ; SDAG-NEXT: .LBB2_11: ; %Flow11 ; SDAG-NEXT: s_or_b64 exec, exec, s[8:9] -; SDAG-NEXT: v_lshl_b64 v[14:15], v[14:15], 1 -; SDAG-NEXT: v_lshrrev_b32_e32 v20, 31, v13 ; SDAG-NEXT: v_lshl_b64 v[12:13], v[12:13], 1 -; SDAG-NEXT: v_or_b32_e32 v14, v14, v20 -; SDAG-NEXT: v_or_b32_e32 v19, v19, v15 -; SDAG-NEXT: v_or_b32_e32 v17, v17, v13 -; SDAG-NEXT: v_or_b32_e32 v18, v18, v14 -; SDAG-NEXT: v_or_b32_e32 v16, v16, v12 +; SDAG-NEXT: v_lshrrev_b32_e32 v20, 31, v11 +; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1 +; SDAG-NEXT: v_or_b32_e32 v12, v12, v20 +; SDAG-NEXT: v_or_b32_e32 v19, v19, v13 +; SDAG-NEXT: v_or_b32_e32 v15, v15, v11 +; SDAG-NEXT: v_or_b32_e32 v18, v18, v12 +; SDAG-NEXT: v_or_b32_e32 v14, v14, v10 ; SDAG-NEXT: .LBB2_12: ; %Flow12 ; SDAG-NEXT: s_or_b64 exec, exec, s[6:7] -; SDAG-NEXT: v_mul_lo_u32 v14, v33, v9 -; SDAG-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v33, v8, 0 -; SDAG-NEXT: v_mul_lo_u32 v24, v27, v8 -; SDAG-NEXT: v_mul_lo_u32 v25, v35, v31 -; SDAG-NEXT: v_mul_lo_u32 v35, v32, v30 -; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v31, v33, 0 -; SDAG-NEXT: v_mov_b32_e32 v15, 0 -; SDAG-NEXT: v_mul_lo_u32 v38, v16, v11 -; SDAG-NEXT: v_mad_u64_u32 v[20:21], s[4:5], v16, v10, 0 -; SDAG-NEXT: v_mul_lo_u32 v39, v17, v10 +; SDAG-NEXT: v_mul_lo_u32 v12, v33, v3 +; SDAG-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v33, v2, 0 +; SDAG-NEXT: v_mul_lo_u32 v24, v27, v2 +; SDAG-NEXT: v_mul_lo_u32 v35, v35, v31 +; SDAG-NEXT: v_mul_lo_u32 v38, v32, v30 +; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v31, v33, 0 +; SDAG-NEXT: v_mov_b32_e32 v13, 0 +; SDAG-NEXT: v_mul_lo_u32 v25, v14, v7 +; SDAG-NEXT: v_mad_u64_u32 v[20:21], s[4:5], v14, v6, 0 +; SDAG-NEXT: v_mul_lo_u32 v39, v15, v6 ; SDAG-NEXT: v_mul_lo_u32 v19, v19, v37 ; SDAG-NEXT: v_mul_lo_u32 v48, v18, v36 -; SDAG-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v37, v16, 0 -; SDAG-NEXT: v_add_i32_e32 v13, vcc, v13, v14 -; SDAG-NEXT: v_mov_b32_e32 v14, v9 -; SDAG-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v30, v33, v[14:15] -; SDAG-NEXT: v_sub_i32_e32 v2, vcc, v2, v8 -; SDAG-NEXT: v_add_i32_e64 v14, s[4:5], v21, v38 -; SDAG-NEXT: v_add_i32_e64 v13, s[4:5], v13, v24 +; SDAG-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v37, v14, 0 +; SDAG-NEXT: v_add_i32_e32 v11, vcc, v11, v12 +; SDAG-NEXT: v_mov_b32_e32 v12, v3 +; SDAG-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v30, v33, v[12:13] +; SDAG-NEXT: v_sub_i32_e32 v12, vcc, v16, v2 +; SDAG-NEXT: v_add_i32_e64 v16, s[4:5], v21, v25 +; SDAG-NEXT: v_add_i32_e64 v11, s[4:5], v11, v24 ; SDAG-NEXT: v_mov_b32_e32 v24, v23 -; SDAG-NEXT: v_mov_b32_e32 v23, v15 -; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v31, v27, v[22:23] -; SDAG-NEXT: v_xor_b32_e32 v33, v2, v28 -; SDAG-NEXT: v_add_i32_e64 v21, s[4:5], v14, v39 -; SDAG-NEXT: v_mov_b32_e32 v14, v11 -; SDAG-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v36, v16, v[14:15] -; SDAG-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v32, v31, v[12:13] -; SDAG-NEXT: v_mov_b32_e32 v2, v9 -; SDAG-NEXT: v_add_i32_e64 v13, s[4:5], v24, v2 -; SDAG-NEXT: v_addc_u32_e64 v14, s[4:5], 0, 0, s[4:5] -; SDAG-NEXT: v_mov_b32_e32 v2, v8 -; SDAG-NEXT: v_subb_u32_e32 v16, vcc, v3, v2, vcc +; SDAG-NEXT: v_mov_b32_e32 v23, v13 +; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v31, v27, v[22:23] +; SDAG-NEXT: v_xor_b32_e32 v33, v12, v28 +; SDAG-NEXT: v_add_i32_e64 v21, s[4:5], v16, v39 +; SDAG-NEXT: v_mov_b32_e32 v12, v7 +; SDAG-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v36, v14, v[12:13] +; SDAG-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v32, v31, v[10:11] +; SDAG-NEXT: v_add_i32_e64 v24, s[4:5], v24, v3 +; SDAG-NEXT: v_addc_u32_e64 v25, s[4:5], 0, 0, s[4:5] +; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v17, v2, vcc ; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v18, v37, v[20:21] -; SDAG-NEXT: v_mov_b32_e32 v18, v23 -; SDAG-NEXT: v_mov_b32_e32 v23, v15 -; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v37, v17, v[22:23] -; SDAG-NEXT: v_add_i32_e64 v20, s[4:5], v25, v12 -; SDAG-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v30, v27, v[13:14] -; SDAG-NEXT: v_xor_b32_e32 v16, v16, v29 +; SDAG-NEXT: v_mov_b32_e32 v14, v23 +; SDAG-NEXT: v_mov_b32_e32 v23, v13 +; SDAG-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v37, v15, v[22:23] +; SDAG-NEXT: v_add_i32_e64 v11, s[4:5], v35, v11 +; SDAG-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v30, v27, v[24:25] +; SDAG-NEXT: v_xor_b32_e32 v7, v7, v29 ; SDAG-NEXT: v_add_i32_e64 v3, s[4:5], v19, v3 -; SDAG-NEXT: v_add_i32_e64 v14, s[4:5], v18, v9 -; SDAG-NEXT: v_addc_u32_e64 v15, s[4:5], 0, 0, s[4:5] -; SDAG-NEXT: v_mov_b32_e32 v18, v8 -; SDAG-NEXT: v_add_i32_e64 v19, s[4:5], v35, v20 +; SDAG-NEXT: v_add_i32_e64 v13, s[4:5], v14, v13 +; SDAG-NEXT: v_addc_u32_e64 v14, s[4:5], 0, 0, s[4:5] +; SDAG-NEXT: v_mov_b32_e32 v18, v12 +; SDAG-NEXT: v_add_i32_e64 v19, s[4:5], v38, v11 ; SDAG-NEXT: v_add_i32_e64 v3, s[4:5], v48, v3 -; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v36, v17, v[14:15] -; SDAG-NEXT: v_add_i32_e64 v11, s[4:5], v12, v11 -; SDAG-NEXT: v_addc_u32_e64 v12, s[4:5], v13, v19, s[4:5] -; SDAG-NEXT: v_subb_u32_e32 v0, vcc, v0, v11, vcc -; SDAG-NEXT: v_add_i32_e64 v8, s[4:5], v8, v2 -; SDAG-NEXT: v_addc_u32_e64 v9, s[4:5], v9, v3, s[4:5] -; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v1, v12, vcc +; SDAG-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v36, v15, v[13:14] +; SDAG-NEXT: v_add_i32_e64 v10, s[4:5], v16, v10 +; SDAG-NEXT: v_addc_u32_e64 v13, s[4:5], v17, v19, s[4:5] +; SDAG-NEXT: v_subb_u32_e32 v0, vcc, v0, v10, vcc +; SDAG-NEXT: v_add_i32_e64 v10, s[4:5], v11, v2 +; SDAG-NEXT: v_addc_u32_e64 v11, s[4:5], v12, v3, s[4:5] +; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v1, v13, vcc ; SDAG-NEXT: v_xor_b32_e32 v2, v0, v28 ; SDAG-NEXT: v_xor_b32_e32 v3, v1, v29 ; SDAG-NEXT: v_sub_i32_e32 v0, vcc, v33, v28 -; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v16, v29, vcc +; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v7, v29, vcc ; SDAG-NEXT: v_subb_u32_e32 v2, vcc, v2, v28, vcc ; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v3, v29, vcc -; SDAG-NEXT: v_sub_i32_e32 v6, vcc, v6, v10 -; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v7, v18, vcc +; SDAG-NEXT: v_sub_i32_e32 v6, vcc, v8, v6 +; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v9, v18, vcc ; SDAG-NEXT: v_xor_b32_e32 v6, v6, v26 -; SDAG-NEXT: v_subb_u32_e32 v4, vcc, v4, v8, vcc +; SDAG-NEXT: v_subb_u32_e32 v4, vcc, v4, v10, vcc ; SDAG-NEXT: v_xor_b32_e32 v7, v7, v34 -; SDAG-NEXT: v_subb_u32_e32 v5, vcc, v5, v9, vcc +; SDAG-NEXT: v_subb_u32_e32 v5, vcc, v5, v11, vcc ; SDAG-NEXT: v_xor_b32_e32 v8, v4, v26 ; SDAG-NEXT: v_xor_b32_e32 v9, v5, v34 ; SDAG-NEXT: v_sub_i32_e32 v4, vcc, v6, v26 diff --git a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll index 687188ed5ca3..d4ff845e1edf 100644 --- a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll +++ b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll @@ -331,15 +331,14 @@ bb3: ; preds = %bb3, %bb define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %arg, i32 %arg1) { ; GFX9-LABEL: sdiv32_invariant_denom: ; GFX9: ; %bb.0: ; %bb -; GFX9-NEXT: s_load_dword s3, s[0:1], 0x2c -; GFX9-NEXT: s_mov_b32 s4, 0 +; GFX9-NEXT: s_load_dword s4, s[0:1], 0x2c +; GFX9-NEXT: s_mov_b32 s3, 0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_ashr_i32 s2, s3, 31 -; GFX9-NEXT: s_add_i32 s3, s3, s2 -; GFX9-NEXT: s_xor_b32 s3, s3, s2 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 -; GFX9-NEXT: s_sub_i32 s5, 0, s3 +; GFX9-NEXT: s_abs_i32 s2, s4 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GFX9-NEXT: s_sub_i32 s5, 0, s2 +; GFX9-NEXT: s_ashr_i32 s4, s4, 31 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 @@ -350,25 +349,25 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar ; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: .LBB2_1: ; %bb3 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX9-NEXT: s_mul_hi_u32 s6, s4, s5 -; GFX9-NEXT: s_mul_i32 s7, s6, s3 -; GFX9-NEXT: s_sub_i32 s7, s4, s7 +; GFX9-NEXT: s_mul_hi_u32 s6, s3, s5 +; GFX9-NEXT: s_mul_i32 s7, s6, s2 +; GFX9-NEXT: s_sub_i32 s7, s3, s7 ; GFX9-NEXT: s_add_i32 s8, s6, 1 -; GFX9-NEXT: s_sub_i32 s9, s7, s3 -; GFX9-NEXT: s_cmp_ge_u32 s7, s3 +; GFX9-NEXT: s_sub_i32 s9, s7, s2 +; GFX9-NEXT: s_cmp_ge_u32 s7, s2 ; GFX9-NEXT: s_cselect_b32 s6, s8, s6 ; GFX9-NEXT: s_cselect_b32 s7, s9, s7 ; GFX9-NEXT: s_add_i32 s8, s6, 1 -; GFX9-NEXT: s_cmp_ge_u32 s7, s3 +; GFX9-NEXT: s_cmp_ge_u32 s7, s2 ; GFX9-NEXT: s_cselect_b32 s6, s8, s6 -; GFX9-NEXT: s_xor_b32 s6, s6, s2 -; GFX9-NEXT: s_sub_i32 s6, s6, s2 -; GFX9-NEXT: s_add_i32 s4, s4, 1 +; GFX9-NEXT: s_xor_b32 s6, s6, s4 +; GFX9-NEXT: s_sub_i32 s6, s6, s4 +; GFX9-NEXT: s_add_i32 s3, s3, 1 ; GFX9-NEXT: v_mov_b32_e32 v1, s6 ; GFX9-NEXT: global_store_dword v0, v1, s[0:1] ; GFX9-NEXT: s_add_u32 s0, s0, 4 ; GFX9-NEXT: s_addc_u32 s1, s1, 0 -; GFX9-NEXT: s_cmpk_eq_i32 s4, 0x400 +; GFX9-NEXT: s_cmpk_eq_i32 s3, 0x400 ; GFX9-NEXT: s_cbranch_scc0 .LBB2_1 ; GFX9-NEXT: ; %bb.2: ; %bb2 ; GFX9-NEXT: s_endpgm @@ -377,12 +376,11 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar ; GFX10: ; %bb.0: ; %bb ; GFX10-NEXT: s_load_dword s3, s[0:1], 0x2c ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_ashr_i32 s2, s3, 31 +; GFX10-NEXT: s_abs_i32 s2, s3 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX10-NEXT: s_add_i32 s3, s3, s2 -; GFX10-NEXT: s_xor_b32 s3, s3, s2 -; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s3 -; GFX10-NEXT: s_sub_i32 s4, 0, s3 +; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GFX10-NEXT: s_sub_i32 s4, 0, s2 +; GFX10-NEXT: s_ashr_i32 s3, s3, 31 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0 @@ -395,19 +393,19 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar ; GFX10-NEXT: .LBB2_1: ; %bb3 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX10-NEXT: s_mul_hi_u32 s6, s4, s5 -; GFX10-NEXT: s_mul_i32 s7, s6, s3 +; GFX10-NEXT: s_mul_i32 s7, s6, s2 ; GFX10-NEXT: s_add_i32 s8, s6, 1 ; GFX10-NEXT: s_sub_i32 s7, s4, s7 -; GFX10-NEXT: s_sub_i32 s9, s7, s3 -; GFX10-NEXT: s_cmp_ge_u32 s7, s3 +; GFX10-NEXT: s_sub_i32 s9, s7, s2 +; GFX10-NEXT: s_cmp_ge_u32 s7, s2 ; GFX10-NEXT: s_cselect_b32 s6, s8, s6 ; GFX10-NEXT: s_cselect_b32 s7, s9, s7 ; GFX10-NEXT: s_add_i32 s8, s6, 1 -; GFX10-NEXT: s_cmp_ge_u32 s7, s3 +; GFX10-NEXT: s_cmp_ge_u32 s7, s2 ; GFX10-NEXT: s_cselect_b32 s6, s8, s6 ; GFX10-NEXT: s_add_i32 s4, s4, 1 -; GFX10-NEXT: s_xor_b32 s6, s6, s2 -; GFX10-NEXT: s_sub_i32 s6, s6, s2 +; GFX10-NEXT: s_xor_b32 s6, s6, s3 +; GFX10-NEXT: s_sub_i32 s6, s6, s3 ; GFX10-NEXT: v_mov_b32_e32 v1, s6 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_store_dword v0, v1, s[0:1] @@ -425,22 +423,20 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar ; GFX11-NEXT: s_load_b32 s3, s[0:1], 0x2c ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_ashr_i32 s2, s3, 31 -; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-NEXT: s_add_i32 s3, s3, s2 -; GFX11-NEXT: s_xor_b32 s3, s3, s2 -; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s3 -; GFX11-NEXT: s_sub_i32 s4, 0, s3 +; GFX11-NEXT: s_abs_i32 s2, s3 +; GFX11-NEXT: s_ashr_i32 s3, s3, 31 +; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GFX11-NEXT: s_sub_i32 s4, 0, s2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX11-NEXT: s_waitcnt_depctr 0xfff ; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_readfirstlane_b32 s5, v0 ; GFX11-NEXT: v_mov_b32_e32 v0, 0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-NEXT: s_mul_i32 s4, s4, s5 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-NEXT: s_mul_hi_u32 s6, s5, s4 ; GFX11-NEXT: s_mov_b32 s4, 0 ; GFX11-NEXT: s_add_i32 s5, s5, s6 @@ -449,21 +445,21 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-NEXT: s_mul_hi_u32 s6, s4, s5 -; GFX11-NEXT: s_mul_i32 s7, s6, s3 +; GFX11-NEXT: s_mul_i32 s7, s6, s2 ; GFX11-NEXT: s_add_i32 s8, s6, 1 ; GFX11-NEXT: s_sub_i32 s7, s4, s7 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-NEXT: s_sub_i32 s9, s7, s3 -; GFX11-NEXT: s_cmp_ge_u32 s7, s3 +; GFX11-NEXT: s_sub_i32 s9, s7, s2 +; GFX11-NEXT: s_cmp_ge_u32 s7, s2 ; GFX11-NEXT: s_cselect_b32 s6, s8, s6 ; GFX11-NEXT: s_cselect_b32 s7, s9, s7 ; GFX11-NEXT: s_add_i32 s8, s6, 1 -; GFX11-NEXT: s_cmp_ge_u32 s7, s3 +; GFX11-NEXT: s_cmp_ge_u32 s7, s2 ; GFX11-NEXT: s_cselect_b32 s6, s8, s6 ; GFX11-NEXT: s_add_i32 s4, s4, 1 -; GFX11-NEXT: s_xor_b32 s6, s6, s2 +; GFX11-NEXT: s_xor_b32 s6, s6, s3 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-NEXT: s_sub_i32 s6, s6, s2 +; GFX11-NEXT: s_sub_i32 s6, s6, s3 ; GFX11-NEXT: v_mov_b32_e32 v1, s6 ; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-NEXT: s_add_u32 s0, s0, 4 @@ -495,14 +491,12 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar ; GFX9-LABEL: srem32_invariant_denom: ; GFX9: ; %bb.0: ; %bb ; GFX9-NEXT: s_load_dword s2, s[0:1], 0x2c +; GFX9-NEXT: s_mov_b32 s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_ashr_i32 s3, s2, 31 -; GFX9-NEXT: s_add_i32 s2, s2, s3 -; GFX9-NEXT: s_xor_b32 s2, s2, s3 +; GFX9-NEXT: s_abs_i32 s2, s2 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2 -; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_sub_i32 s4, 0, s2 -; GFX9-NEXT: s_mov_b32 s3, 0 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 @@ -524,7 +518,6 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar ; GFX9-NEXT: s_cselect_b32 s5, s6, s5 ; GFX9-NEXT: s_add_i32 s3, s3, 1 ; GFX9-NEXT: v_mov_b32_e32 v1, s5 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_store_dword v0, v1, s[0:1] ; GFX9-NEXT: s_add_u32 s0, s0, 4 ; GFX9-NEXT: s_addc_u32 s1, s1, 0 @@ -537,10 +530,8 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar ; GFX10: ; %bb.0: ; %bb ; GFX10-NEXT: s_load_dword s2, s[0:1], 0x2c ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_ashr_i32 s3, s2, 31 +; GFX10-NEXT: s_abs_i32 s2, s2 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX10-NEXT: s_add_i32 s2, s2, s3 -; GFX10-NEXT: s_xor_b32 s2, s2, s3 ; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GFX10-NEXT: s_sub_i32 s3, 0, s2 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 @@ -581,10 +572,7 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar ; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x2c ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_ashr_i32 s3, s2, 31 -; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-NEXT: s_add_i32 s2, s2, s3 -; GFX11-NEXT: s_xor_b32 s2, s2, s3 +; GFX11-NEXT: s_abs_i32 s2, s2 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GFX11-NEXT: s_sub_i32 s3, 0, s2 diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll index f950717c591a..3e6de3249245 100644 --- a/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll +++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll @@ -17,15 +17,16 @@ define bfloat @sitofp_i128_to_bf16(i128 %x) { ; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc ; GCN-NEXT: s_cbranch_execz .LBB0_14 ; GCN-NEXT: ; %bb.1: ; %itofp-if-end -; GCN-NEXT: v_ashrrev_i32_e32 v5, 31, v3 -; GCN-NEXT: v_xor_b32_e32 v0, v5, v0 -; GCN-NEXT: v_xor_b32_e32 v1, v5, v1 -; GCN-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v5 -; GCN-NEXT: v_xor_b32_e32 v2, v5, v2 -; GCN-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc -; GCN-NEXT: v_xor_b32_e32 v6, v5, v3 -; GCN-NEXT: v_subb_co_u32_e32 v4, vcc, v2, v5, vcc -; GCN-NEXT: v_subb_co_u32_e32 v5, vcc, v6, v5, vcc +; GCN-NEXT: v_sub_co_u32_e32 v4, vcc, 0, v0 +; GCN-NEXT: v_subb_co_u32_e32 v5, vcc, 0, v1, vcc +; GCN-NEXT: v_subb_co_u32_e32 v6, vcc, 0, v2, vcc +; GCN-NEXT: v_subb_co_u32_e32 v7, vcc, 0, v3, vcc +; GCN-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[2:3] +; GCN-NEXT: ; implicit-def: $vgpr8 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc +; GCN-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc +; GCN-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; GCN-NEXT: v_ffbh_u32_e32 v2, v4 ; GCN-NEXT: v_add_u32_e32 v2, 32, v2 ; GCN-NEXT: v_ffbh_u32_e32 v6, v5 @@ -40,7 +41,6 @@ define bfloat @sitofp_i128_to_bf16(i128 %x) { ; GCN-NEXT: v_sub_u32_e32 v6, 0x80, v7 ; GCN-NEXT: v_sub_u32_e32 v2, 0x7f, v7 ; GCN-NEXT: v_cmp_gt_i32_e32 vcc, 25, v6 -; GCN-NEXT: ; implicit-def: $vgpr8 ; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GCN-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GCN-NEXT: ; %bb.2: ; %itofp-if-else diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll index c6aa2182aec8..c5198cdb421a 100644 --- a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll +++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll @@ -13,15 +13,16 @@ define float @sitofp_i128_to_f32(i128 %x) { ; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc ; SDAG-NEXT: s_cbranch_execz .LBB0_14 ; SDAG-NEXT: ; %bb.1: ; %itofp-if-end -; SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v3 -; SDAG-NEXT: v_xor_b32_e32 v0, v5, v0 -; SDAG-NEXT: v_xor_b32_e32 v1, v5, v1 -; SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v5 -; SDAG-NEXT: v_xor_b32_e32 v2, v5, v2 -; SDAG-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc -; SDAG-NEXT: v_xor_b32_e32 v6, v5, v3 -; SDAG-NEXT: v_subb_co_u32_e32 v4, vcc, v2, v5, vcc -; SDAG-NEXT: v_subb_co_u32_e32 v5, vcc, v6, v5, vcc +; SDAG-NEXT: v_sub_co_u32_e32 v4, vcc, 0, v0 +; SDAG-NEXT: v_subb_co_u32_e32 v5, vcc, 0, v1, vcc +; SDAG-NEXT: v_subb_co_u32_e32 v6, vcc, 0, v2, vcc +; SDAG-NEXT: v_subb_co_u32_e32 v7, vcc, 0, v3, vcc +; SDAG-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[2:3] +; SDAG-NEXT: ; implicit-def: $vgpr8 +; SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; SDAG-NEXT: v_ffbh_u32_e32 v2, v4 ; SDAG-NEXT: v_add_u32_e32 v2, 32, v2 ; SDAG-NEXT: v_ffbh_u32_e32 v6, v5 @@ -36,7 +37,6 @@ define float @sitofp_i128_to_f32(i128 %x) { ; SDAG-NEXT: v_sub_u32_e32 v6, 0x80, v7 ; SDAG-NEXT: v_sub_u32_e32 v2, 0x7f, v7 ; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v6 -; SDAG-NEXT: ; implicit-def: $vgpr8 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc ; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; SDAG-NEXT: ; %bb.2: ; %itofp-if-else @@ -524,16 +524,17 @@ define double @sitofp_i128_to_f64(i128 %x) { ; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc ; SDAG-NEXT: s_cbranch_execz .LBB2_14 ; SDAG-NEXT: ; %bb.1: ; %itofp-if-end -; SDAG-NEXT: v_ashrrev_i32_e32 v0, 31, v3 -; SDAG-NEXT: v_xor_b32_e32 v4, v0, v4 -; SDAG-NEXT: v_xor_b32_e32 v5, v0, v5 -; SDAG-NEXT: v_sub_co_u32_e32 v4, vcc, v4, v0 -; SDAG-NEXT: v_xor_b32_e32 v2, v0, v2 -; SDAG-NEXT: v_subb_co_u32_e32 v5, vcc, v5, v0, vcc -; SDAG-NEXT: v_xor_b32_e32 v1, v0, v3 -; SDAG-NEXT: v_subb_co_u32_e32 v6, vcc, v2, v0, vcc -; SDAG-NEXT: v_subb_co_u32_e32 v7, vcc, v1, v0, vcc +; SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, 0, v4 +; SDAG-NEXT: v_subb_co_u32_e32 v1, vcc, 0, v5, vcc +; SDAG-NEXT: v_subb_co_u32_e32 v6, vcc, 0, v2, vcc +; SDAG-NEXT: v_subb_co_u32_e32 v7, vcc, 0, v3, vcc +; SDAG-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[2:3] +; SDAG-NEXT: ; implicit-def: $vgpr10 +; SDAG-NEXT: v_cndmask_b32_e32 v6, v2, v6, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v7, v3, v7, vcc ; SDAG-NEXT: v_ffbh_u32_e32 v0, v6 +; SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v1, vcc ; SDAG-NEXT: v_add_u32_e32 v0, 32, v0 ; SDAG-NEXT: v_ffbh_u32_e32 v1, v7 ; SDAG-NEXT: v_min_u32_e32 v0, v0, v1 @@ -547,7 +548,6 @@ define double @sitofp_i128_to_f64(i128 %x) { ; SDAG-NEXT: v_sub_u32_e32 v8, 0x80, v9 ; SDAG-NEXT: v_sub_u32_e32 v2, 0x7f, v9 ; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 54, v8 -; SDAG-NEXT: ; implicit-def: $vgpr10 ; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc ; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5] @@ -1103,15 +1103,16 @@ define half @sitofp_i128_to_f16(i128 %x) { ; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc ; SDAG-NEXT: s_cbranch_execz .LBB4_14 ; SDAG-NEXT: ; %bb.1: ; %itofp-if-end -; SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v3 -; SDAG-NEXT: v_xor_b32_e32 v0, v5, v0 -; SDAG-NEXT: v_xor_b32_e32 v1, v5, v1 -; SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v5 -; SDAG-NEXT: v_xor_b32_e32 v2, v5, v2 -; SDAG-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc -; SDAG-NEXT: v_xor_b32_e32 v6, v5, v3 -; SDAG-NEXT: v_subb_co_u32_e32 v4, vcc, v2, v5, vcc -; SDAG-NEXT: v_subb_co_u32_e32 v5, vcc, v6, v5, vcc +; SDAG-NEXT: v_sub_co_u32_e32 v4, vcc, 0, v0 +; SDAG-NEXT: v_subb_co_u32_e32 v5, vcc, 0, v1, vcc +; SDAG-NEXT: v_subb_co_u32_e32 v6, vcc, 0, v2, vcc +; SDAG-NEXT: v_subb_co_u32_e32 v7, vcc, 0, v3, vcc +; SDAG-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[2:3] +; SDAG-NEXT: ; implicit-def: $vgpr8 +; SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc ; SDAG-NEXT: v_ffbh_u32_e32 v2, v4 ; SDAG-NEXT: v_add_u32_e32 v2, 32, v2 ; SDAG-NEXT: v_ffbh_u32_e32 v6, v5 @@ -1126,7 +1127,6 @@ define half @sitofp_i128_to_f16(i128 %x) { ; SDAG-NEXT: v_sub_u32_e32 v6, 0x80, v7 ; SDAG-NEXT: v_sub_u32_e32 v2, 0x7f, v7 ; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v6 -; SDAG-NEXT: ; implicit-def: $vgpr8 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc ; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; SDAG-NEXT: ; %bb.2: ; %itofp-if-else diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll index b068d87c4d6f..6b036f675929 100644 --- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll +++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll @@ -10,29 +10,31 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-LABEL: v_srem_i128_vv: ; GFX9: ; %bb.0: ; %_udiv-special-cases ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, 0, v0 +; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, 0, v1, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, 0, v2, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, 0, v3, vcc +; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[2:3] ; GFX9-NEXT: v_ashrrev_i32_e32 v20, 31, v3 -; GFX9-NEXT: v_xor_b32_e32 v0, v0, v20 -; GFX9-NEXT: v_xor_b32_e32 v10, v2, v20 -; GFX9-NEXT: v_xor_b32_e32 v1, v1, v20 -; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v20 -; GFX9-NEXT: v_xor_b32_e32 v9, v3, v20 -; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v20, vcc -; GFX9-NEXT: v_ashrrev_i32_e32 v8, 31, v7 -; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v10, v20, vcc -; GFX9-NEXT: v_xor_b32_e32 v4, v4, v8 -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v9, v20, vcc -; GFX9-NEXT: v_xor_b32_e32 v5, v5, v8 -; GFX9-NEXT: v_sub_co_u32_e32 v23, vcc, v4, v8 -; GFX9-NEXT: v_xor_b32_e32 v6, v6, v8 -; GFX9-NEXT: v_subb_co_u32_e32 v21, vcc, v5, v8, vcc -; GFX9-NEXT: v_xor_b32_e32 v7, v7, v8 -; GFX9-NEXT: v_subb_co_u32_e32 v4, vcc, v6, v8, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v8, vcc -; GFX9-NEXT: v_or_b32_e32 v7, v21, v5 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc +; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, 0, v4 +; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, 0, v5, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, 0, v6, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, 0, v7, vcc +; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[6:7] +; GFX9-NEXT: v_mov_b32_e32 v21, v20 +; GFX9-NEXT: v_cndmask_b32_e32 v22, v5, v9, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v23, v4, v8, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v10, vcc +; GFX9-NEXT: v_or_b32_e32 v7, v22, v5 ; GFX9-NEXT: v_or_b32_e32 v6, v23, v4 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] -; GFX9-NEXT: v_or_b32_e32 v7, v3, v1 -; GFX9-NEXT: v_or_b32_e32 v6, v2, v0 +; GFX9-NEXT: v_or_b32_e32 v7, v1, v3 +; GFX9-NEXT: v_or_b32_e32 v6, v0, v2 ; GFX9-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[6:7] ; GFX9-NEXT: v_ffbh_u32_e32 v6, v4 ; GFX9-NEXT: v_add_u32_e32 v6, 32, v6 @@ -40,38 +42,37 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-NEXT: v_min_u32_e32 v6, v6, v7 ; GFX9-NEXT: v_ffbh_u32_e32 v7, v23 ; GFX9-NEXT: v_add_u32_e32 v7, 32, v7 -; GFX9-NEXT: v_ffbh_u32_e32 v8, v21 +; GFX9-NEXT: v_ffbh_u32_e32 v8, v22 ; GFX9-NEXT: v_min_u32_e32 v7, v7, v8 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, 64, v7 ; GFX9-NEXT: v_addc_co_u32_e64 v8, s[6:7], 0, 0, vcc ; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] -; GFX9-NEXT: v_ffbh_u32_e32 v9, v1 +; GFX9-NEXT: v_ffbh_u32_e32 v10, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc -; GFX9-NEXT: v_ffbh_u32_e32 v7, v0 +; GFX9-NEXT: v_ffbh_u32_e32 v7, v2 ; GFX9-NEXT: v_add_u32_e32 v7, 32, v7 -; GFX9-NEXT: v_min_u32_e32 v7, v7, v9 -; GFX9-NEXT: v_ffbh_u32_e32 v9, v2 -; GFX9-NEXT: v_add_u32_e32 v9, 32, v9 -; GFX9-NEXT: v_ffbh_u32_e32 v10, v3 -; GFX9-NEXT: v_min_u32_e32 v9, v9, v10 +; GFX9-NEXT: v_min_u32_e32 v7, v7, v10 +; GFX9-NEXT: v_ffbh_u32_e32 v10, v0 +; GFX9-NEXT: v_add_u32_e32 v10, 32, v10 +; GFX9-NEXT: v_ffbh_u32_e32 v11, v1 +; GFX9-NEXT: v_min_u32_e32 v10, v10, v11 ; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, 0, vcc -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, 64, v9 -; GFX9-NEXT: v_addc_co_u32_e64 v10, s[6:7], 0, 0, vcc -; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] -; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f -; GFX9-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc -; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, v6, v7 -; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v8, v10, vcc +; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, 64, v10 +; GFX9-NEXT: v_addc_co_u32_e64 v11, s[6:7], 0, 0, vcc +; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] ; GFX9-NEXT: v_mov_b32_e32 v9, 0 +; GFX9-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, 0, vcc +; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, v6, v7 +; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v8, v11, vcc ; GFX9-NEXT: v_subbrev_co_u32_e32 v8, vcc, 0, v9, vcc ; GFX9-NEXT: v_subbrev_co_u32_e32 v9, vcc, 0, v9, vcc +; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f ; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7] ; GFX9-NEXT: v_or_b32_e32 v13, v7, v9 ; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc ; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9] -; GFX9-NEXT: v_mov_b32_e32 v22, v20 ; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9] ; GFX9-NEXT: v_cndmask_b32_e32 v10, v11, v10, vcc @@ -82,10 +83,10 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc ; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13] ; GFX9-NEXT: s_xor_b64 s[6:7], s[4:5], -1 -; GFX9-NEXT: v_cndmask_b32_e64 v11, v1, 0, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v12, v0, 0, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v10, v3, 0, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v13, v2, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v11, v3, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v12, v2, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v10, v1, 0, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v13, v0, 0, s[4:5] ; GFX9-NEXT: s_and_b64 s[4:5], s[6:7], vcc ; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] ; GFX9-NEXT: s_cbranch_execz .LBB0_6 @@ -98,22 +99,22 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-NEXT: v_sub_u32_e32 v11, 64, v13 ; GFX9-NEXT: v_or_b32_e32 v8, v25, v27 ; GFX9-NEXT: v_or_b32_e32 v7, v24, v26 -; GFX9-NEXT: v_lshlrev_b64 v[9:10], v13, v[0:1] -; GFX9-NEXT: v_lshrrev_b64 v[11:12], v11, v[2:3] +; GFX9-NEXT: v_lshlrev_b64 v[9:10], v13, v[2:3] +; GFX9-NEXT: v_lshrrev_b64 v[11:12], v11, v[0:1] ; GFX9-NEXT: v_sub_u32_e32 v6, 63, v6 ; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[7:8] -; GFX9-NEXT: v_lshlrev_b64 v[6:7], v6, v[2:3] +; GFX9-NEXT: v_lshlrev_b64 v[6:7], v6, v[0:1] ; GFX9-NEXT: v_or_b32_e32 v8, v10, v12 ; GFX9-NEXT: v_or_b32_e32 v9, v9, v11 ; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v13 ; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v13 -; GFX9-NEXT: v_lshlrev_b64 v[12:13], v13, v[2:3] +; GFX9-NEXT: v_lshlrev_b64 v[12:13], v13, v[0:1] ; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v8, s[4:5] ; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[4:5] ; GFX9-NEXT: v_mov_b32_e32 v8, 0 ; GFX9-NEXT: v_mov_b32_e32 v10, 0 -; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v1, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v0, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v3, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v2, s[6:7] ; GFX9-NEXT: v_cndmask_b32_e64 v13, 0, v13, s[4:5] ; GFX9-NEXT: v_mov_b32_e32 v9, 0 ; GFX9-NEXT: v_mov_b32_e32 v11, 0 @@ -123,23 +124,23 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-NEXT: s_cbranch_execz .LBB0_5 ; GFX9-NEXT: ; %bb.2: ; %udiv-preheader ; GFX9-NEXT: v_sub_u32_e32 v10, 64, v24 -; GFX9-NEXT: v_lshrrev_b64 v[8:9], v24, v[2:3] -; GFX9-NEXT: v_lshlrev_b64 v[10:11], v10, v[0:1] +; GFX9-NEXT: v_lshrrev_b64 v[8:9], v24, v[0:1] +; GFX9-NEXT: v_lshlrev_b64 v[10:11], v10, v[2:3] ; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v24 ; GFX9-NEXT: v_or_b32_e32 v10, v8, v10 ; GFX9-NEXT: v_subrev_u32_e32 v8, 64, v24 ; GFX9-NEXT: v_or_b32_e32 v11, v9, v11 -; GFX9-NEXT: v_lshrrev_b64 v[8:9], v8, v[0:1] +; GFX9-NEXT: v_lshrrev_b64 v[8:9], v8, v[2:3] ; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v24 ; GFX9-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v15, v9, v3, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v15, v9, v1, s[4:5] ; GFX9-NEXT: v_cndmask_b32_e32 v10, v8, v10, vcc -; GFX9-NEXT: v_lshrrev_b64 v[8:9], v24, v[0:1] -; GFX9-NEXT: v_cndmask_b32_e64 v14, v10, v2, s[4:5] +; GFX9-NEXT: v_lshrrev_b64 v[8:9], v24, v[2:3] +; GFX9-NEXT: v_cndmask_b32_e64 v14, v10, v0, s[4:5] ; GFX9-NEXT: v_cndmask_b32_e32 v17, 0, v9, vcc ; GFX9-NEXT: v_cndmask_b32_e32 v16, 0, v8, vcc ; GFX9-NEXT: v_add_co_u32_e32 v28, vcc, -1, v23 -; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v21, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v22, vcc ; GFX9-NEXT: v_addc_co_u32_e32 v30, vcc, -1, v4, vcc ; GFX9-NEXT: v_mov_b32_e32 v18, 0 ; GFX9-NEXT: v_mov_b32_e32 v10, 0 @@ -168,7 +169,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-NEXT: v_or_b32_e32 v12, v18, v12 ; GFX9-NEXT: v_and_b32_e32 v18, v8, v23 ; GFX9-NEXT: v_or_b32_e32 v13, v19, v13 -; GFX9-NEXT: v_and_b32_e32 v19, v8, v21 +; GFX9-NEXT: v_and_b32_e32 v19, v8, v22 ; GFX9-NEXT: v_sub_co_u32_e32 v14, vcc, v14, v18 ; GFX9-NEXT: v_and_b32_e32 v32, v8, v4 ; GFX9-NEXT: v_subb_co_u32_e32 v15, vcc, v15, v19, vcc @@ -207,7 +208,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-NEXT: v_mov_b32_e32 v15, 0 ; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v13, v4, 0 ; GFX9-NEXT: v_mov_b32_e32 v14, v6 -; GFX9-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v21, v13, v[14:15] +; GFX9-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v22, v13, v[14:15] ; GFX9-NEXT: v_mul_lo_u32 v9, v10, v4 ; GFX9-NEXT: v_mul_lo_u32 v11, v11, v23 ; GFX9-NEXT: v_mov_b32_e32 v4, v14 @@ -218,272 +219,283 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-NEXT: v_mov_b32_e32 v8, v14 ; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v4, v8 ; GFX9-NEXT: v_addc_co_u32_e64 v9, s[4:5], 0, 0, vcc -; GFX9-NEXT: v_mul_lo_u32 v12, v12, v21 -; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v21, v10, v[8:9] +; GFX9-NEXT: v_mul_lo_u32 v12, v12, v22 +; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v22, v10, v[8:9] ; GFX9-NEXT: v_add3_u32 v4, v11, v7, v12 ; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v8, v6 ; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v9, v4, vcc ; GFX9-NEXT: v_mov_b32_e32 v7, v13 -; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v5 -; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v7, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v6, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v4, vcc -; GFX9-NEXT: v_xor_b32_e32 v5, v0, v20 -; GFX9-NEXT: v_xor_b32_e32 v0, v2, v20 -; GFX9-NEXT: v_xor_b32_e32 v4, v1, v22 -; GFX9-NEXT: v_xor_b32_e32 v1, v3, v22 +; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v5 +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v7, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v6, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v4, vcc +; GFX9-NEXT: v_xor_b32_e32 v0, v0, v20 +; GFX9-NEXT: v_xor_b32_e32 v1, v1, v21 ; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v20 -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v22, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v5, v20, vcc -; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v22, vcc +; GFX9-NEXT: v_xor_b32_e32 v2, v2, v20 +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v21, vcc +; GFX9-NEXT: v_xor_b32_e32 v3, v3, v21 +; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v20, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v21, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-O0-LABEL: v_srem_i128_vv: ; GFX9-O0: ; %bb.0: ; %_udiv-special-cases ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-O0-NEXT: ; implicit-def: $vgpr8 : SGPR spill to VGPR lane -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v7 -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v2 -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v4 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v1 ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: s_waitcnt vmcnt(1) -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v2 +; GFX9-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v7 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v7 +; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v5 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v9 +; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v3 +; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v3 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4_sgpr5 ; GFX9-O0-NEXT: s_mov_b32 s4, 63 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v8 -; GFX9-O0-NEXT: v_ashrrev_i64 v[11:12], s4, v[10:11] -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v13 +; GFX9-O0-NEXT: v_ashrrev_i64 v[3:4], s4, v[3:4] +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v13, v11 -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v13, v6 -; GFX9-O0-NEXT: v_ashrrev_i64 v[15:16], s4, v[13:14] -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v12 -; GFX9-O0-NEXT: v_xor_b32_e64 v3, v3, v10 -; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 killed $vgpr8_vgpr9 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v11 -; GFX9-O0-NEXT: v_xor_b32_e64 v13, v8, v12 -; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 -; GFX9-O0-NEXT: v_xor_b32_e64 v3, v3, v10 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_xor_b32_e64 v1, v1, v12 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v16 -; GFX9-O0-NEXT: v_xor_b32_e64 v9, v8, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v15 -; GFX9-O0-NEXT: v_xor_b32_e64 v7, v7, v6 -; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v5 -; GFX9-O0-NEXT: v_xor_b32_e64 v9, v9, v3 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: v_xor_b32_e64 v4, v4, v6 -; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v1 -; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v14 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v9, vcc, v9, v12 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v10, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v13, vcc, v11, v12, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v10, vcc +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v13 +; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 +; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 0 +; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 1 +; GFX9-O0-NEXT: s_mov_b32 s10, s6 +; GFX9-O0-NEXT: v_writelane_b32 v0, s10, 2 +; GFX9-O0-NEXT: s_mov_b32 s11, s7 +; GFX9-O0-NEXT: v_writelane_b32 v0, s11, 3 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v10, vcc, s10, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s11 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v6, vcc, v5, v4, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s10 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v5, vcc, v5, v1, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v7, s11 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v7, v2, vcc ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v2 +; GFX9-O0-NEXT: ; kill: def $vgpr10 killed $vgpr10 def $vgpr10_vgpr11 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 +; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7] +; GFX9-O0-NEXT: v_cmp_lt_i64_e64 s[4:5], v[12:13], s[4:5] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v6, s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v6, s[4:5] +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: v_mov_b32_e32 v16, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v17, v4 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v2, v7, s[4:5] +; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr5_vgpr6 killed $exec +; GFX9-O0-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[4:5] ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v14, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v8 -; GFX9-O0-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v6 -; GFX9-O0-NEXT: v_subb_co_u32_e32 v4, vcc, v4, v3, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v5, v6, vcc -; GFX9-O0-NEXT: v_subb_co_u32_e32 v3, vcc, v2, v3, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v18 +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v19 +; GFX9-O0-NEXT: v_sub_co_u32_e32 v14, vcc, s10, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s11 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v12, vcc, v11, v10, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s10 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v11, v8, vcc +; GFX9-O0-NEXT: v_mov_b32_e32 v13, s11 +; GFX9-O0-NEXT: v_subb_co_u32_e32 v13, vcc, v13, v9, vcc ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v4 +; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v15 +; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7] +; GFX9-O0-NEXT: v_cmp_lt_i64_e64 s[4:5], v[18:19], s[4:5] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v10, v10, v12, s[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v14 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, v7, v12, s[4:5] +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: v_mov_b32_e32 v18, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v19, v10 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v12 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, v9, v13, s[4:5] +; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 killed $vgpr11_vgpr12 killed $exec +; GFX9-O0-NEXT: v_cndmask_b32_e64 v8, v8, v11, s[4:5] ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 -; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v12, v3 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v9 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v5 +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v16 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v17 +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v12 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v18 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v19 +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v12 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v18 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v19 +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v5 +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v16 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v17 +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v12 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v2 -; GFX9-O0-NEXT: v_or_b32_e64 v3, v8, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 -; GFX9-O0-NEXT: v_or_b32_e64 v1, v5, v6 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0 -; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 0 -; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 1 -; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[1:2], s[6:7] -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v14 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10 -; GFX9-O0-NEXT: v_or_b32_e64 v15, v4, v2 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v13 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v9 -; GFX9-O0-NEXT: v_or_b32_e64 v9, v3, v1 -; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v15 -; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[9:10], s[6:7] +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v19 +; GFX9-O0-NEXT: v_or_b32_e64 v15, v13, v14 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v18 +; GFX9-O0-NEXT: v_or_b32_e64 v13, v13, v14 +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v15 +; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[13:14], s[6:7] +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v6 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v17 +; GFX9-O0-NEXT: v_or_b32_e64 v15, v13, v14 +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v16 +; GFX9-O0-NEXT: v_or_b32_e64 v13, v13, v14 +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v15 +; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[13:14], s[6:7] ; GFX9-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9] -; GFX9-O0-NEXT: v_ffbh_u32_e64 v6, v6 -; GFX9-O0-NEXT: s_mov_b32 s9, 32 -; GFX9-O0-NEXT: v_add_u32_e64 v6, v6, s9 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v7, v7 -; GFX9-O0-NEXT: v_min_u32_e64 v6, v6, v7 -; GFX9-O0-NEXT: s_mov_b32 s8, 0 -; GFX9-O0-NEXT: ; implicit-def: $sgpr10 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v7 -; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v5 -; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 +; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[8:9], v[11:12], s[8:9] ; GFX9-O0-NEXT: v_ffbh_u32_e64 v8, v8 -; GFX9-O0-NEXT: v_min_u32_e64 v15, v5, v8 -; GFX9-O0-NEXT: ; implicit-def: $sgpr10 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v5 -; GFX9-O0-NEXT: s_mov_b64 s[10:11], 64 -; GFX9-O0-NEXT: v_mov_b32_e32 v8, v15 -; GFX9-O0-NEXT: s_mov_b32 s12, s10 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16 -; GFX9-O0-NEXT: s_mov_b32 s14, s11 -; GFX9-O0-NEXT: v_add_co_u32_e64 v8, s[12:13], v8, s12 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, s14 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[12:13], v5, v9, s[12:13] +; GFX9-O0-NEXT: s_mov_b32 s13, 32 +; GFX9-O0-NEXT: v_add_u32_e64 v8, v8, s13 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v9, v9 +; GFX9-O0-NEXT: v_min_u32_e64 v8, v8, v9 +; GFX9-O0-NEXT: s_mov_b32 s12, 0 +; GFX9-O0-NEXT: ; implicit-def: $sgpr14 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s12 ; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 -; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[6:7] -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[12:13], v[11:12], s[12:13] -; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v10, s[12:13] -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v8 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, v6, v7, s[12:13] -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v11 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, v9 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v7, v7 +; GFX9-O0-NEXT: v_add_u32_e64 v7, v7, s13 +; GFX9-O0-NEXT: v_ffbh_u32_e64 v10, v10 +; GFX9-O0-NEXT: v_min_u32_e64 v13, v7, v10 +; GFX9-O0-NEXT: ; implicit-def: $sgpr14 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, s12 +; GFX9-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v14, v7 +; GFX9-O0-NEXT: s_mov_b64 s[14:15], 64 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v13 +; GFX9-O0-NEXT: s_mov_b32 s16, s14 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v14 +; GFX9-O0-NEXT: s_mov_b32 s18, s15 +; GFX9-O0-NEXT: v_add_co_u32_e64 v10, s[16:17], v10, s16 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, s18 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v7, s[16:17], v7, v11, s[16:17] +; GFX9-O0-NEXT: ; kill: def $vgpr10 killed $vgpr10 def $vgpr10_vgpr11 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v7, v11 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, v7, v12, s[8:9] +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v10 +; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, v8, v9, s[8:9] +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr8 ; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v10, v7 +; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] +; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[8:9], v[5:6], s[8:9] ; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v1 -; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 +; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s13 ; GFX9-O0-NEXT: v_ffbh_u32_e64 v6, v2 ; GFX9-O0-NEXT: v_min_u32_e64 v6, v5, v6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr12 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 +; GFX9-O0-NEXT: ; implicit-def: $sgpr16 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s12 ; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5 ; GFX9-O0-NEXT: v_mov_b32_e32 v8, v7 ; GFX9-O0-NEXT: v_ffbh_u32_e64 v5, v3 -; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s9 +; GFX9-O0-NEXT: v_add_u32_e64 v5, v5, s13 ; GFX9-O0-NEXT: v_ffbh_u32_e64 v11, v4 -; GFX9-O0-NEXT: v_min_u32_e64 v15, v5, v11 -; GFX9-O0-NEXT: ; implicit-def: $sgpr9 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8 -; GFX9-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v16, v5 -; GFX9-O0-NEXT: v_mov_b32_e32 v11, v15 -; GFX9-O0-NEXT: s_mov_b32 s8, s10 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16 -; GFX9-O0-NEXT: s_mov_b32 s10, s11 -; GFX9-O0-NEXT: v_add_co_u32_e64 v11, s[8:9], v11, s8 -; GFX9-O0-NEXT: v_mov_b32_e32 v12, s10 -; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[8:9], v5, v12, s[8:9] +; GFX9-O0-NEXT: v_min_u32_e64 v12, v5, v11 +; GFX9-O0-NEXT: ; implicit-def: $sgpr13 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, s12 +; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v13, v5 +; GFX9-O0-NEXT: v_mov_b32_e32 v11, v12 +; GFX9-O0-NEXT: s_mov_b32 s12, s14 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v13 +; GFX9-O0-NEXT: s_mov_b32 s14, s15 +; GFX9-O0-NEXT: v_add_co_u32_e64 v11, s[12:13], v11, s12 +; GFX9-O0-NEXT: v_mov_b32_e32 v12, s14 +; GFX9-O0-NEXT: v_addc_co_u32_e64 v5, s[12:13], v5, v12, s[12:13] ; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v12, v5 ; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12 -; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] -; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[8:9], v[13:14], s[8:9] ; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6 ; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11 @@ -496,8 +508,6 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 ; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr6_vgpr7 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 -; GFX9-O0-NEXT: s_mov_b32 s10, s6 -; GFX9-O0-NEXT: s_mov_b32 s11, s7 ; GFX9-O0-NEXT: v_sub_co_u32_e32 v5, vcc, v5, v8 ; GFX9-O0-NEXT: v_subb_co_u32_e32 v9, vcc, v6, v7, vcc ; GFX9-O0-NEXT: v_mov_b32_e32 v7, s10 @@ -580,75 +590,75 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) ; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_mov_b64 s[4:5], exec -; GFX9-O0-NEXT: v_writelane_b32 v0, s4, 2 -; GFX9-O0-NEXT: v_writelane_b32 v0, s5, 3 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: v_writelane_b32 v0, s4, 4 +; GFX9-O0-NEXT: v_writelane_b32 v0, s5, 5 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7] ; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-O0-NEXT: s_cbranch_execz .LBB0_3 ; GFX9-O0-NEXT: s_branch .LBB0_8 ; GFX9-O0-NEXT: .LBB0_1: ; %Flow -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s4, v0, 4 -; GFX9-O0-NEXT: v_readlane_b32 s5, v0, 5 +; GFX9-O0-NEXT: v_readlane_b32 s4, v0, 6 +; GFX9-O0-NEXT: v_readlane_b32 s5, v0, 7 ; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX9-O0-NEXT: ; %bb.2: ; %Flow -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_waitcnt vmcnt(6) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_branch .LBB0_5 ; GFX9-O0-NEXT: .LBB0_3: ; %Flow2 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s4, v4, 2 -; GFX9-O0-NEXT: v_readlane_b32 s5, v4, 3 +; GFX9-O0-NEXT: v_readlane_b32 s4, v4, 4 +; GFX9-O0-NEXT: v_readlane_b32 s5, v4, 5 ; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_branch .LBB0_9 ; GFX9-O0-NEXT: .LBB0_4: ; %udiv-loop-exit -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_mov_b32 s4, 1 ; GFX9-O0-NEXT: s_waitcnt vmcnt(2) ; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[0:1] @@ -681,67 +691,67 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_branch .LBB0_3 ; GFX9-O0-NEXT: .LBB0_5: ; %Flow1 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s4, v8, 6 -; GFX9-O0-NEXT: v_readlane_b32 s5, v8, 7 +; GFX9-O0-NEXT: v_readlane_b32 s4, v8, 8 +; GFX9-O0-NEXT: v_readlane_b32 s5, v8, 9 ; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_branch .LBB0_4 ; GFX9-O0-NEXT: .LBB0_6: ; %udiv-do-while ; GFX9-O0-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: v_readlane_b32 s6, v16, 8 -; GFX9-O0-NEXT: v_readlane_b32 s7, v16, 9 -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] +; GFX9-O0-NEXT: s_waitcnt vmcnt(0) +; GFX9-O0-NEXT: v_readlane_b32 s6, v16, 10 +; GFX9-O0-NEXT: v_readlane_b32 s7, v16, 11 +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_mov_b32 s4, 63 ; GFX9-O0-NEXT: s_waitcnt vmcnt(16) ; GFX9-O0-NEXT: v_lshrrev_b64 v[29:30], s4, v[2:3] @@ -881,72 +891,72 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7] ; GFX9-O0-NEXT: v_mov_b32_e32 v18, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v17, v2 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill ; GFX9-O0-NEXT: v_mov_b32_e32 v18, v1 ; GFX9-O0-NEXT: v_mov_b32_e32 v17, v0 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill ; GFX9-O0-NEXT: v_mov_b32_e32 v18, v15 ; GFX9-O0-NEXT: v_mov_b32_e32 v17, v14 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill ; GFX9-O0-NEXT: v_mov_b32_e32 v18, v13 ; GFX9-O0-NEXT: v_mov_b32_e32 v17, v12 -; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5] -; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 4 -; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 5 +; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 6 +; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 7 ; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5] -; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 8 -; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 9 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: v_writelane_b32 v16, s6, 10 +; GFX9-O0-NEXT: v_writelane_b32 v16, s7, 11 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GFX9-O0-NEXT: s_cbranch_execnz .LBB0_6 ; GFX9-O0-NEXT: s_branch .LBB0_1 ; GFX9-O0-NEXT: .LBB0_7: ; %udiv-preheader -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload @@ -1027,51 +1037,51 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v13, v17 ; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7] -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7] ; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9 ; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8 ; GFX9-O0-NEXT: v_mov_b32_e32 v13, s7 ; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6 -; GFX9-O0-NEXT: v_writelane_b32 v16, s4, 8 -; GFX9-O0-NEXT: v_writelane_b32 v16, s5, 9 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: v_writelane_b32 v16, s4, 10 +; GFX9-O0-NEXT: v_writelane_b32 v16, s5, 11 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] -; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] +; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_branch .LBB0_6 ; GFX9-O0-NEXT: .LBB0_8: ; %udiv-bb1 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload @@ -1108,14 +1118,14 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2 ; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1 -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill ; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 ; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10 -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_mov_b32 s4, 0x7f ; GFX9-O0-NEXT: v_sub_u32_e64 v3, s4, v4 ; GFX9-O0-NEXT: v_lshlrev_b64 v[5:6], v3, v[11:12] @@ -1161,12 +1171,12 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: ; implicit-def: $sgpr4 ; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v8, v3 -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill ; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 ; GFX9-O0-NEXT: v_mov_b32_e32 v3, v10 ; GFX9-O0-NEXT: v_or_b32_e64 v3, v3, v4 @@ -1181,33 +1191,33 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: v_mov_b32_e32 v4, s9 ; GFX9-O0-NEXT: v_mov_b32_e32 v1, s6 ; GFX9-O0-NEXT: v_mov_b32_e32 v2, s7 -; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_mov_b64 s[6:7], exec ; GFX9-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5] ; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7] -; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 6 -; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 7 -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: v_writelane_b32 v0, s6, 8 +; GFX9-O0-NEXT: v_writelane_b32 v0, s7, 9 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-O0-NEXT: s_cbranch_execz .LBB0_5 ; GFX9-O0-NEXT: s_branch .LBB0_7 ; GFX9-O0-NEXT: .LBB0_9: ; %udiv-end -; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1 +; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1 ; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload -; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19] +; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23] ; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload @@ -1218,10 +1228,10 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload ; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_mov_b32 s4, 32 @@ -1495,11 +1505,11 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5 ; GFX9-O0-NEXT: ; kill: killed $vgpr4 ; GFX9-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1 -; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_nop 0 -; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload -; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5] ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) ; GFX9-O0-NEXT: s_setpc_b64 s[30:31] diff --git a/llvm/test/CodeGen/AMDGPU/sdiv.ll b/llvm/test/CodeGen/AMDGPU/sdiv.ll index ea30a63b0be1..6372d74161fa 100644 --- a/llvm/test/CodeGen/AMDGPU/sdiv.ll +++ b/llvm/test/CodeGen/AMDGPU/sdiv.ll @@ -28,34 +28,33 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) ; GCN-NEXT: s_mov_b32 s0, s4 ; GCN-NEXT: s_mov_b32 s1, s5 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_ashrrev_i32_e32 v2, 31, v1 -; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 -; GCN-NEXT: v_xor_b32_e32 v1, v1, v2 -; GCN-NEXT: v_cvt_f32_u32_e32 v3, v1 -; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v1 -; GCN-NEXT: v_ashrrev_i32_e32 v5, 31, v0 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v1 +; GCN-NEXT: v_max_i32_e32 v2, v1, v2 +; GCN-NEXT: v_cvt_f32_u32_e32 v3, v2 +; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v2 +; GCN-NEXT: v_sub_i32_e32 v5, vcc, 0, v0 ; GCN-NEXT: v_rcp_iflag_f32_e32 v3, v3 -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v5 -; GCN-NEXT: v_xor_b32_e32 v0, v0, v5 +; GCN-NEXT: v_max_i32_e32 v5, v0, v5 +; GCN-NEXT: v_xor_b32_e32 v0, v0, v1 +; GCN-NEXT: v_ashrrev_i32_e32 v0, 31, v0 ; GCN-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3 ; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_xor_b32_e32 v2, v5, v2 ; GCN-NEXT: v_mul_lo_u32 v4, v4, v3 ; GCN-NEXT: v_mul_hi_u32 v4, v3, v4 ; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v4 -; GCN-NEXT: v_mul_hi_u32 v3, v0, v3 -; GCN-NEXT: v_mul_lo_u32 v4, v3, v1 -; GCN-NEXT: v_add_i32_e32 v5, vcc, 1, v3 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_sub_i32_e32 v4, vcc, v0, v1 -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 -; GCN-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc -; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc +; GCN-NEXT: v_mul_hi_u32 v3, v5, v3 +; GCN-NEXT: v_mul_lo_u32 v1, v3, v2 +; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v3 +; GCN-NEXT: v_sub_i32_e32 v1, vcc, v5, v1 +; GCN-NEXT: v_sub_i32_e32 v5, vcc, v1, v2 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2 +; GCN-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc ; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v3 -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 -; GCN-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc -; GCN-NEXT: v_xor_b32_e32 v0, v0, v2 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2 +; GCN-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc +; GCN-NEXT: v_xor_b32_e32 v1, v1, v0 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v1, v0 ; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GCN-NEXT: s_endpgm ; @@ -73,34 +72,33 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) ; TONGA-NEXT: s_mov_b32 s0, s4 ; TONGA-NEXT: s_mov_b32 s1, s5 ; TONGA-NEXT: s_waitcnt vmcnt(0) -; TONGA-NEXT: v_ashrrev_i32_e32 v2, 31, v1 -; TONGA-NEXT: v_add_u32_e32 v1, vcc, v1, v2 -; TONGA-NEXT: v_xor_b32_e32 v1, v1, v2 -; TONGA-NEXT: v_cvt_f32_u32_e32 v3, v1 -; TONGA-NEXT: v_sub_u32_e32 v4, vcc, 0, v1 -; TONGA-NEXT: v_ashrrev_i32_e32 v5, 31, v0 +; TONGA-NEXT: v_sub_u32_e32 v2, vcc, 0, v1 +; TONGA-NEXT: v_max_i32_e32 v2, v1, v2 +; TONGA-NEXT: v_cvt_f32_u32_e32 v3, v2 +; TONGA-NEXT: v_sub_u32_e32 v4, vcc, 0, v2 +; TONGA-NEXT: v_sub_u32_e32 v5, vcc, 0, v0 ; TONGA-NEXT: v_rcp_iflag_f32_e32 v3, v3 -; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v5 -; TONGA-NEXT: v_xor_b32_e32 v0, v0, v5 +; TONGA-NEXT: v_max_i32_e32 v5, v0, v5 +; TONGA-NEXT: v_xor_b32_e32 v0, v0, v1 +; TONGA-NEXT: v_ashrrev_i32_e32 v0, 31, v0 ; TONGA-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3 ; TONGA-NEXT: v_cvt_u32_f32_e32 v3, v3 -; TONGA-NEXT: v_xor_b32_e32 v2, v5, v2 ; TONGA-NEXT: v_mul_lo_u32 v4, v4, v3 ; TONGA-NEXT: v_mul_hi_u32 v4, v3, v4 ; TONGA-NEXT: v_add_u32_e32 v3, vcc, v3, v4 -; TONGA-NEXT: v_mul_hi_u32 v3, v0, v3 -; TONGA-NEXT: v_mul_lo_u32 v4, v3, v1 -; TONGA-NEXT: v_add_u32_e32 v5, vcc, 1, v3 -; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v0, v4 -; TONGA-NEXT: v_sub_u32_e32 v4, vcc, v0, v1 -; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 -; TONGA-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc -; TONGA-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc +; TONGA-NEXT: v_mul_hi_u32 v3, v5, v3 +; TONGA-NEXT: v_mul_lo_u32 v1, v3, v2 +; TONGA-NEXT: v_add_u32_e32 v4, vcc, 1, v3 +; TONGA-NEXT: v_sub_u32_e32 v1, vcc, v5, v1 +; TONGA-NEXT: v_sub_u32_e32 v5, vcc, v1, v2 +; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2 +; TONGA-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc +; TONGA-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc ; TONGA-NEXT: v_add_u32_e32 v4, vcc, 1, v3 -; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 -; TONGA-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc -; TONGA-NEXT: v_xor_b32_e32 v0, v0, v2 -; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v0, v2 +; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2 +; TONGA-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc +; TONGA-NEXT: v_xor_b32_e32 v1, v1, v0 +; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v1, v0 ; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; TONGA-NEXT: s_endpgm ; @@ -115,40 +113,37 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) ; GFX9-NEXT: s_mov_b32 s8, s6 ; GFX9-NEXT: s_mov_b32 s9, s7 ; GFX9-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0 +; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_mov_b32 s1, s5 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_readfirstlane_b32 s0, v1 -; GFX9-NEXT: s_ashr_i32 s6, s0, 31 -; GFX9-NEXT: s_add_i32 s0, s0, s6 -; GFX9-NEXT: s_xor_b32 s7, s0, s6 +; GFX9-NEXT: v_readfirstlane_b32 s6, v1 +; GFX9-NEXT: s_abs_i32 s7, s6 ; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s7 -; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: v_readfirstlane_b32 s4, v0 -; GFX9-NEXT: s_ashr_i32 s5, s4, 31 +; GFX9-NEXT: s_xor_b32 s5, s4, s6 +; GFX9-NEXT: s_sub_i32 s6, 0, s7 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 -; GFX9-NEXT: s_add_i32 s4, s4, s5 -; GFX9-NEXT: s_xor_b32 s6, s5, s6 -; GFX9-NEXT: s_xor_b32 s4, s4, s5 +; GFX9-NEXT: s_abs_i32 s4, s4 +; GFX9-NEXT: s_ashr_i32 s5, s5, 31 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v1 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: s_sub_i32 s5, 0, s7 ; GFX9-NEXT: v_readfirstlane_b32 s8, v0 -; GFX9-NEXT: s_mul_i32 s5, s5, s8 -; GFX9-NEXT: s_mul_hi_u32 s5, s8, s5 -; GFX9-NEXT: s_add_i32 s8, s8, s5 -; GFX9-NEXT: s_mul_hi_u32 s5, s4, s8 -; GFX9-NEXT: s_mul_i32 s8, s5, s7 +; GFX9-NEXT: s_mul_i32 s6, s6, s8 +; GFX9-NEXT: s_mul_hi_u32 s6, s8, s6 +; GFX9-NEXT: s_add_i32 s8, s8, s6 +; GFX9-NEXT: s_mul_hi_u32 s6, s4, s8 +; GFX9-NEXT: s_mul_i32 s8, s6, s7 ; GFX9-NEXT: s_sub_i32 s4, s4, s8 -; GFX9-NEXT: s_add_i32 s9, s5, 1 +; GFX9-NEXT: s_add_i32 s9, s6, 1 ; GFX9-NEXT: s_sub_i32 s8, s4, s7 ; GFX9-NEXT: s_cmp_ge_u32 s4, s7 -; GFX9-NEXT: s_cselect_b32 s5, s9, s5 +; GFX9-NEXT: s_cselect_b32 s6, s9, s6 ; GFX9-NEXT: s_cselect_b32 s4, s8, s4 -; GFX9-NEXT: s_add_i32 s8, s5, 1 +; GFX9-NEXT: s_add_i32 s8, s6, 1 ; GFX9-NEXT: s_cmp_ge_u32 s4, s7 -; GFX9-NEXT: s_cselect_b32 s4, s8, s5 -; GFX9-NEXT: s_xor_b32 s4, s4, s6 -; GFX9-NEXT: s_sub_i32 s4, s4, s6 +; GFX9-NEXT: s_cselect_b32 s4, s8, s6 +; GFX9-NEXT: s_xor_b32 s4, s4, s5 +; GFX9-NEXT: s_sub_i32 s4, s4, s5 ; GFX9-NEXT: v_mov_b32_e32 v0, s4 ; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX9-NEXT: s_endpgm @@ -408,62 +403,60 @@ define amdgpu_kernel void @sdiv_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GCN-NEXT: s_mov_b32 s4, s0 ; GCN-NEXT: s_mov_b32 s5, s1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_ashrrev_i32_e32 v5, 31, v2 -; GCN-NEXT: v_ashrrev_i32_e32 v7, 31, v3 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v5 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v7 -; GCN-NEXT: v_ashrrev_i32_e32 v4, 31, v0 -; GCN-NEXT: v_ashrrev_i32_e32 v6, 31, v1 -; GCN-NEXT: v_xor_b32_e32 v2, v2, v5 -; GCN-NEXT: v_xor_b32_e32 v3, v3, v7 -; GCN-NEXT: v_xor_b32_e32 v8, v4, v5 -; GCN-NEXT: v_xor_b32_e32 v9, v6, v7 -; GCN-NEXT: v_cvt_f32_u32_e32 v5, v2 -; GCN-NEXT: v_cvt_f32_u32_e32 v7, v3 -; GCN-NEXT: v_sub_i32_e32 v10, vcc, 0, v2 -; GCN-NEXT: v_rcp_iflag_f32_e32 v5, v5 -; GCN-NEXT: v_rcp_iflag_f32_e32 v7, v7 -; GCN-NEXT: v_sub_i32_e32 v11, vcc, 0, v3 +; GCN-NEXT: v_sub_i32_e32 v6, vcc, 0, v2 +; GCN-NEXT: v_sub_i32_e32 v9, vcc, 0, v3 +; GCN-NEXT: v_xor_b32_e32 v4, v0, v2 +; GCN-NEXT: v_xor_b32_e32 v7, v1, v3 +; GCN-NEXT: v_max_i32_e32 v2, v2, v6 +; GCN-NEXT: v_max_i32_e32 v3, v3, v9 +; GCN-NEXT: v_cvt_f32_u32_e32 v6, v2 +; GCN-NEXT: v_cvt_f32_u32_e32 v9, v3 +; GCN-NEXT: v_sub_i32_e32 v5, vcc, 0, v0 +; GCN-NEXT: v_rcp_iflag_f32_e32 v6, v6 +; GCN-NEXT: v_max_i32_e32 v0, v0, v5 +; GCN-NEXT: v_rcp_iflag_f32_e32 v5, v9 +; GCN-NEXT: v_sub_i32_e32 v9, vcc, 0, v2 +; GCN-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6 ; GCN-NEXT: v_mul_f32_e32 v5, 0x4f7ffffe, v5 -; GCN-NEXT: v_mul_f32_e32 v7, 0x4f7ffffe, v7 +; GCN-NEXT: v_cvt_u32_f32_e32 v6, v6 ; GCN-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GCN-NEXT: v_cvt_u32_f32_e32 v7, v7 -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 +; GCN-NEXT: v_sub_i32_e32 v10, vcc, 0, v3 +; GCN-NEXT: v_mul_lo_u32 v9, v9, v6 ; GCN-NEXT: v_mul_lo_u32 v10, v10, v5 -; GCN-NEXT: v_mul_lo_u32 v11, v11, v7 -; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v6 -; GCN-NEXT: v_xor_b32_e32 v0, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v4, v5, v10 -; GCN-NEXT: v_xor_b32_e32 v1, v1, v6 -; GCN-NEXT: v_mul_hi_u32 v6, v7, v11 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v7, v6 -; GCN-NEXT: v_mul_hi_u32 v4, v0, v4 +; GCN-NEXT: v_sub_i32_e32 v8, vcc, 0, v1 +; GCN-NEXT: v_mul_hi_u32 v9, v6, v9 +; GCN-NEXT: v_max_i32_e32 v1, v1, v8 +; GCN-NEXT: v_mul_hi_u32 v8, v5, v10 +; GCN-NEXT: v_ashrrev_i32_e32 v4, 31, v4 +; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v9 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v8 +; GCN-NEXT: v_mul_hi_u32 v6, v0, v6 ; GCN-NEXT: v_mul_hi_u32 v5, v1, v5 -; GCN-NEXT: v_mul_lo_u32 v6, v4, v2 +; GCN-NEXT: v_ashrrev_i32_e32 v7, 31, v7 +; GCN-NEXT: v_mul_lo_u32 v8, v6, v2 ; GCN-NEXT: v_mul_lo_u32 v10, v5, v3 -; GCN-NEXT: v_add_i32_e32 v7, vcc, 1, v4 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v6 +; GCN-NEXT: v_add_i32_e32 v9, vcc, 1, v6 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v8 ; GCN-NEXT: v_sub_i32_e32 v1, vcc, v1, v10 ; GCN-NEXT: v_add_i32_e32 v11, vcc, 1, v5 ; GCN-NEXT: v_cmp_ge_u32_e64 s[0:1], v0, v2 ; GCN-NEXT: v_cmp_ge_u32_e64 s[2:3], v1, v3 -; GCN-NEXT: v_sub_i32_e32 v6, vcc, v0, v2 -; GCN-NEXT: v_cndmask_b32_e64 v4, v4, v7, s[0:1] -; GCN-NEXT: v_sub_i32_e32 v7, vcc, v1, v3 +; GCN-NEXT: v_sub_i32_e32 v8, vcc, v0, v2 +; GCN-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[0:1] +; GCN-NEXT: v_sub_i32_e32 v9, vcc, v1, v3 ; GCN-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[2:3] -; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[0:1] -; GCN-NEXT: v_add_i32_e32 v6, vcc, 1, v4 -; GCN-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[2:3] -; GCN-NEXT: v_add_i32_e32 v7, vcc, 1, v5 +; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[0:1] +; GCN-NEXT: v_add_i32_e32 v8, vcc, 1, v6 +; GCN-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[2:3] +; GCN-NEXT: v_add_i32_e32 v9, vcc, 1, v5 ; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 -; GCN-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc +; GCN-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc ; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 -; GCN-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc -; GCN-NEXT: v_xor_b32_e32 v0, v0, v8 -; GCN-NEXT: v_xor_b32_e32 v1, v1, v9 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v8 -; GCN-NEXT: v_sub_i32_e32 v1, vcc, v1, v9 +; GCN-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc +; GCN-NEXT: v_xor_b32_e32 v0, v0, v4 +; GCN-NEXT: v_xor_b32_e32 v1, v1, v7 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v4 +; GCN-NEXT: v_sub_i32_e32 v1, vcc, v1, v7 ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; @@ -481,62 +474,60 @@ define amdgpu_kernel void @sdiv_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %i ; TONGA-NEXT: s_mov_b32 s4, s0 ; TONGA-NEXT: s_mov_b32 s5, s1 ; TONGA-NEXT: s_waitcnt vmcnt(0) -; TONGA-NEXT: v_ashrrev_i32_e32 v5, 31, v2 -; TONGA-NEXT: v_ashrrev_i32_e32 v7, 31, v3 -; TONGA-NEXT: v_add_u32_e32 v2, vcc, v2, v5 -; TONGA-NEXT: v_add_u32_e32 v3, vcc, v3, v7 -; TONGA-NEXT: v_ashrrev_i32_e32 v4, 31, v0 -; TONGA-NEXT: v_ashrrev_i32_e32 v6, 31, v1 -; TONGA-NEXT: v_xor_b32_e32 v2, v2, v5 -; TONGA-NEXT: v_xor_b32_e32 v3, v3, v7 -; TONGA-NEXT: v_xor_b32_e32 v8, v4, v5 -; TONGA-NEXT: v_xor_b32_e32 v9, v6, v7 -; TONGA-NEXT: v_cvt_f32_u32_e32 v5, v2 -; TONGA-NEXT: v_cvt_f32_u32_e32 v7, v3 -; TONGA-NEXT: v_sub_u32_e32 v10, vcc, 0, v2 -; TONGA-NEXT: v_rcp_iflag_f32_e32 v5, v5 -; TONGA-NEXT: v_rcp_iflag_f32_e32 v7, v7 -; TONGA-NEXT: v_sub_u32_e32 v11, vcc, 0, v3 +; TONGA-NEXT: v_sub_u32_e32 v6, vcc, 0, v2 +; TONGA-NEXT: v_sub_u32_e32 v9, vcc, 0, v3 +; TONGA-NEXT: v_xor_b32_e32 v4, v0, v2 +; TONGA-NEXT: v_xor_b32_e32 v7, v1, v3 +; TONGA-NEXT: v_max_i32_e32 v2, v2, v6 +; TONGA-NEXT: v_max_i32_e32 v3, v3, v9 +; TONGA-NEXT: v_cvt_f32_u32_e32 v6, v2 +; TONGA-NEXT: v_cvt_f32_u32_e32 v9, v3 +; TONGA-NEXT: v_sub_u32_e32 v5, vcc, 0, v0 +; TONGA-NEXT: v_rcp_iflag_f32_e32 v6, v6 +; TONGA-NEXT: v_max_i32_e32 v0, v0, v5 +; TONGA-NEXT: v_rcp_iflag_f32_e32 v5, v9 +; TONGA-NEXT: v_sub_u32_e32 v9, vcc, 0, v2 +; TONGA-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6 ; TONGA-NEXT: v_mul_f32_e32 v5, 0x4f7ffffe, v5 -; TONGA-NEXT: v_mul_f32_e32 v7, 0x4f7ffffe, v7 +; TONGA-NEXT: v_cvt_u32_f32_e32 v6, v6 ; TONGA-NEXT: v_cvt_u32_f32_e32 v5, v5 -; TONGA-NEXT: v_cvt_u32_f32_e32 v7, v7 -; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v4 +; TONGA-NEXT: v_sub_u32_e32 v10, vcc, 0, v3 +; TONGA-NEXT: v_mul_lo_u32 v9, v9, v6 ; TONGA-NEXT: v_mul_lo_u32 v10, v10, v5 -; TONGA-NEXT: v_mul_lo_u32 v11, v11, v7 -; TONGA-NEXT: v_add_u32_e32 v1, vcc, v1, v6 -; TONGA-NEXT: v_xor_b32_e32 v0, v0, v4 -; TONGA-NEXT: v_mul_hi_u32 v4, v5, v10 -; TONGA-NEXT: v_xor_b32_e32 v1, v1, v6 -; TONGA-NEXT: v_mul_hi_u32 v6, v7, v11 -; TONGA-NEXT: v_add_u32_e32 v4, vcc, v5, v4 -; TONGA-NEXT: v_add_u32_e32 v5, vcc, v7, v6 -; TONGA-NEXT: v_mul_hi_u32 v4, v0, v4 +; TONGA-NEXT: v_sub_u32_e32 v8, vcc, 0, v1 +; TONGA-NEXT: v_mul_hi_u32 v9, v6, v9 +; TONGA-NEXT: v_max_i32_e32 v1, v1, v8 +; TONGA-NEXT: v_mul_hi_u32 v8, v5, v10 +; TONGA-NEXT: v_ashrrev_i32_e32 v4, 31, v4 +; TONGA-NEXT: v_add_u32_e32 v6, vcc, v6, v9 +; TONGA-NEXT: v_add_u32_e32 v5, vcc, v5, v8 +; TONGA-NEXT: v_mul_hi_u32 v6, v0, v6 ; TONGA-NEXT: v_mul_hi_u32 v5, v1, v5 -; TONGA-NEXT: v_mul_lo_u32 v6, v4, v2 +; TONGA-NEXT: v_ashrrev_i32_e32 v7, 31, v7 +; TONGA-NEXT: v_mul_lo_u32 v8, v6, v2 ; TONGA-NEXT: v_mul_lo_u32 v10, v5, v3 -; TONGA-NEXT: v_add_u32_e32 v7, vcc, 1, v4 -; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v0, v6 +; TONGA-NEXT: v_add_u32_e32 v9, vcc, 1, v6 +; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v0, v8 ; TONGA-NEXT: v_sub_u32_e32 v1, vcc, v1, v10 ; TONGA-NEXT: v_add_u32_e32 v11, vcc, 1, v5 ; TONGA-NEXT: v_cmp_ge_u32_e64 s[0:1], v0, v2 ; TONGA-NEXT: v_cmp_ge_u32_e64 s[2:3], v1, v3 -; TONGA-NEXT: v_sub_u32_e32 v6, vcc, v0, v2 -; TONGA-NEXT: v_cndmask_b32_e64 v4, v4, v7, s[0:1] -; TONGA-NEXT: v_sub_u32_e32 v7, vcc, v1, v3 +; TONGA-NEXT: v_sub_u32_e32 v8, vcc, v0, v2 +; TONGA-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[0:1] +; TONGA-NEXT: v_sub_u32_e32 v9, vcc, v1, v3 ; TONGA-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[2:3] -; TONGA-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[0:1] -; TONGA-NEXT: v_add_u32_e32 v6, vcc, 1, v4 -; TONGA-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[2:3] -; TONGA-NEXT: v_add_u32_e32 v7, vcc, 1, v5 +; TONGA-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[0:1] +; TONGA-NEXT: v_add_u32_e32 v8, vcc, 1, v6 +; TONGA-NEXT: v_cndmask_b32_e64 v1, v1, v9, s[2:3] +; TONGA-NEXT: v_add_u32_e32 v9, vcc, 1, v5 ; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 -; TONGA-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc +; TONGA-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc ; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 -; TONGA-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc -; TONGA-NEXT: v_xor_b32_e32 v0, v0, v8 -; TONGA-NEXT: v_xor_b32_e32 v1, v1, v9 -; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v0, v8 -; TONGA-NEXT: v_sub_u32_e32 v1, vcc, v1, v9 +; TONGA-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc +; TONGA-NEXT: v_xor_b32_e32 v0, v0, v4 +; TONGA-NEXT: v_xor_b32_e32 v1, v1, v7 +; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v0, v4 +; TONGA-NEXT: v_sub_u32_e32 v1, vcc, v1, v7 ; TONGA-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; TONGA-NEXT: s_endpgm ; @@ -553,69 +544,63 @@ define amdgpu_kernel void @sdiv_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_readfirstlane_b32 s0, v2 -; GFX9-NEXT: s_ashr_i32 s1, s0, 31 -; GFX9-NEXT: s_add_i32 s0, s0, s1 -; GFX9-NEXT: s_xor_b32 s6, s0, s1 -; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s6 +; GFX9-NEXT: s_abs_i32 s1, s0 +; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s1 ; GFX9-NEXT: v_readfirstlane_b32 s7, v0 -; GFX9-NEXT: s_ashr_i32 s8, s7, 31 -; GFX9-NEXT: s_add_i32 s7, s7, s8 +; GFX9-NEXT: s_xor_b32 s0, s7, s0 +; GFX9-NEXT: s_ashr_i32 s8, s0, 31 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v2, v2 -; GFX9-NEXT: s_xor_b32 s9, s8, s1 -; GFX9-NEXT: s_xor_b32 s1, s7, s8 -; GFX9-NEXT: s_sub_i32 s7, 0, s6 +; GFX9-NEXT: s_sub_i32 s0, 0, s1 +; GFX9-NEXT: s_abs_i32 s7, s7 +; GFX9-NEXT: v_readfirstlane_b32 s6, v3 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v2 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX9-NEXT: v_readfirstlane_b32 s9, v0 +; GFX9-NEXT: s_mul_i32 s0, s0, s9 +; GFX9-NEXT: s_mul_hi_u32 s0, s9, s0 +; GFX9-NEXT: s_add_i32 s9, s9, s0 +; GFX9-NEXT: s_mul_hi_u32 s0, s7, s9 +; GFX9-NEXT: s_mul_i32 s9, s0, s1 +; GFX9-NEXT: s_sub_i32 s7, s7, s9 +; GFX9-NEXT: s_add_i32 s10, s0, 1 +; GFX9-NEXT: s_sub_i32 s9, s7, s1 +; GFX9-NEXT: s_cmp_ge_u32 s7, s1 +; GFX9-NEXT: s_cselect_b32 s0, s10, s0 +; GFX9-NEXT: s_cselect_b32 s7, s9, s7 +; GFX9-NEXT: s_add_i32 s9, s0, 1 +; GFX9-NEXT: s_cmp_ge_u32 s7, s1 +; GFX9-NEXT: s_cselect_b32 s7, s9, s0 +; GFX9-NEXT: s_abs_i32 s9, s6 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s9 ; GFX9-NEXT: s_mov_b32 s0, s4 -; GFX9-NEXT: v_readfirstlane_b32 s4, v3 +; GFX9-NEXT: v_readfirstlane_b32 s4, v1 +; GFX9-NEXT: s_mov_b32 s1, s5 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 +; GFX9-NEXT: s_xor_b32 s5, s4, s6 +; GFX9-NEXT: s_xor_b32 s6, s7, s8 +; GFX9-NEXT: s_sub_i32 s7, 0, s9 +; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX9-NEXT: s_sub_i32 s6, s6, s8 +; GFX9-NEXT: s_abs_i32 s4, s4 +; GFX9-NEXT: s_ashr_i32 s5, s5, 31 ; GFX9-NEXT: v_readfirstlane_b32 s8, v0 ; GFX9-NEXT: s_mul_i32 s7, s7, s8 ; GFX9-NEXT: s_mul_hi_u32 s7, s8, s7 ; GFX9-NEXT: s_add_i32 s8, s8, s7 -; GFX9-NEXT: s_mul_hi_u32 s7, s1, s8 -; GFX9-NEXT: s_mul_i32 s8, s7, s6 -; GFX9-NEXT: s_sub_i32 s1, s1, s8 +; GFX9-NEXT: s_mul_hi_u32 s7, s4, s8 +; GFX9-NEXT: s_mul_i32 s8, s7, s9 +; GFX9-NEXT: s_sub_i32 s4, s4, s8 ; GFX9-NEXT: s_add_i32 s10, s7, 1 -; GFX9-NEXT: s_sub_i32 s8, s1, s6 -; GFX9-NEXT: s_cmp_ge_u32 s1, s6 +; GFX9-NEXT: s_sub_i32 s8, s4, s9 +; GFX9-NEXT: s_cmp_ge_u32 s4, s9 ; GFX9-NEXT: s_cselect_b32 s7, s10, s7 -; GFX9-NEXT: s_cselect_b32 s1, s8, s1 +; GFX9-NEXT: s_cselect_b32 s4, s8, s4 ; GFX9-NEXT: s_add_i32 s8, s7, 1 -; GFX9-NEXT: s_cmp_ge_u32 s1, s6 -; GFX9-NEXT: s_cselect_b32 s6, s8, s7 -; GFX9-NEXT: s_ashr_i32 s7, s4, 31 -; GFX9-NEXT: s_add_i32 s4, s4, s7 -; GFX9-NEXT: s_xor_b32 s4, s4, s7 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4 -; GFX9-NEXT: s_mov_b32 s1, s5 -; GFX9-NEXT: v_readfirstlane_b32 s5, v1 -; GFX9-NEXT: s_ashr_i32 s8, s5, 31 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s6, s6, s9 -; GFX9-NEXT: s_add_i32 s5, s5, s8 -; GFX9-NEXT: s_xor_b32 s7, s8, s7 -; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 -; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: s_sub_i32 s6, s6, s9 -; GFX9-NEXT: s_xor_b32 s5, s5, s8 -; GFX9-NEXT: s_sub_i32 s8, 0, s4 -; GFX9-NEXT: v_readfirstlane_b32 s9, v0 -; GFX9-NEXT: s_mul_i32 s8, s8, s9 -; GFX9-NEXT: s_mul_hi_u32 s8, s9, s8 -; GFX9-NEXT: s_add_i32 s9, s9, s8 -; GFX9-NEXT: s_mul_hi_u32 s8, s5, s9 -; GFX9-NEXT: s_mul_i32 s9, s8, s4 -; GFX9-NEXT: s_sub_i32 s5, s5, s9 -; GFX9-NEXT: s_add_i32 s10, s8, 1 -; GFX9-NEXT: s_sub_i32 s9, s5, s4 -; GFX9-NEXT: s_cmp_ge_u32 s5, s4 -; GFX9-NEXT: s_cselect_b32 s8, s10, s8 -; GFX9-NEXT: s_cselect_b32 s5, s9, s5 -; GFX9-NEXT: s_add_i32 s9, s8, 1 -; GFX9-NEXT: s_cmp_ge_u32 s5, s4 -; GFX9-NEXT: s_cselect_b32 s4, s9, s8 -; GFX9-NEXT: s_xor_b32 s4, s4, s7 -; GFX9-NEXT: s_sub_i32 s4, s4, s7 +; GFX9-NEXT: s_cmp_ge_u32 s4, s9 +; GFX9-NEXT: s_cselect_b32 s4, s8, s7 +; GFX9-NEXT: s_xor_b32 s4, s4, s5 +; GFX9-NEXT: s_sub_i32 s4, s4, s5 ; GFX9-NEXT: v_mov_b32_e32 v0, s6 ; GFX9-NEXT: v_mov_b32_e32 v1, s4 ; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 @@ -819,119 +804,115 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GCN-NEXT: s_mov_b32 s8, s0 ; GCN-NEXT: s_mov_b32 s9, s1 ; GCN-NEXT: s_waitcnt vmcnt(1) -; GCN-NEXT: v_ashrrev_i32_e32 v8, 31, v0 +; GCN-NEXT: v_sub_i32_e32 v12, vcc, 0, v1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_ashrrev_i32_e32 v9, 31, v4 -; GCN-NEXT: v_ashrrev_i32_e32 v11, 31, v5 -; GCN-NEXT: v_ashrrev_i32_e32 v10, 31, v1 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v9 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v11 -; GCN-NEXT: v_ashrrev_i32_e32 v13, 31, v6 -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v8 -; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v10 -; GCN-NEXT: v_xor_b32_e32 v4, v4, v9 -; GCN-NEXT: v_xor_b32_e32 v5, v5, v11 -; GCN-NEXT: v_ashrrev_i32_e32 v12, 31, v2 -; GCN-NEXT: v_xor_b32_e32 v15, v8, v9 -; GCN-NEXT: v_xor_b32_e32 v16, v10, v11 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v13 -; GCN-NEXT: v_xor_b32_e32 v0, v0, v8 -; GCN-NEXT: v_xor_b32_e32 v1, v1, v10 -; GCN-NEXT: v_cvt_f32_u32_e32 v8, v4 -; GCN-NEXT: v_cvt_f32_u32_e32 v10, v5 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v12 -; GCN-NEXT: v_xor_b32_e32 v6, v6, v13 -; GCN-NEXT: v_xor_b32_e32 v17, v12, v13 -; GCN-NEXT: v_xor_b32_e32 v2, v2, v12 -; GCN-NEXT: v_cvt_f32_u32_e32 v12, v6 -; GCN-NEXT: v_rcp_iflag_f32_e32 v8, v8 -; GCN-NEXT: v_rcp_iflag_f32_e32 v10, v10 -; GCN-NEXT: v_sub_i32_e32 v9, vcc, 0, v4 +; GCN-NEXT: v_sub_i32_e32 v10, vcc, 0, v4 +; GCN-NEXT: v_sub_i32_e32 v13, vcc, 0, v5 +; GCN-NEXT: v_sub_i32_e32 v16, vcc, 0, v6 +; GCN-NEXT: v_xor_b32_e32 v8, v0, v4 +; GCN-NEXT: v_xor_b32_e32 v11, v1, v5 +; GCN-NEXT: v_xor_b32_e32 v14, v2, v6 +; GCN-NEXT: v_max_i32_e32 v4, v4, v10 +; GCN-NEXT: v_max_i32_e32 v5, v5, v13 +; GCN-NEXT: v_max_i32_e32 v6, v6, v16 +; GCN-NEXT: v_max_i32_e32 v1, v1, v12 +; GCN-NEXT: v_ashrrev_i32_e32 v10, 31, v14 +; GCN-NEXT: v_cvt_f32_u32_e32 v12, v4 +; GCN-NEXT: v_cvt_f32_u32_e32 v14, v5 +; GCN-NEXT: v_cvt_f32_u32_e32 v16, v6 +; GCN-NEXT: v_sub_i32_e32 v9, vcc, 0, v0 ; GCN-NEXT: v_rcp_iflag_f32_e32 v12, v12 -; GCN-NEXT: v_mul_f32_e32 v8, 0x4f7ffffe, v8 -; GCN-NEXT: v_mul_f32_e32 v10, 0x4f7ffffe, v10 -; GCN-NEXT: v_cvt_u32_f32_e32 v8, v8 -; GCN-NEXT: v_cvt_u32_f32_e32 v10, v10 +; GCN-NEXT: v_rcp_iflag_f32_e32 v14, v14 +; GCN-NEXT: v_rcp_iflag_f32_e32 v16, v16 +; GCN-NEXT: v_sub_i32_e32 v15, vcc, 0, v2 ; GCN-NEXT: v_mul_f32_e32 v12, 0x4f7ffffe, v12 +; GCN-NEXT: v_mul_f32_e32 v14, 0x4f7ffffe, v14 +; GCN-NEXT: v_mul_f32_e32 v16, 0x4f7ffffe, v16 ; GCN-NEXT: v_cvt_u32_f32_e32 v12, v12 -; GCN-NEXT: v_sub_i32_e32 v11, vcc, 0, v5 -; GCN-NEXT: v_mul_lo_u32 v9, v9, v8 -; GCN-NEXT: v_mul_lo_u32 v11, v11, v10 -; GCN-NEXT: v_sub_i32_e32 v13, vcc, 0, v6 +; GCN-NEXT: v_cvt_u32_f32_e32 v14, v14 +; GCN-NEXT: v_cvt_u32_f32_e32 v16, v16 +; GCN-NEXT: v_sub_i32_e32 v17, vcc, 0, v7 +; GCN-NEXT: v_max_i32_e32 v0, v0, v9 +; GCN-NEXT: v_ashrrev_i32_e32 v9, 31, v11 +; GCN-NEXT: v_max_i32_e32 v2, v2, v15 +; GCN-NEXT: v_max_i32_e32 v11, v7, v17 +; GCN-NEXT: v_sub_i32_e32 v13, vcc, 0, v4 +; GCN-NEXT: v_sub_i32_e32 v15, vcc, 0, v5 +; GCN-NEXT: v_sub_i32_e32 v17, vcc, 0, v6 ; GCN-NEXT: v_mul_lo_u32 v13, v13, v12 -; GCN-NEXT: v_mul_hi_u32 v9, v8, v9 -; GCN-NEXT: v_mul_hi_u32 v11, v10, v11 -; GCN-NEXT: v_ashrrev_i32_e32 v14, 31, v7 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v7, v14 +; GCN-NEXT: v_mul_lo_u32 v15, v15, v14 +; GCN-NEXT: v_mul_lo_u32 v17, v17, v16 +; GCN-NEXT: v_cvt_f32_u32_e32 v18, v11 ; GCN-NEXT: v_mul_hi_u32 v13, v12, v13 -; GCN-NEXT: v_xor_b32_e32 v7, v7, v14 -; GCN-NEXT: v_cvt_f32_u32_e32 v18, v7 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v9 -; GCN-NEXT: v_add_i32_e32 v9, vcc, v10, v11 -; GCN-NEXT: v_mul_hi_u32 v8, v0, v8 -; GCN-NEXT: v_mul_hi_u32 v9, v1, v9 -; GCN-NEXT: v_add_i32_e32 v10, vcc, v12, v13 -; GCN-NEXT: v_mul_hi_u32 v10, v2, v10 +; GCN-NEXT: v_mul_hi_u32 v15, v14, v15 +; GCN-NEXT: v_mul_hi_u32 v17, v16, v17 ; GCN-NEXT: v_rcp_iflag_f32_e32 v18, v18 -; GCN-NEXT: v_mul_lo_u32 v11, v8, v4 -; GCN-NEXT: v_mul_lo_u32 v13, v9, v5 -; GCN-NEXT: v_mul_lo_u32 v21, v10, v6 +; GCN-NEXT: v_add_i32_e32 v12, vcc, v12, v13 +; GCN-NEXT: v_add_i32_e32 v13, vcc, v14, v15 +; GCN-NEXT: v_add_i32_e32 v14, vcc, v16, v17 +; GCN-NEXT: v_mul_hi_u32 v12, v0, v12 +; GCN-NEXT: v_mul_hi_u32 v13, v1, v13 +; GCN-NEXT: v_mul_hi_u32 v14, v2, v14 ; GCN-NEXT: v_mul_f32_e32 v18, 0x4f7ffffe, v18 +; GCN-NEXT: v_mul_lo_u32 v15, v12, v4 +; GCN-NEXT: v_mul_lo_u32 v17, v13, v5 +; GCN-NEXT: v_mul_lo_u32 v21, v14, v6 ; GCN-NEXT: v_cvt_u32_f32_e32 v18, v18 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v11 -; GCN-NEXT: v_sub_i32_e32 v1, vcc, v1, v13 -; GCN-NEXT: v_add_i32_e32 v12, vcc, 1, v8 -; GCN-NEXT: v_add_i32_e32 v20, vcc, 1, v9 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v15 +; GCN-NEXT: v_sub_i32_e32 v1, vcc, v1, v17 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, v2, v21 +; GCN-NEXT: v_add_i32_e32 v16, vcc, 1, v12 +; GCN-NEXT: v_add_i32_e32 v20, vcc, 1, v13 +; GCN-NEXT: v_add_i32_e32 v15, vcc, 1, v14 ; GCN-NEXT: v_cmp_ge_u32_e64 s[0:1], v0, v4 ; GCN-NEXT: v_cmp_ge_u32_e64 s[2:3], v1, v5 -; GCN-NEXT: v_sub_i32_e32 v2, vcc, v2, v21 -; GCN-NEXT: v_sub_i32_e32 v11, vcc, v0, v4 -; GCN-NEXT: v_cndmask_b32_e64 v8, v8, v12, s[0:1] -; GCN-NEXT: v_sub_i32_e32 v12, vcc, v1, v5 -; GCN-NEXT: v_cndmask_b32_e64 v9, v9, v20, s[2:3] -; GCN-NEXT: v_sub_i32_e32 v19, vcc, 0, v7 -; GCN-NEXT: v_add_i32_e32 v22, vcc, 1, v10 -; GCN-NEXT: v_sub_i32_e32 v13, vcc, v2, v6 -; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v11, s[0:1] -; GCN-NEXT: v_add_i32_e32 v11, vcc, 1, v8 -; GCN-NEXT: v_cndmask_b32_e64 v1, v1, v12, s[2:3] -; GCN-NEXT: v_add_i32_e32 v12, vcc, 1, v9 +; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6 +; GCN-NEXT: v_sub_i32_e32 v19, vcc, 0, v11 +; GCN-NEXT: v_sub_i32_e32 v17, vcc, v0, v4 +; GCN-NEXT: v_cndmask_b32_e64 v12, v12, v16, s[0:1] +; GCN-NEXT: v_sub_i32_e32 v16, vcc, v1, v5 +; GCN-NEXT: v_cndmask_b32_e64 v13, v13, v20, s[2:3] +; GCN-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[4:5] +; GCN-NEXT: v_mul_lo_u32 v19, v19, v18 +; GCN-NEXT: v_sub_i32_e32 v20, vcc, v2, v6 +; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v17, s[0:1] +; GCN-NEXT: v_add_i32_e32 v15, vcc, 1, v12 +; GCN-NEXT: v_cndmask_b32_e64 v1, v1, v16, s[2:3] +; GCN-NEXT: v_add_i32_e32 v16, vcc, 1, v13 +; GCN-NEXT: v_add_i32_e32 v17, vcc, 1, v14 ; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v19, v18 -; GCN-NEXT: v_cndmask_b32_e32 v0, v8, v11, vcc +; GCN-NEXT: v_cndmask_b32_e32 v0, v12, v15, vcc ; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5 -; GCN-NEXT: v_ashrrev_i32_e32 v8, 31, v3 -; GCN-NEXT: v_mul_hi_u32 v4, v18, v4 -; GCN-NEXT: v_cndmask_b32_e32 v1, v9, v12, vcc -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v8 -; GCN-NEXT: v_xor_b32_e32 v3, v3, v8 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v18, v4 -; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6 -; GCN-NEXT: v_mul_hi_u32 v4, v3, v4 -; GCN-NEXT: v_cndmask_b32_e64 v10, v10, v22, s[4:5] -; GCN-NEXT: v_xor_b32_e32 v0, v0, v15 -; GCN-NEXT: v_xor_b32_e32 v1, v1, v16 -; GCN-NEXT: v_cndmask_b32_e64 v2, v2, v13, s[4:5] -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v15 -; GCN-NEXT: v_sub_i32_e32 v1, vcc, v1, v16 -; GCN-NEXT: v_add_i32_e32 v5, vcc, 1, v10 +; GCN-NEXT: v_ashrrev_i32_e32 v8, 31, v8 +; GCN-NEXT: v_cndmask_b32_e32 v1, v13, v16, vcc +; GCN-NEXT: v_xor_b32_e32 v0, v0, v8 +; GCN-NEXT: v_xor_b32_e32 v1, v1, v9 +; GCN-NEXT: v_mul_hi_u32 v4, v18, v19 +; GCN-NEXT: v_cndmask_b32_e64 v2, v2, v20, s[4:5] +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v8 +; GCN-NEXT: v_sub_i32_e32 v1, vcc, v1, v9 ; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6 -; GCN-NEXT: v_cndmask_b32_e32 v2, v10, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v5, v4, v7 -; GCN-NEXT: v_xor_b32_e32 v2, v2, v17 -; GCN-NEXT: v_sub_i32_e32 v2, vcc, v2, v17 -; GCN-NEXT: v_sub_i32_e32 v3, vcc, v3, v5 -; GCN-NEXT: v_xor_b32_e32 v6, v8, v14 -; GCN-NEXT: v_add_i32_e32 v5, vcc, 1, v4 -; GCN-NEXT: v_sub_i32_e32 v8, vcc, v3, v7 -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v7 -; GCN-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc -; GCN-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc -; GCN-NEXT: v_add_i32_e32 v5, vcc, 1, v4 -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v7 -; GCN-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc -; GCN-NEXT: v_xor_b32_e32 v3, v3, v6 -; GCN-NEXT: v_sub_i32_e32 v3, vcc, v3, v6 +; GCN-NEXT: v_cndmask_b32_e32 v2, v14, v17, vcc +; GCN-NEXT: v_sub_i32_e32 v5, vcc, 0, v3 +; GCN-NEXT: v_max_i32_e32 v5, v3, v5 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v18, v4 +; GCN-NEXT: v_mul_hi_u32 v4, v5, v4 +; GCN-NEXT: v_xor_b32_e32 v2, v2, v10 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, v2, v10 +; GCN-NEXT: v_mul_lo_u32 v6, v4, v11 +; GCN-NEXT: v_xor_b32_e32 v3, v3, v7 +; GCN-NEXT: v_ashrrev_i32_e32 v3, 31, v3 +; GCN-NEXT: v_sub_i32_e32 v5, vcc, v5, v6 +; GCN-NEXT: v_add_i32_e32 v6, vcc, 1, v4 +; GCN-NEXT: v_sub_i32_e32 v7, vcc, v5, v11 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v5, v11 +; GCN-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; GCN-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc +; GCN-NEXT: v_add_i32_e32 v6, vcc, 1, v4 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v5, v11 +; GCN-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; GCN-NEXT: v_xor_b32_e32 v4, v4, v3 +; GCN-NEXT: v_sub_i32_e32 v3, vcc, v4, v3 ; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 ; GCN-NEXT: s_endpgm ; @@ -950,119 +931,115 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %i ; TONGA-NEXT: s_mov_b32 s8, s0 ; TONGA-NEXT: s_mov_b32 s9, s1 ; TONGA-NEXT: s_waitcnt vmcnt(1) -; TONGA-NEXT: v_ashrrev_i32_e32 v8, 31, v0 +; TONGA-NEXT: v_sub_u32_e32 v12, vcc, 0, v1 ; TONGA-NEXT: s_waitcnt vmcnt(0) -; TONGA-NEXT: v_ashrrev_i32_e32 v9, 31, v4 -; TONGA-NEXT: v_ashrrev_i32_e32 v11, 31, v5 -; TONGA-NEXT: v_ashrrev_i32_e32 v10, 31, v1 -; TONGA-NEXT: v_add_u32_e32 v4, vcc, v4, v9 -; TONGA-NEXT: v_add_u32_e32 v5, vcc, v5, v11 -; TONGA-NEXT: v_ashrrev_i32_e32 v13, 31, v6 -; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v8 -; TONGA-NEXT: v_add_u32_e32 v1, vcc, v1, v10 -; TONGA-NEXT: v_xor_b32_e32 v4, v4, v9 -; TONGA-NEXT: v_xor_b32_e32 v5, v5, v11 -; TONGA-NEXT: v_ashrrev_i32_e32 v12, 31, v2 -; TONGA-NEXT: v_xor_b32_e32 v15, v8, v9 -; TONGA-NEXT: v_xor_b32_e32 v16, v10, v11 -; TONGA-NEXT: v_add_u32_e32 v6, vcc, v6, v13 -; TONGA-NEXT: v_xor_b32_e32 v0, v0, v8 -; TONGA-NEXT: v_xor_b32_e32 v1, v1, v10 -; TONGA-NEXT: v_cvt_f32_u32_e32 v8, v4 -; TONGA-NEXT: v_cvt_f32_u32_e32 v10, v5 -; TONGA-NEXT: v_add_u32_e32 v2, vcc, v2, v12 -; TONGA-NEXT: v_xor_b32_e32 v6, v6, v13 -; TONGA-NEXT: v_xor_b32_e32 v17, v12, v13 -; TONGA-NEXT: v_xor_b32_e32 v2, v2, v12 -; TONGA-NEXT: v_cvt_f32_u32_e32 v12, v6 -; TONGA-NEXT: v_rcp_iflag_f32_e32 v8, v8 -; TONGA-NEXT: v_rcp_iflag_f32_e32 v10, v10 -; TONGA-NEXT: v_sub_u32_e32 v9, vcc, 0, v4 +; TONGA-NEXT: v_sub_u32_e32 v10, vcc, 0, v4 +; TONGA-NEXT: v_sub_u32_e32 v13, vcc, 0, v5 +; TONGA-NEXT: v_sub_u32_e32 v16, vcc, 0, v6 +; TONGA-NEXT: v_xor_b32_e32 v8, v0, v4 +; TONGA-NEXT: v_xor_b32_e32 v11, v1, v5 +; TONGA-NEXT: v_xor_b32_e32 v14, v2, v6 +; TONGA-NEXT: v_max_i32_e32 v4, v4, v10 +; TONGA-NEXT: v_max_i32_e32 v5, v5, v13 +; TONGA-NEXT: v_max_i32_e32 v6, v6, v16 +; TONGA-NEXT: v_max_i32_e32 v1, v1, v12 +; TONGA-NEXT: v_ashrrev_i32_e32 v10, 31, v14 +; TONGA-NEXT: v_cvt_f32_u32_e32 v12, v4 +; TONGA-NEXT: v_cvt_f32_u32_e32 v14, v5 +; TONGA-NEXT: v_cvt_f32_u32_e32 v16, v6 +; TONGA-NEXT: v_sub_u32_e32 v9, vcc, 0, v0 ; TONGA-NEXT: v_rcp_iflag_f32_e32 v12, v12 -; TONGA-NEXT: v_mul_f32_e32 v8, 0x4f7ffffe, v8 -; TONGA-NEXT: v_mul_f32_e32 v10, 0x4f7ffffe, v10 -; TONGA-NEXT: v_cvt_u32_f32_e32 v8, v8 -; TONGA-NEXT: v_cvt_u32_f32_e32 v10, v10 +; TONGA-NEXT: v_rcp_iflag_f32_e32 v14, v14 +; TONGA-NEXT: v_rcp_iflag_f32_e32 v16, v16 +; TONGA-NEXT: v_sub_u32_e32 v15, vcc, 0, v2 ; TONGA-NEXT: v_mul_f32_e32 v12, 0x4f7ffffe, v12 +; TONGA-NEXT: v_mul_f32_e32 v14, 0x4f7ffffe, v14 +; TONGA-NEXT: v_mul_f32_e32 v16, 0x4f7ffffe, v16 ; TONGA-NEXT: v_cvt_u32_f32_e32 v12, v12 -; TONGA-NEXT: v_sub_u32_e32 v11, vcc, 0, v5 -; TONGA-NEXT: v_mul_lo_u32 v9, v9, v8 -; TONGA-NEXT: v_mul_lo_u32 v11, v11, v10 -; TONGA-NEXT: v_sub_u32_e32 v13, vcc, 0, v6 +; TONGA-NEXT: v_cvt_u32_f32_e32 v14, v14 +; TONGA-NEXT: v_cvt_u32_f32_e32 v16, v16 +; TONGA-NEXT: v_sub_u32_e32 v17, vcc, 0, v7 +; TONGA-NEXT: v_max_i32_e32 v0, v0, v9 +; TONGA-NEXT: v_ashrrev_i32_e32 v9, 31, v11 +; TONGA-NEXT: v_max_i32_e32 v2, v2, v15 +; TONGA-NEXT: v_max_i32_e32 v11, v7, v17 +; TONGA-NEXT: v_sub_u32_e32 v13, vcc, 0, v4 +; TONGA-NEXT: v_sub_u32_e32 v15, vcc, 0, v5 +; TONGA-NEXT: v_sub_u32_e32 v17, vcc, 0, v6 ; TONGA-NEXT: v_mul_lo_u32 v13, v13, v12 -; TONGA-NEXT: v_mul_hi_u32 v9, v8, v9 -; TONGA-NEXT: v_mul_hi_u32 v11, v10, v11 -; TONGA-NEXT: v_ashrrev_i32_e32 v14, 31, v7 -; TONGA-NEXT: v_add_u32_e32 v7, vcc, v7, v14 +; TONGA-NEXT: v_mul_lo_u32 v15, v15, v14 +; TONGA-NEXT: v_mul_lo_u32 v17, v17, v16 +; TONGA-NEXT: v_cvt_f32_u32_e32 v18, v11 ; TONGA-NEXT: v_mul_hi_u32 v13, v12, v13 -; TONGA-NEXT: v_xor_b32_e32 v7, v7, v14 -; TONGA-NEXT: v_cvt_f32_u32_e32 v18, v7 -; TONGA-NEXT: v_add_u32_e32 v8, vcc, v8, v9 -; TONGA-NEXT: v_add_u32_e32 v9, vcc, v10, v11 -; TONGA-NEXT: v_mul_hi_u32 v8, v0, v8 -; TONGA-NEXT: v_mul_hi_u32 v9, v1, v9 -; TONGA-NEXT: v_add_u32_e32 v10, vcc, v12, v13 -; TONGA-NEXT: v_mul_hi_u32 v10, v2, v10 +; TONGA-NEXT: v_mul_hi_u32 v15, v14, v15 +; TONGA-NEXT: v_mul_hi_u32 v17, v16, v17 ; TONGA-NEXT: v_rcp_iflag_f32_e32 v18, v18 -; TONGA-NEXT: v_mul_lo_u32 v11, v8, v4 -; TONGA-NEXT: v_mul_lo_u32 v13, v9, v5 -; TONGA-NEXT: v_mul_lo_u32 v21, v10, v6 +; TONGA-NEXT: v_add_u32_e32 v12, vcc, v12, v13 +; TONGA-NEXT: v_add_u32_e32 v13, vcc, v14, v15 +; TONGA-NEXT: v_add_u32_e32 v14, vcc, v16, v17 +; TONGA-NEXT: v_mul_hi_u32 v12, v0, v12 +; TONGA-NEXT: v_mul_hi_u32 v13, v1, v13 +; TONGA-NEXT: v_mul_hi_u32 v14, v2, v14 ; TONGA-NEXT: v_mul_f32_e32 v18, 0x4f7ffffe, v18 +; TONGA-NEXT: v_mul_lo_u32 v15, v12, v4 +; TONGA-NEXT: v_mul_lo_u32 v17, v13, v5 +; TONGA-NEXT: v_mul_lo_u32 v21, v14, v6 ; TONGA-NEXT: v_cvt_u32_f32_e32 v18, v18 -; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v0, v11 -; TONGA-NEXT: v_sub_u32_e32 v1, vcc, v1, v13 -; TONGA-NEXT: v_add_u32_e32 v12, vcc, 1, v8 -; TONGA-NEXT: v_add_u32_e32 v20, vcc, 1, v9 +; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v0, v15 +; TONGA-NEXT: v_sub_u32_e32 v1, vcc, v1, v17 +; TONGA-NEXT: v_sub_u32_e32 v2, vcc, v2, v21 +; TONGA-NEXT: v_add_u32_e32 v16, vcc, 1, v12 +; TONGA-NEXT: v_add_u32_e32 v20, vcc, 1, v13 +; TONGA-NEXT: v_add_u32_e32 v15, vcc, 1, v14 ; TONGA-NEXT: v_cmp_ge_u32_e64 s[0:1], v0, v4 ; TONGA-NEXT: v_cmp_ge_u32_e64 s[2:3], v1, v5 -; TONGA-NEXT: v_sub_u32_e32 v2, vcc, v2, v21 -; TONGA-NEXT: v_sub_u32_e32 v11, vcc, v0, v4 -; TONGA-NEXT: v_cndmask_b32_e64 v8, v8, v12, s[0:1] -; TONGA-NEXT: v_sub_u32_e32 v12, vcc, v1, v5 -; TONGA-NEXT: v_cndmask_b32_e64 v9, v9, v20, s[2:3] -; TONGA-NEXT: v_sub_u32_e32 v19, vcc, 0, v7 -; TONGA-NEXT: v_add_u32_e32 v22, vcc, 1, v10 -; TONGA-NEXT: v_sub_u32_e32 v13, vcc, v2, v6 -; TONGA-NEXT: v_cndmask_b32_e64 v0, v0, v11, s[0:1] -; TONGA-NEXT: v_add_u32_e32 v11, vcc, 1, v8 -; TONGA-NEXT: v_cndmask_b32_e64 v1, v1, v12, s[2:3] -; TONGA-NEXT: v_add_u32_e32 v12, vcc, 1, v9 +; TONGA-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6 +; TONGA-NEXT: v_sub_u32_e32 v19, vcc, 0, v11 +; TONGA-NEXT: v_sub_u32_e32 v17, vcc, v0, v4 +; TONGA-NEXT: v_cndmask_b32_e64 v12, v12, v16, s[0:1] +; TONGA-NEXT: v_sub_u32_e32 v16, vcc, v1, v5 +; TONGA-NEXT: v_cndmask_b32_e64 v13, v13, v20, s[2:3] +; TONGA-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[4:5] +; TONGA-NEXT: v_mul_lo_u32 v19, v19, v18 +; TONGA-NEXT: v_sub_u32_e32 v20, vcc, v2, v6 +; TONGA-NEXT: v_cndmask_b32_e64 v0, v0, v17, s[0:1] +; TONGA-NEXT: v_add_u32_e32 v15, vcc, 1, v12 +; TONGA-NEXT: v_cndmask_b32_e64 v1, v1, v16, s[2:3] +; TONGA-NEXT: v_add_u32_e32 v16, vcc, 1, v13 +; TONGA-NEXT: v_add_u32_e32 v17, vcc, 1, v14 ; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4 -; TONGA-NEXT: v_mul_lo_u32 v4, v19, v18 -; TONGA-NEXT: v_cndmask_b32_e32 v0, v8, v11, vcc +; TONGA-NEXT: v_cndmask_b32_e32 v0, v12, v15, vcc ; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5 -; TONGA-NEXT: v_ashrrev_i32_e32 v8, 31, v3 -; TONGA-NEXT: v_mul_hi_u32 v4, v18, v4 -; TONGA-NEXT: v_cndmask_b32_e32 v1, v9, v12, vcc -; TONGA-NEXT: v_add_u32_e32 v3, vcc, v3, v8 -; TONGA-NEXT: v_xor_b32_e32 v3, v3, v8 -; TONGA-NEXT: v_add_u32_e32 v4, vcc, v18, v4 -; TONGA-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6 -; TONGA-NEXT: v_mul_hi_u32 v4, v3, v4 -; TONGA-NEXT: v_cndmask_b32_e64 v10, v10, v22, s[4:5] -; TONGA-NEXT: v_xor_b32_e32 v0, v0, v15 -; TONGA-NEXT: v_xor_b32_e32 v1, v1, v16 -; TONGA-NEXT: v_cndmask_b32_e64 v2, v2, v13, s[4:5] -; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v0, v15 -; TONGA-NEXT: v_sub_u32_e32 v1, vcc, v1, v16 -; TONGA-NEXT: v_add_u32_e32 v5, vcc, 1, v10 +; TONGA-NEXT: v_ashrrev_i32_e32 v8, 31, v8 +; TONGA-NEXT: v_cndmask_b32_e32 v1, v13, v16, vcc +; TONGA-NEXT: v_xor_b32_e32 v0, v0, v8 +; TONGA-NEXT: v_xor_b32_e32 v1, v1, v9 +; TONGA-NEXT: v_mul_hi_u32 v4, v18, v19 +; TONGA-NEXT: v_cndmask_b32_e64 v2, v2, v20, s[4:5] +; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v0, v8 +; TONGA-NEXT: v_sub_u32_e32 v1, vcc, v1, v9 ; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6 -; TONGA-NEXT: v_cndmask_b32_e32 v2, v10, v5, vcc -; TONGA-NEXT: v_mul_lo_u32 v5, v4, v7 -; TONGA-NEXT: v_xor_b32_e32 v2, v2, v17 -; TONGA-NEXT: v_sub_u32_e32 v2, vcc, v2, v17 -; TONGA-NEXT: v_sub_u32_e32 v3, vcc, v3, v5 -; TONGA-NEXT: v_xor_b32_e32 v6, v8, v14 -; TONGA-NEXT: v_add_u32_e32 v5, vcc, 1, v4 -; TONGA-NEXT: v_sub_u32_e32 v8, vcc, v3, v7 -; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v3, v7 -; TONGA-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc -; TONGA-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc -; TONGA-NEXT: v_add_u32_e32 v5, vcc, 1, v4 -; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v3, v7 -; TONGA-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc -; TONGA-NEXT: v_xor_b32_e32 v3, v3, v6 -; TONGA-NEXT: v_sub_u32_e32 v3, vcc, v3, v6 +; TONGA-NEXT: v_cndmask_b32_e32 v2, v14, v17, vcc +; TONGA-NEXT: v_sub_u32_e32 v5, vcc, 0, v3 +; TONGA-NEXT: v_max_i32_e32 v5, v3, v5 +; TONGA-NEXT: v_add_u32_e32 v4, vcc, v18, v4 +; TONGA-NEXT: v_mul_hi_u32 v4, v5, v4 +; TONGA-NEXT: v_xor_b32_e32 v2, v2, v10 +; TONGA-NEXT: v_sub_u32_e32 v2, vcc, v2, v10 +; TONGA-NEXT: v_mul_lo_u32 v6, v4, v11 +; TONGA-NEXT: v_xor_b32_e32 v3, v3, v7 +; TONGA-NEXT: v_ashrrev_i32_e32 v3, 31, v3 +; TONGA-NEXT: v_sub_u32_e32 v5, vcc, v5, v6 +; TONGA-NEXT: v_add_u32_e32 v6, vcc, 1, v4 +; TONGA-NEXT: v_sub_u32_e32 v7, vcc, v5, v11 +; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v5, v11 +; TONGA-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; TONGA-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc +; TONGA-NEXT: v_add_u32_e32 v6, vcc, 1, v4 +; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v5, v11 +; TONGA-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; TONGA-NEXT: v_xor_b32_e32 v4, v4, v3 +; TONGA-NEXT: v_sub_u32_e32 v3, vcc, v4, v3 ; TONGA-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 ; TONGA-NEXT: s_endpgm ; @@ -1078,138 +1055,126 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: s_mov_b32 s9, s7 ; GFX9-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16 ; GFX9-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 -; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_readfirstlane_b32 s1, v0 -; GFX9-NEXT: s_ashr_i32 s4, s1, 31 -; GFX9-NEXT: s_add_i32 s1, s1, s4 -; GFX9-NEXT: s_xor_b32 s6, s1, s4 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6 +; GFX9-NEXT: v_readfirstlane_b32 s0, v0 +; GFX9-NEXT: s_abs_i32 s1, s0 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s1 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_readfirstlane_b32 s8, v4 -; GFX9-NEXT: s_ashr_i32 s9, s8, 31 -; GFX9-NEXT: s_add_i32 s8, s8, s9 +; GFX9-NEXT: v_readfirstlane_b32 s7, v4 +; GFX9-NEXT: s_xor_b32 s0, s7, s0 +; GFX9-NEXT: s_ashr_i32 s8, s0, 31 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s4, s9, s4 -; GFX9-NEXT: s_xor_b32 s8, s8, s9 -; GFX9-NEXT: s_sub_i32 s9, 0, s6 +; GFX9-NEXT: s_sub_i32 s0, 0, s1 +; GFX9-NEXT: s_abs_i32 s7, s7 +; GFX9-NEXT: v_readfirstlane_b32 s6, v1 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: v_readfirstlane_b32 s7, v1 -; GFX9-NEXT: s_mov_b32 s1, s5 -; GFX9-NEXT: v_readfirstlane_b32 s5, v3 -; GFX9-NEXT: v_readfirstlane_b32 s10, v0 -; GFX9-NEXT: s_mul_i32 s9, s9, s10 -; GFX9-NEXT: s_mul_hi_u32 s9, s10, s9 -; GFX9-NEXT: s_add_i32 s10, s10, s9 -; GFX9-NEXT: s_mul_hi_u32 s9, s8, s10 -; GFX9-NEXT: s_mul_i32 s10, s9, s6 -; GFX9-NEXT: s_sub_i32 s8, s8, s10 -; GFX9-NEXT: s_add_i32 s11, s9, 1 -; GFX9-NEXT: s_sub_i32 s10, s8, s6 -; GFX9-NEXT: s_cmp_ge_u32 s8, s6 -; GFX9-NEXT: s_cselect_b32 s9, s11, s9 -; GFX9-NEXT: s_cselect_b32 s8, s10, s8 -; GFX9-NEXT: s_add_i32 s10, s9, 1 -; GFX9-NEXT: s_cmp_ge_u32 s8, s6 -; GFX9-NEXT: s_cselect_b32 s6, s10, s9 -; GFX9-NEXT: s_ashr_i32 s8, s7, 31 -; GFX9-NEXT: s_add_i32 s7, s7, s8 -; GFX9-NEXT: s_xor_b32 s7, s7, s8 +; GFX9-NEXT: v_readfirstlane_b32 s9, v0 +; GFX9-NEXT: s_mul_i32 s0, s0, s9 +; GFX9-NEXT: s_mul_hi_u32 s0, s9, s0 +; GFX9-NEXT: s_add_i32 s9, s9, s0 +; GFX9-NEXT: s_mul_hi_u32 s0, s7, s9 +; GFX9-NEXT: s_mul_i32 s9, s0, s1 +; GFX9-NEXT: s_sub_i32 s7, s7, s9 +; GFX9-NEXT: s_add_i32 s10, s0, 1 +; GFX9-NEXT: s_sub_i32 s9, s7, s1 +; GFX9-NEXT: s_cmp_ge_u32 s7, s1 +; GFX9-NEXT: s_cselect_b32 s0, s10, s0 +; GFX9-NEXT: s_cselect_b32 s7, s9, s7 +; GFX9-NEXT: s_add_i32 s9, s0, 1 +; GFX9-NEXT: s_cmp_ge_u32 s7, s1 +; GFX9-NEXT: s_cselect_b32 s1, s9, s0 +; GFX9-NEXT: s_abs_i32 s7, s6 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7 -; GFX9-NEXT: v_readfirstlane_b32 s10, v5 -; GFX9-NEXT: s_ashr_i32 s11, s10, 31 -; GFX9-NEXT: s_xor_b32 s6, s6, s4 +; GFX9-NEXT: s_xor_b32 s1, s1, s8 +; GFX9-NEXT: s_sub_i32 s10, 0, s7 +; GFX9-NEXT: s_sub_i32 s8, s1, s8 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_add_i32 s10, s10, s11 -; GFX9-NEXT: s_xor_b32 s8, s11, s8 -; GFX9-NEXT: s_sub_i32 s4, s6, s4 +; GFX9-NEXT: v_readfirstlane_b32 s9, v5 +; GFX9-NEXT: s_xor_b32 s6, s9, s6 +; GFX9-NEXT: s_abs_i32 s9, s9 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s6, s10, s11 -; GFX9-NEXT: s_sub_i32 s10, 0, s7 -; GFX9-NEXT: v_readfirstlane_b32 s9, v2 -; GFX9-NEXT: v_readfirstlane_b32 s11, v0 -; GFX9-NEXT: s_mul_i32 s10, s10, s11 -; GFX9-NEXT: s_mul_hi_u32 s10, s11, s10 -; GFX9-NEXT: s_add_i32 s11, s11, s10 -; GFX9-NEXT: s_mul_hi_u32 s10, s6, s11 -; GFX9-NEXT: s_mul_i32 s11, s10, s7 -; GFX9-NEXT: s_sub_i32 s6, s6, s11 -; GFX9-NEXT: s_add_i32 s12, s10, 1 -; GFX9-NEXT: s_sub_i32 s11, s6, s7 -; GFX9-NEXT: s_cmp_ge_u32 s6, s7 -; GFX9-NEXT: s_cselect_b32 s10, s12, s10 -; GFX9-NEXT: s_cselect_b32 s6, s11, s6 -; GFX9-NEXT: s_add_i32 s11, s10, 1 -; GFX9-NEXT: s_cmp_ge_u32 s6, s7 -; GFX9-NEXT: s_cselect_b32 s6, s11, s10 -; GFX9-NEXT: s_ashr_i32 s7, s9, 31 -; GFX9-NEXT: s_add_i32 s9, s9, s7 -; GFX9-NEXT: s_xor_b32 s9, s9, s7 +; GFX9-NEXT: s_ashr_i32 s6, s6, 31 +; GFX9-NEXT: s_mov_b32 s0, s4 +; GFX9-NEXT: v_readfirstlane_b32 s4, v2 +; GFX9-NEXT: v_readfirstlane_b32 s1, v0 +; GFX9-NEXT: s_mul_i32 s10, s10, s1 +; GFX9-NEXT: s_mul_hi_u32 s10, s1, s10 +; GFX9-NEXT: s_add_i32 s1, s1, s10 +; GFX9-NEXT: s_mul_hi_u32 s1, s9, s1 +; GFX9-NEXT: s_mul_i32 s10, s1, s7 +; GFX9-NEXT: s_sub_i32 s9, s9, s10 +; GFX9-NEXT: s_add_i32 s11, s1, 1 +; GFX9-NEXT: s_sub_i32 s10, s9, s7 +; GFX9-NEXT: s_cmp_ge_u32 s9, s7 +; GFX9-NEXT: s_cselect_b32 s1, s11, s1 +; GFX9-NEXT: s_cselect_b32 s9, s10, s9 +; GFX9-NEXT: s_add_i32 s10, s1, 1 +; GFX9-NEXT: s_cmp_ge_u32 s9, s7 +; GFX9-NEXT: s_cselect_b32 s7, s10, s1 +; GFX9-NEXT: s_abs_i32 s9, s4 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s9 -; GFX9-NEXT: v_readfirstlane_b32 s11, v6 -; GFX9-NEXT: s_ashr_i32 s12, s11, 31 -; GFX9-NEXT: s_xor_b32 s6, s6, s8 +; GFX9-NEXT: s_xor_b32 s7, s7, s6 +; GFX9-NEXT: s_sub_i32 s11, 0, s9 +; GFX9-NEXT: s_sub_i32 s6, s7, s6 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_add_i32 s11, s11, s12 -; GFX9-NEXT: s_xor_b32 s7, s12, s7 -; GFX9-NEXT: s_sub_i32 s6, s6, s8 +; GFX9-NEXT: v_readfirstlane_b32 s10, v6 +; GFX9-NEXT: s_xor_b32 s4, s10, s4 +; GFX9-NEXT: s_abs_i32 s10, s10 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s8, s11, s12 -; GFX9-NEXT: s_sub_i32 s11, 0, s9 +; GFX9-NEXT: s_ashr_i32 s4, s4, 31 +; GFX9-NEXT: s_mov_b32 s1, s5 +; GFX9-NEXT: v_readfirstlane_b32 s5, v3 +; GFX9-NEXT: v_readfirstlane_b32 s7, v0 +; GFX9-NEXT: s_mul_i32 s11, s11, s7 +; GFX9-NEXT: s_mul_hi_u32 s11, s7, s11 +; GFX9-NEXT: s_add_i32 s7, s7, s11 +; GFX9-NEXT: s_mul_hi_u32 s7, s10, s7 +; GFX9-NEXT: s_mul_i32 s11, s7, s9 +; GFX9-NEXT: s_sub_i32 s10, s10, s11 +; GFX9-NEXT: s_add_i32 s12, s7, 1 +; GFX9-NEXT: s_sub_i32 s11, s10, s9 +; GFX9-NEXT: s_cmp_ge_u32 s10, s9 +; GFX9-NEXT: s_cselect_b32 s7, s12, s7 +; GFX9-NEXT: s_cselect_b32 s10, s11, s10 +; GFX9-NEXT: s_add_i32 s11, s7, 1 +; GFX9-NEXT: s_cmp_ge_u32 s10, s9 +; GFX9-NEXT: s_cselect_b32 s7, s11, s7 +; GFX9-NEXT: s_abs_i32 s9, s5 +; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s9 +; GFX9-NEXT: s_xor_b32 s7, s7, s4 +; GFX9-NEXT: v_mov_b32_e32 v0, s8 +; GFX9-NEXT: s_sub_i32 s8, 0, s9 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v2, v2 +; GFX9-NEXT: s_sub_i32 s4, s7, s4 ; GFX9-NEXT: v_readfirstlane_b32 s10, v7 -; GFX9-NEXT: v_readfirstlane_b32 s12, v0 -; GFX9-NEXT: s_mul_i32 s11, s11, s12 -; GFX9-NEXT: s_mul_hi_u32 s11, s12, s11 -; GFX9-NEXT: s_add_i32 s12, s12, s11 -; GFX9-NEXT: s_mul_hi_u32 s11, s8, s12 -; GFX9-NEXT: s_mul_i32 s12, s11, s9 -; GFX9-NEXT: s_sub_i32 s8, s8, s12 -; GFX9-NEXT: s_add_i32 s13, s11, 1 -; GFX9-NEXT: s_sub_i32 s12, s8, s9 -; GFX9-NEXT: s_cmp_ge_u32 s8, s9 -; GFX9-NEXT: s_cselect_b32 s11, s13, s11 -; GFX9-NEXT: s_cselect_b32 s8, s12, s8 -; GFX9-NEXT: s_add_i32 s12, s11, 1 -; GFX9-NEXT: s_cmp_ge_u32 s8, s9 -; GFX9-NEXT: s_cselect_b32 s8, s12, s11 -; GFX9-NEXT: s_ashr_i32 s9, s5, 31 -; GFX9-NEXT: s_add_i32 s5, s5, s9 -; GFX9-NEXT: s_xor_b32 s5, s5, s9 -; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s5 -; GFX9-NEXT: v_mov_b32_e32 v0, s4 ; GFX9-NEXT: v_mov_b32_e32 v1, s6 -; GFX9-NEXT: s_ashr_i32 s4, s10, 31 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v2, v2 -; GFX9-NEXT: s_xor_b32 s6, s8, s7 -; GFX9-NEXT: s_xor_b32 s8, s4, s9 -; GFX9-NEXT: s_sub_i32 s6, s6, s7 ; GFX9-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GFX9-NEXT: s_sub_i32 s7, 0, s5 -; GFX9-NEXT: s_add_i32 s10, s10, s4 -; GFX9-NEXT: s_xor_b32 s4, s10, s4 -; GFX9-NEXT: v_readfirstlane_b32 s9, v2 -; GFX9-NEXT: s_mul_i32 s7, s7, s9 -; GFX9-NEXT: s_mul_hi_u32 s7, s9, s7 -; GFX9-NEXT: s_add_i32 s9, s9, s7 -; GFX9-NEXT: s_mul_hi_u32 s7, s4, s9 -; GFX9-NEXT: s_mul_i32 s9, s7, s5 -; GFX9-NEXT: s_sub_i32 s4, s4, s9 +; GFX9-NEXT: s_abs_i32 s6, s10 +; GFX9-NEXT: s_xor_b32 s5, s10, s5 +; GFX9-NEXT: s_ashr_i32 s5, s5, 31 +; GFX9-NEXT: v_readfirstlane_b32 s7, v2 +; GFX9-NEXT: s_mul_i32 s8, s8, s7 +; GFX9-NEXT: s_mul_hi_u32 s8, s7, s8 +; GFX9-NEXT: s_add_i32 s7, s7, s8 +; GFX9-NEXT: s_mul_hi_u32 s7, s6, s7 +; GFX9-NEXT: s_mul_i32 s8, s7, s9 +; GFX9-NEXT: s_sub_i32 s6, s6, s8 ; GFX9-NEXT: s_add_i32 s10, s7, 1 -; GFX9-NEXT: s_sub_i32 s9, s4, s5 -; GFX9-NEXT: s_cmp_ge_u32 s4, s5 +; GFX9-NEXT: s_sub_i32 s8, s6, s9 +; GFX9-NEXT: s_cmp_ge_u32 s6, s9 ; GFX9-NEXT: s_cselect_b32 s7, s10, s7 -; GFX9-NEXT: s_cselect_b32 s4, s9, s4 -; GFX9-NEXT: s_add_i32 s9, s7, 1 -; GFX9-NEXT: s_cmp_ge_u32 s4, s5 -; GFX9-NEXT: s_cselect_b32 s4, s9, s7 -; GFX9-NEXT: s_xor_b32 s4, s4, s8 -; GFX9-NEXT: s_sub_i32 s4, s4, s8 -; GFX9-NEXT: v_mov_b32_e32 v2, s6 -; GFX9-NEXT: v_mov_b32_e32 v3, s4 +; GFX9-NEXT: s_cselect_b32 s6, s8, s6 +; GFX9-NEXT: s_add_i32 s8, s7, 1 +; GFX9-NEXT: s_cmp_ge_u32 s6, s9 +; GFX9-NEXT: s_cselect_b32 s6, s8, s7 +; GFX9-NEXT: s_xor_b32 s6, s6, s5 +; GFX9-NEXT: s_sub_i32 s5, s6, s5 +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 ; GFX9-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; GFX9-NEXT: s_endpgm ; @@ -2009,20 +1974,19 @@ define amdgpu_kernel void @v_sdiv_i25(ptr addrspace(1) %out, ptr addrspace(1) %i ; GCN-NEXT: s_mov_b32 s0, s4 ; GCN-NEXT: s_mov_b32 s1, s5 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_bfe_i32 v2, v1, 0, 25 -; GCN-NEXT: v_bfe_i32 v1, v1, 24, 1 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v1 -; GCN-NEXT: v_xor_b32_e32 v2, v2, v1 +; GCN-NEXT: v_bfe_i32 v1, v1, 0, 25 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v1 +; GCN-NEXT: v_max_i32_e32 v2, v1, v2 ; GCN-NEXT: v_cvt_f32_u32_e32 v3, v2 ; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v2 -; GCN-NEXT: v_bfe_i32 v5, v0, 0, 25 +; GCN-NEXT: v_bfe_i32 v0, v0, 0, 25 ; GCN-NEXT: v_rcp_iflag_f32_e32 v3, v3 -; GCN-NEXT: v_bfe_i32 v0, v0, 24, 1 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v0 +; GCN-NEXT: v_sub_i32_e32 v5, vcc, 0, v0 +; GCN-NEXT: v_max_i32_e32 v5, v0, v5 ; GCN-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3 ; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_xor_b32_e32 v5, v5, v0 ; GCN-NEXT: v_xor_b32_e32 v0, v0, v1 +; GCN-NEXT: v_ashrrev_i32_e32 v0, 31, v0 ; GCN-NEXT: v_mul_lo_u32 v4, v4, v3 ; GCN-NEXT: v_mul_hi_u32 v4, v3, v4 ; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v4 @@ -2030,7 +1994,7 @@ define amdgpu_kernel void @v_sdiv_i25(ptr addrspace(1) %out, ptr addrspace(1) %i ; GCN-NEXT: v_mul_lo_u32 v1, v3, v2 ; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v3 ; GCN-NEXT: v_sub_i32_e32 v1, vcc, v5, v1 -; GCN-NEXT: v_sub_i32_e32 v5, vcc, v1, v2 +; GCN-NEXT: v_subrev_i32_e32 v5, vcc, v2, v1 ; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2 ; GCN-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc @@ -2057,20 +2021,19 @@ define amdgpu_kernel void @v_sdiv_i25(ptr addrspace(1) %out, ptr addrspace(1) %i ; TONGA-NEXT: s_mov_b32 s0, s4 ; TONGA-NEXT: s_mov_b32 s1, s5 ; TONGA-NEXT: s_waitcnt vmcnt(0) -; TONGA-NEXT: v_bfe_i32 v2, v1, 0, 25 -; TONGA-NEXT: v_bfe_i32 v1, v1, 24, 1 -; TONGA-NEXT: v_add_u32_e32 v2, vcc, v2, v1 -; TONGA-NEXT: v_xor_b32_e32 v2, v2, v1 +; TONGA-NEXT: v_bfe_i32 v1, v1, 0, 25 +; TONGA-NEXT: v_sub_u32_e32 v2, vcc, 0, v1 +; TONGA-NEXT: v_max_i32_e32 v2, v1, v2 ; TONGA-NEXT: v_cvt_f32_u32_e32 v3, v2 ; TONGA-NEXT: v_sub_u32_e32 v4, vcc, 0, v2 -; TONGA-NEXT: v_bfe_i32 v5, v0, 0, 25 +; TONGA-NEXT: v_bfe_i32 v0, v0, 0, 25 ; TONGA-NEXT: v_rcp_iflag_f32_e32 v3, v3 -; TONGA-NEXT: v_bfe_i32 v0, v0, 24, 1 -; TONGA-NEXT: v_add_u32_e32 v5, vcc, v5, v0 +; TONGA-NEXT: v_sub_u32_e32 v5, vcc, 0, v0 +; TONGA-NEXT: v_max_i32_e32 v5, v0, v5 ; TONGA-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3 ; TONGA-NEXT: v_cvt_u32_f32_e32 v3, v3 -; TONGA-NEXT: v_xor_b32_e32 v5, v5, v0 ; TONGA-NEXT: v_xor_b32_e32 v0, v0, v1 +; TONGA-NEXT: v_ashrrev_i32_e32 v0, 31, v0 ; TONGA-NEXT: v_mul_lo_u32 v4, v4, v3 ; TONGA-NEXT: v_mul_hi_u32 v4, v3, v4 ; TONGA-NEXT: v_add_u32_e32 v3, vcc, v3, v4 @@ -2078,7 +2041,7 @@ define amdgpu_kernel void @v_sdiv_i25(ptr addrspace(1) %out, ptr addrspace(1) %i ; TONGA-NEXT: v_mul_lo_u32 v1, v3, v2 ; TONGA-NEXT: v_add_u32_e32 v4, vcc, 1, v3 ; TONGA-NEXT: v_sub_u32_e32 v1, vcc, v5, v1 -; TONGA-NEXT: v_sub_u32_e32 v5, vcc, v1, v2 +; TONGA-NEXT: v_subrev_u32_e32 v5, vcc, v2, v1 ; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2 ; TONGA-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; TONGA-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc @@ -2102,42 +2065,39 @@ define amdgpu_kernel void @v_sdiv_i25(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: s_mov_b32 s8, s6 ; GFX9-NEXT: s_mov_b32 s9, s7 ; GFX9-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0 +; GFX9-NEXT: s_mov_b32 s1, s5 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_readfirstlane_b32 s0, v1 -; GFX9-NEXT: s_bfe_i32 s1, s0, 0x190000 -; GFX9-NEXT: s_bfe_i32 s6, s0, 0x10018 -; GFX9-NEXT: s_add_i32 s1, s1, s6 -; GFX9-NEXT: s_xor_b32 s7, s1, s6 +; GFX9-NEXT: s_bfe_i32 s6, s0, 0x190000 +; GFX9-NEXT: s_abs_i32 s7, s6 ; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s7 ; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: v_readfirstlane_b32 s4, v0 -; GFX9-NEXT: s_mov_b32 s1, s5 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v1 -; GFX9-NEXT: s_bfe_i32 s5, s4, 0x190000 -; GFX9-NEXT: s_bfe_i32 s4, s4, 0x10018 -; GFX9-NEXT: s_add_i32 s5, s5, s4 -; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GFX9-NEXT: s_bfe_i32 s4, s4, 0x190000 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 +; GFX9-NEXT: s_xor_b32 s5, s4, s6 +; GFX9-NEXT: s_sub_i32 s6, 0, s7 +; GFX9-NEXT: s_abs_i32 s4, s4 +; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v1 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s6, s4, s6 -; GFX9-NEXT: s_xor_b32 s4, s5, s4 -; GFX9-NEXT: s_sub_i32 s5, 0, s7 +; GFX9-NEXT: s_ashr_i32 s5, s5, 31 ; GFX9-NEXT: v_readfirstlane_b32 s8, v0 -; GFX9-NEXT: s_mul_i32 s5, s5, s8 -; GFX9-NEXT: s_mul_hi_u32 s5, s8, s5 -; GFX9-NEXT: s_add_i32 s8, s8, s5 -; GFX9-NEXT: s_mul_hi_u32 s5, s4, s8 -; GFX9-NEXT: s_mul_i32 s8, s5, s7 +; GFX9-NEXT: s_mul_i32 s6, s6, s8 +; GFX9-NEXT: s_mul_hi_u32 s6, s8, s6 +; GFX9-NEXT: s_add_i32 s8, s8, s6 +; GFX9-NEXT: s_mul_hi_u32 s6, s4, s8 +; GFX9-NEXT: s_mul_i32 s8, s6, s7 ; GFX9-NEXT: s_sub_i32 s4, s4, s8 -; GFX9-NEXT: s_add_i32 s9, s5, 1 +; GFX9-NEXT: s_add_i32 s9, s6, 1 ; GFX9-NEXT: s_sub_i32 s8, s4, s7 ; GFX9-NEXT: s_cmp_ge_u32 s4, s7 -; GFX9-NEXT: s_cselect_b32 s5, s9, s5 +; GFX9-NEXT: s_cselect_b32 s6, s9, s6 ; GFX9-NEXT: s_cselect_b32 s4, s8, s4 -; GFX9-NEXT: s_add_i32 s8, s5, 1 +; GFX9-NEXT: s_add_i32 s8, s6, 1 ; GFX9-NEXT: s_cmp_ge_u32 s4, s7 -; GFX9-NEXT: s_cselect_b32 s4, s8, s5 -; GFX9-NEXT: s_xor_b32 s4, s4, s6 -; GFX9-NEXT: s_sub_i32 s4, s4, s6 +; GFX9-NEXT: s_cselect_b32 s4, s8, s6 +; GFX9-NEXT: s_xor_b32 s4, s4, s5 +; GFX9-NEXT: s_sub_i32 s4, s4, s5 ; GFX9-NEXT: s_bfe_i32 s4, s4, 0x190000 ; GFX9-NEXT: v_mov_b32_e32 v0, s4 ; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll index b086640c72f8..c310e257adad 100644 --- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll +++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll @@ -147,11 +147,11 @@ define amdgpu_kernel void @s_test_sdiv(ptr addrspace(1) %out, i64 %x, i64 %y) { ; GCN-IR-NEXT: s_ashr_i32 s0, s7, 31 ; GCN-IR-NEXT: s_mov_b32 s1, s0 ; GCN-IR-NEXT: s_ashr_i32 s2, s9, 31 -; GCN-IR-NEXT: s_xor_b64 s[6:7], s[0:1], s[6:7] +; GCN-IR-NEXT: s_xor_b64 s[6:7], s[6:7], s[0:1] ; GCN-IR-NEXT: s_mov_b32 s3, s2 ; GCN-IR-NEXT: s_sub_u32 s12, s6, s0 ; GCN-IR-NEXT: s_subb_u32 s13, s7, s0 -; GCN-IR-NEXT: s_xor_b64 s[6:7], s[2:3], s[8:9] +; GCN-IR-NEXT: s_xor_b64 s[6:7], s[8:9], s[2:3] ; GCN-IR-NEXT: s_sub_u32 s6, s6, s2 ; GCN-IR-NEXT: s_subb_u32 s7, s7, s2 ; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[12:13], 0 @@ -357,13 +357,13 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) { ; GCN-IR: ; %bb.0: ; %_udiv-special-cases ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_ashrrev_i32_e32 v12, 31, v1 -; GCN-IR-NEXT: v_xor_b32_e32 v0, v12, v0 +; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v12 ; GCN-IR-NEXT: v_ashrrev_i32_e32 v13, 31, v3 -; GCN-IR-NEXT: v_xor_b32_e32 v1, v12, v1 +; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v12 ; GCN-IR-NEXT: v_sub_i32_e32 v6, vcc, v0, v12 ; GCN-IR-NEXT: v_subb_u32_e32 v7, vcc, v1, v12, vcc -; GCN-IR-NEXT: v_xor_b32_e32 v0, v13, v2 -; GCN-IR-NEXT: v_xor_b32_e32 v1, v13, v3 +; GCN-IR-NEXT: v_xor_b32_e32 v0, v2, v13 +; GCN-IR-NEXT: v_xor_b32_e32 v1, v3, v13 ; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v13 ; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v13, vcc ; GCN-IR-NEXT: v_ffbh_u32_e32 v2, v0 @@ -587,43 +587,39 @@ define i64 @v_test_sdiv24_64(i64 %x, i64 %y) { define amdgpu_kernel void @s_test_sdiv32_64(ptr addrspace(1) %out, i64 %x, i64 %y) { ; GCN-LABEL: s_test_sdiv32_64: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dword s2, s[0:1], 0xe +; GCN-NEXT: s_load_dword s8, s[0:1], 0xe +; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_ashr_i32 s8, s2, 31 -; GCN-NEXT: s_add_i32 s2, s2, s8 -; GCN-NEXT: s_xor_b32 s9, s2, s8 +; GCN-NEXT: s_abs_i32 s9, s8 ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s9 -; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 -; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_sub_i32 s2, 0, s9 +; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: s_abs_i32 s0, s3 ; GCN-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: s_xor_b32 s1, s3, s8 +; GCN-NEXT: s_ashr_i32 s1, s1, 31 ; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 -; GCN-NEXT: s_ashr_i32 s2, s3, 31 -; GCN-NEXT: s_add_i32 s3, s3, s2 -; GCN-NEXT: s_xor_b32 s3, s3, s2 ; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 -; GCN-NEXT: s_xor_b32 s0, s2, s8 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-NEXT: v_mul_hi_u32 v0, s3, v0 -; GCN-NEXT: v_readfirstlane_b32 s1, v0 -; GCN-NEXT: s_mul_i32 s2, s1, s9 -; GCN-NEXT: s_sub_i32 s2, s3, s2 -; GCN-NEXT: s_add_i32 s8, s1, 1 -; GCN-NEXT: s_sub_i32 s3, s2, s9 -; GCN-NEXT: s_cmp_ge_u32 s2, s9 -; GCN-NEXT: s_cselect_b32 s1, s8, s1 -; GCN-NEXT: s_cselect_b32 s2, s3, s2 -; GCN-NEXT: s_add_i32 s3, s1, 1 -; GCN-NEXT: s_cmp_ge_u32 s2, s9 -; GCN-NEXT: s_cselect_b32 s1, s3, s1 -; GCN-NEXT: s_xor_b32 s1, s1, s0 -; GCN-NEXT: s_sub_i32 s0, s1, s0 +; GCN-NEXT: v_mul_hi_u32 v0, s0, v0 +; GCN-NEXT: v_readfirstlane_b32 s2, v0 +; GCN-NEXT: s_mul_i32 s3, s2, s9 +; GCN-NEXT: s_sub_i32 s0, s0, s3 +; GCN-NEXT: s_add_i32 s8, s2, 1 +; GCN-NEXT: s_sub_i32 s3, s0, s9 +; GCN-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-NEXT: s_cselect_b32 s2, s8, s2 +; GCN-NEXT: s_cselect_b32 s0, s3, s0 +; GCN-NEXT: s_add_i32 s3, s2, 1 +; GCN-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-NEXT: s_cselect_b32 s0, s3, s2 +; GCN-NEXT: s_xor_b32 s0, s0, s1 +; GCN-NEXT: s_sub_i32 s0, s0, s1 ; GCN-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-NEXT: v_mov_b32_e32 v0, s0 ; GCN-NEXT: v_mov_b32_e32 v1, s1 @@ -632,43 +628,39 @@ define amdgpu_kernel void @s_test_sdiv32_64(ptr addrspace(1) %out, i64 %x, i64 % ; ; GCN-IR-LABEL: s_test_sdiv32_64: ; GCN-IR: ; %bb.0: -; GCN-IR-NEXT: s_load_dword s2, s[0:1], 0xe +; GCN-IR-NEXT: s_load_dword s8, s[0:1], 0xe +; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s6, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_ashr_i32 s8, s2, 31 -; GCN-IR-NEXT: s_add_i32 s2, s2, s8 -; GCN-IR-NEXT: s_xor_b32 s9, s2, s8 +; GCN-IR-NEXT: s_abs_i32 s9, s8 ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s9 -; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 -; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_sub_i32 s2, 0, s9 +; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: s_abs_i32 s0, s3 ; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: s_xor_b32 s1, s3, s8 +; GCN-IR-NEXT: s_ashr_i32 s1, s1, 31 ; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 -; GCN-IR-NEXT: s_ashr_i32 s2, s3, 31 -; GCN-IR-NEXT: s_add_i32 s3, s3, s2 -; GCN-IR-NEXT: s_xor_b32 s3, s3, s2 ; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 -; GCN-IR-NEXT: s_xor_b32 s0, s2, s8 ; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-IR-NEXT: v_mul_hi_u32 v0, s3, v0 -; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 -; GCN-IR-NEXT: s_mul_i32 s2, s1, s9 -; GCN-IR-NEXT: s_sub_i32 s2, s3, s2 -; GCN-IR-NEXT: s_add_i32 s8, s1, 1 -; GCN-IR-NEXT: s_sub_i32 s3, s2, s9 -; GCN-IR-NEXT: s_cmp_ge_u32 s2, s9 -; GCN-IR-NEXT: s_cselect_b32 s1, s8, s1 -; GCN-IR-NEXT: s_cselect_b32 s2, s3, s2 -; GCN-IR-NEXT: s_add_i32 s3, s1, 1 -; GCN-IR-NEXT: s_cmp_ge_u32 s2, s9 -; GCN-IR-NEXT: s_cselect_b32 s1, s3, s1 -; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 -; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 +; GCN-IR-NEXT: v_mul_hi_u32 v0, s0, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s2, v0 +; GCN-IR-NEXT: s_mul_i32 s3, s2, s9 +; GCN-IR-NEXT: s_sub_i32 s0, s0, s3 +; GCN-IR-NEXT: s_add_i32 s8, s2, 1 +; GCN-IR-NEXT: s_sub_i32 s3, s0, s9 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-IR-NEXT: s_cselect_b32 s2, s8, s2 +; GCN-IR-NEXT: s_cselect_b32 s0, s3, s0 +; GCN-IR-NEXT: s_add_i32 s3, s2, 1 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-IR-NEXT: s_cselect_b32 s0, s3, s2 +; GCN-IR-NEXT: s_xor_b32 s0, s0, s1 +; GCN-IR-NEXT: s_sub_i32 s0, s0, s1 ; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 @@ -688,41 +680,38 @@ define amdgpu_kernel void @s_test_sdiv31_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_ashr_i64 s[2:3], s[2:3], 33 -; GCN-NEXT: s_ashr_i32 s8, s2, 31 -; GCN-NEXT: s_add_i32 s2, s2, s8 -; GCN-NEXT: s_xor_b32 s9, s2, s8 +; GCN-NEXT: s_ashr_i64 s[8:9], s[2:3], 33 +; GCN-NEXT: s_abs_i32 s9, s8 ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s9 ; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_sub_i32 s2, 0, s9 ; GCN-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GCN-NEXT: s_mov_b32 s5, s1 ; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: s_mov_b32 s5, s1 ; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 ; GCN-NEXT: s_ashr_i64 s[2:3], s[2:3], 33 -; GCN-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-NEXT: s_add_i32 s2, s2, s3 +; GCN-NEXT: s_abs_i32 s0, s2 +; GCN-NEXT: s_xor_b32 s1, s2, s8 ; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 -; GCN-NEXT: s_xor_b32 s2, s2, s3 -; GCN-NEXT: s_xor_b32 s0, s3, s8 +; GCN-NEXT: s_ashr_i32 s1, s1, 31 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-NEXT: v_mul_hi_u32 v0, s2, v0 -; GCN-NEXT: v_readfirstlane_b32 s1, v0 -; GCN-NEXT: s_mul_i32 s3, s1, s9 -; GCN-NEXT: s_sub_i32 s2, s2, s3 -; GCN-NEXT: s_add_i32 s8, s1, 1 -; GCN-NEXT: s_sub_i32 s3, s2, s9 -; GCN-NEXT: s_cmp_ge_u32 s2, s9 -; GCN-NEXT: s_cselect_b32 s1, s8, s1 -; GCN-NEXT: s_cselect_b32 s2, s3, s2 -; GCN-NEXT: s_add_i32 s3, s1, 1 -; GCN-NEXT: s_cmp_ge_u32 s2, s9 -; GCN-NEXT: s_cselect_b32 s1, s3, s1 -; GCN-NEXT: s_xor_b32 s1, s1, s0 -; GCN-NEXT: s_sub_i32 s0, s1, s0 +; GCN-NEXT: v_mul_hi_u32 v0, s0, v0 +; GCN-NEXT: v_readfirstlane_b32 s2, v0 +; GCN-NEXT: s_mul_i32 s3, s2, s9 +; GCN-NEXT: s_sub_i32 s0, s0, s3 +; GCN-NEXT: s_add_i32 s8, s2, 1 +; GCN-NEXT: s_sub_i32 s3, s0, s9 +; GCN-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-NEXT: s_cselect_b32 s2, s8, s2 +; GCN-NEXT: s_cselect_b32 s0, s3, s0 +; GCN-NEXT: s_add_i32 s3, s2, 1 +; GCN-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-NEXT: s_cselect_b32 s0, s3, s2 +; GCN-NEXT: s_xor_b32 s0, s0, s1 +; GCN-NEXT: s_sub_i32 s0, s0, s1 ; GCN-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-NEXT: v_mov_b32_e32 v0, s0 ; GCN-NEXT: v_mov_b32_e32 v1, s1 @@ -735,41 +724,38 @@ define amdgpu_kernel void @s_test_sdiv31_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s6, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_ashr_i64 s[2:3], s[2:3], 33 -; GCN-IR-NEXT: s_ashr_i32 s8, s2, 31 -; GCN-IR-NEXT: s_add_i32 s2, s2, s8 -; GCN-IR-NEXT: s_xor_b32 s9, s2, s8 +; GCN-IR-NEXT: s_ashr_i64 s[8:9], s[2:3], 33 +; GCN-IR-NEXT: s_abs_i32 s9, s8 ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s9 ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_sub_i32 s2, 0, s9 ; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GCN-IR-NEXT: s_mov_b32 s5, s1 ; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: s_mov_b32 s5, s1 ; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 ; GCN-IR-NEXT: s_ashr_i64 s[2:3], s[2:3], 33 -; GCN-IR-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-IR-NEXT: s_add_i32 s2, s2, s3 +; GCN-IR-NEXT: s_abs_i32 s0, s2 +; GCN-IR-NEXT: s_xor_b32 s1, s2, s8 ; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 -; GCN-IR-NEXT: s_xor_b32 s2, s2, s3 -; GCN-IR-NEXT: s_xor_b32 s0, s3, s8 +; GCN-IR-NEXT: s_ashr_i32 s1, s1, 31 ; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-IR-NEXT: v_mul_hi_u32 v0, s2, v0 -; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 -; GCN-IR-NEXT: s_mul_i32 s3, s1, s9 -; GCN-IR-NEXT: s_sub_i32 s2, s2, s3 -; GCN-IR-NEXT: s_add_i32 s8, s1, 1 -; GCN-IR-NEXT: s_sub_i32 s3, s2, s9 -; GCN-IR-NEXT: s_cmp_ge_u32 s2, s9 -; GCN-IR-NEXT: s_cselect_b32 s1, s8, s1 -; GCN-IR-NEXT: s_cselect_b32 s2, s3, s2 -; GCN-IR-NEXT: s_add_i32 s3, s1, 1 -; GCN-IR-NEXT: s_cmp_ge_u32 s2, s9 -; GCN-IR-NEXT: s_cselect_b32 s1, s3, s1 -; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 -; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 +; GCN-IR-NEXT: v_mul_hi_u32 v0, s0, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s2, v0 +; GCN-IR-NEXT: s_mul_i32 s3, s2, s9 +; GCN-IR-NEXT: s_sub_i32 s0, s0, s3 +; GCN-IR-NEXT: s_add_i32 s8, s2, 1 +; GCN-IR-NEXT: s_sub_i32 s3, s0, s9 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-IR-NEXT: s_cselect_b32 s2, s8, s2 +; GCN-IR-NEXT: s_cselect_b32 s0, s3, s0 +; GCN-IR-NEXT: s_add_i32 s3, s2, 1 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-IR-NEXT: s_cselect_b32 s0, s3, s2 +; GCN-IR-NEXT: s_xor_b32 s0, s0, s1 +; GCN-IR-NEXT: s_sub_i32 s0, s0, s1 ; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 @@ -856,41 +842,38 @@ define amdgpu_kernel void @s_test_sdiv25_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_ashr_i64 s[2:3], s[2:3], 39 -; GCN-NEXT: s_ashr_i32 s8, s2, 31 -; GCN-NEXT: s_add_i32 s2, s2, s8 -; GCN-NEXT: s_xor_b32 s9, s2, s8 +; GCN-NEXT: s_ashr_i64 s[8:9], s[2:3], 39 +; GCN-NEXT: s_abs_i32 s9, s8 ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s9 ; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_sub_i32 s2, 0, s9 ; GCN-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GCN-NEXT: s_mov_b32 s5, s1 ; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: s_mov_b32 s5, s1 ; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 ; GCN-NEXT: s_ashr_i64 s[2:3], s[2:3], 39 -; GCN-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-NEXT: s_add_i32 s2, s2, s3 +; GCN-NEXT: s_abs_i32 s0, s2 +; GCN-NEXT: s_xor_b32 s1, s2, s8 ; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 -; GCN-NEXT: s_xor_b32 s2, s2, s3 -; GCN-NEXT: s_xor_b32 s0, s3, s8 +; GCN-NEXT: s_ashr_i32 s1, s1, 31 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-NEXT: v_mul_hi_u32 v0, s2, v0 -; GCN-NEXT: v_readfirstlane_b32 s1, v0 -; GCN-NEXT: s_mul_i32 s3, s1, s9 -; GCN-NEXT: s_sub_i32 s2, s2, s3 -; GCN-NEXT: s_add_i32 s8, s1, 1 -; GCN-NEXT: s_sub_i32 s3, s2, s9 -; GCN-NEXT: s_cmp_ge_u32 s2, s9 -; GCN-NEXT: s_cselect_b32 s1, s8, s1 -; GCN-NEXT: s_cselect_b32 s2, s3, s2 -; GCN-NEXT: s_add_i32 s3, s1, 1 -; GCN-NEXT: s_cmp_ge_u32 s2, s9 -; GCN-NEXT: s_cselect_b32 s1, s3, s1 -; GCN-NEXT: s_xor_b32 s1, s1, s0 -; GCN-NEXT: s_sub_i32 s0, s1, s0 +; GCN-NEXT: v_mul_hi_u32 v0, s0, v0 +; GCN-NEXT: v_readfirstlane_b32 s2, v0 +; GCN-NEXT: s_mul_i32 s3, s2, s9 +; GCN-NEXT: s_sub_i32 s0, s0, s3 +; GCN-NEXT: s_add_i32 s8, s2, 1 +; GCN-NEXT: s_sub_i32 s3, s0, s9 +; GCN-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-NEXT: s_cselect_b32 s2, s8, s2 +; GCN-NEXT: s_cselect_b32 s0, s3, s0 +; GCN-NEXT: s_add_i32 s3, s2, 1 +; GCN-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-NEXT: s_cselect_b32 s0, s3, s2 +; GCN-NEXT: s_xor_b32 s0, s0, s1 +; GCN-NEXT: s_sub_i32 s0, s0, s1 ; GCN-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-NEXT: v_mov_b32_e32 v0, s0 ; GCN-NEXT: v_mov_b32_e32 v1, s1 @@ -903,41 +886,38 @@ define amdgpu_kernel void @s_test_sdiv25_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s6, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_ashr_i64 s[2:3], s[2:3], 39 -; GCN-IR-NEXT: s_ashr_i32 s8, s2, 31 -; GCN-IR-NEXT: s_add_i32 s2, s2, s8 -; GCN-IR-NEXT: s_xor_b32 s9, s2, s8 +; GCN-IR-NEXT: s_ashr_i64 s[8:9], s[2:3], 39 +; GCN-IR-NEXT: s_abs_i32 s9, s8 ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s9 ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_sub_i32 s2, 0, s9 ; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GCN-IR-NEXT: s_mov_b32 s5, s1 ; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: s_mov_b32 s5, s1 ; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 ; GCN-IR-NEXT: s_ashr_i64 s[2:3], s[2:3], 39 -; GCN-IR-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-IR-NEXT: s_add_i32 s2, s2, s3 +; GCN-IR-NEXT: s_abs_i32 s0, s2 +; GCN-IR-NEXT: s_xor_b32 s1, s2, s8 ; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 -; GCN-IR-NEXT: s_xor_b32 s2, s2, s3 -; GCN-IR-NEXT: s_xor_b32 s0, s3, s8 +; GCN-IR-NEXT: s_ashr_i32 s1, s1, 31 ; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-IR-NEXT: v_mul_hi_u32 v0, s2, v0 -; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 -; GCN-IR-NEXT: s_mul_i32 s3, s1, s9 -; GCN-IR-NEXT: s_sub_i32 s2, s2, s3 -; GCN-IR-NEXT: s_add_i32 s8, s1, 1 -; GCN-IR-NEXT: s_sub_i32 s3, s2, s9 -; GCN-IR-NEXT: s_cmp_ge_u32 s2, s9 -; GCN-IR-NEXT: s_cselect_b32 s1, s8, s1 -; GCN-IR-NEXT: s_cselect_b32 s2, s3, s2 -; GCN-IR-NEXT: s_add_i32 s3, s1, 1 -; GCN-IR-NEXT: s_cmp_ge_u32 s2, s9 -; GCN-IR-NEXT: s_cselect_b32 s1, s3, s1 -; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 -; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 +; GCN-IR-NEXT: v_mul_hi_u32 v0, s0, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s2, v0 +; GCN-IR-NEXT: s_mul_i32 s3, s2, s9 +; GCN-IR-NEXT: s_sub_i32 s0, s0, s3 +; GCN-IR-NEXT: s_add_i32 s8, s2, 1 +; GCN-IR-NEXT: s_sub_i32 s3, s0, s9 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-IR-NEXT: s_cselect_b32 s2, s8, s2 +; GCN-IR-NEXT: s_cselect_b32 s0, s3, s0 +; GCN-IR-NEXT: s_add_i32 s3, s2, 1 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-IR-NEXT: s_cselect_b32 s0, s3, s2 +; GCN-IR-NEXT: s_xor_b32 s0, s0, s1 +; GCN-IR-NEXT: s_sub_i32 s0, s0, s1 ; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 @@ -1100,11 +1080,11 @@ define amdgpu_kernel void @s_test_sdiv24_48(ptr addrspace(1) %out, i48 %x, i48 % ; GCN-IR-NEXT: s_mov_b32 s3, s2 ; GCN-IR-NEXT: s_ashr_i64 s[8:9], s[4:5], 16 ; GCN-IR-NEXT: s_ashr_i32 s4, s5, 31 -; GCN-IR-NEXT: s_xor_b64 s[6:7], s[2:3], s[6:7] +; GCN-IR-NEXT: s_xor_b64 s[6:7], s[6:7], s[2:3] ; GCN-IR-NEXT: s_mov_b32 s5, s4 ; GCN-IR-NEXT: s_sub_u32 s12, s6, s2 ; GCN-IR-NEXT: s_subb_u32 s13, s7, s2 -; GCN-IR-NEXT: s_xor_b64 s[6:7], s[4:5], s[8:9] +; GCN-IR-NEXT: s_xor_b64 s[6:7], s[8:9], s[4:5] ; GCN-IR-NEXT: s_sub_u32 s6, s6, s4 ; GCN-IR-NEXT: s_subb_u32 s7, s7, s4 ; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[6:7], 0 @@ -1310,7 +1290,7 @@ define amdgpu_kernel void @s_test_sdiv_k_num_i64(ptr addrspace(1) %out, i64 %x) ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_ashr_i32 s4, s3, 31 ; GCN-IR-NEXT: s_mov_b32 s5, s4 -; GCN-IR-NEXT: s_xor_b64 s[2:3], s[4:5], s[2:3] +; GCN-IR-NEXT: s_xor_b64 s[2:3], s[2:3], s[4:5] ; GCN-IR-NEXT: s_sub_u32 s2, s2, s4 ; GCN-IR-NEXT: s_subb_u32 s3, s3, s4 ; GCN-IR-NEXT: s_flbit_i32_b64 s14, s[2:3] @@ -1493,8 +1473,8 @@ define i64 @v_test_sdiv_k_num_i64(i64 %x) { ; GCN-IR: ; %bb.0: ; %_udiv-special-cases ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_ashrrev_i32_e32 v12, 31, v1 -; GCN-IR-NEXT: v_xor_b32_e32 v0, v12, v0 -; GCN-IR-NEXT: v_xor_b32_e32 v1, v12, v1 +; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v12 +; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v12 ; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v12 ; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v12, vcc ; GCN-IR-NEXT: v_ffbh_u32_e32 v2, v0 @@ -1686,8 +1666,8 @@ define i64 @v_test_sdiv_pow2_k_num_i64(i64 %x) { ; GCN-IR: ; %bb.0: ; %_udiv-special-cases ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_ashrrev_i32_e32 v12, 31, v1 -; GCN-IR-NEXT: v_xor_b32_e32 v0, v12, v0 -; GCN-IR-NEXT: v_xor_b32_e32 v1, v12, v1 +; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v12 +; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v12 ; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v12 ; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v12, vcc ; GCN-IR-NEXT: v_ffbh_u32_e32 v2, v0 @@ -1788,8 +1768,8 @@ define i64 @v_test_sdiv_pow2_k_den_i64(i64 %x) { ; GCN-IR: ; %bb.0: ; %_udiv-special-cases ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_ashrrev_i32_e32 v10, 31, v1 -; GCN-IR-NEXT: v_xor_b32_e32 v0, v10, v0 -; GCN-IR-NEXT: v_xor_b32_e32 v1, v10, v1 +; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v10 +; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v10 ; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v0, v10 ; GCN-IR-NEXT: v_subb_u32_e32 v5, vcc, v1, v10, vcc ; GCN-IR-NEXT: v_ffbh_u32_e32 v0, v4 diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll index ed7f27b367fd..93fab7dff253 100644 --- a/llvm/test/CodeGen/AMDGPU/srem64.ll +++ b/llvm/test/CodeGen/AMDGPU/srem64.ll @@ -335,11 +335,11 @@ define i64 @v_test_srem(i64 %x, i64 %y) { ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_ashrrev_i32_e32 v14, 31, v1 ; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v14 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v4, 31, v3 ; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v14 ; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v14 -; GCN-IR-NEXT: v_xor_b32_e32 v2, v2, v4 +; GCN-IR-NEXT: v_ashrrev_i32_e32 v4, 31, v3 ; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v14, vcc +; GCN-IR-NEXT: v_xor_b32_e32 v2, v2, v4 ; GCN-IR-NEXT: v_xor_b32_e32 v3, v3, v4 ; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v2, v4 ; GCN-IR-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc @@ -655,37 +655,34 @@ define amdgpu_kernel void @s_test_srem25_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_ashr_i64 s[2:3], s[2:3], 39 -; GCN-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-NEXT: s_add_i32 s2, s2, s3 -; GCN-NEXT: s_xor_b32 s8, s2, s3 +; GCN-NEXT: s_abs_i32 s8, s2 ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 ; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_sub_i32 s2, 0, s8 ; GCN-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GCN-NEXT: s_mov_b32 s4, s0 ; GCN-NEXT: s_mov_b32 s5, s1 +; GCN-NEXT: s_mov_b32 s4, s0 ; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 ; GCN-NEXT: s_ashr_i64 s[2:3], s[2:3], 39 -; GCN-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-NEXT: s_add_i32 s2, s2, s3 +; GCN-NEXT: s_abs_i32 s3, s2 +; GCN-NEXT: s_ashr_i32 s0, s2, 31 ; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 -; GCN-NEXT: s_xor_b32 s2, s2, s3 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-NEXT: v_mul_hi_u32 v0, s2, v0 -; GCN-NEXT: v_readfirstlane_b32 s0, v0 -; GCN-NEXT: s_mul_i32 s0, s0, s8 -; GCN-NEXT: s_sub_i32 s0, s2, s0 -; GCN-NEXT: s_sub_i32 s1, s0, s8 -; GCN-NEXT: s_cmp_ge_u32 s0, s8 -; GCN-NEXT: s_cselect_b32 s0, s1, s0 -; GCN-NEXT: s_sub_i32 s1, s0, s8 -; GCN-NEXT: s_cmp_ge_u32 s0, s8 -; GCN-NEXT: s_cselect_b32 s0, s1, s0 -; GCN-NEXT: s_xor_b32 s0, s0, s3 -; GCN-NEXT: s_sub_i32 s0, s0, s3 +; GCN-NEXT: v_mul_hi_u32 v0, s3, v0 +; GCN-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-NEXT: s_mul_i32 s1, s1, s8 +; GCN-NEXT: s_sub_i32 s1, s3, s1 +; GCN-NEXT: s_sub_i32 s2, s1, s8 +; GCN-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-NEXT: s_sub_i32 s2, s1, s8 +; GCN-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-NEXT: s_xor_b32 s1, s1, s0 +; GCN-NEXT: s_sub_i32 s0, s1, s0 ; GCN-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-NEXT: v_mov_b32_e32 v0, s0 ; GCN-NEXT: v_mov_b32_e32 v1, s1 @@ -699,37 +696,34 @@ define amdgpu_kernel void @s_test_srem25_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-IR-NEXT: s_mov_b32 s6, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_ashr_i64 s[2:3], s[2:3], 39 -; GCN-IR-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-IR-NEXT: s_add_i32 s2, s2, s3 -; GCN-IR-NEXT: s_xor_b32 s8, s2, s3 +; GCN-IR-NEXT: s_abs_i32 s8, s2 ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s8 ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_sub_i32 s2, 0, s8 ; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GCN-IR-NEXT: s_mov_b32 s4, s0 ; GCN-IR-NEXT: s_mov_b32 s5, s1 +; GCN-IR-NEXT: s_mov_b32 s4, s0 ; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 ; GCN-IR-NEXT: s_ashr_i64 s[2:3], s[2:3], 39 -; GCN-IR-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-IR-NEXT: s_add_i32 s2, s2, s3 +; GCN-IR-NEXT: s_abs_i32 s3, s2 +; GCN-IR-NEXT: s_ashr_i32 s0, s2, 31 ; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 -; GCN-IR-NEXT: s_xor_b32 s2, s2, s3 ; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-IR-NEXT: v_mul_hi_u32 v0, s2, v0 -; GCN-IR-NEXT: v_readfirstlane_b32 s0, v0 -; GCN-IR-NEXT: s_mul_i32 s0, s0, s8 -; GCN-IR-NEXT: s_sub_i32 s0, s2, s0 -; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 -; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 -; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 -; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 -; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 -; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 -; GCN-IR-NEXT: s_xor_b32 s0, s0, s3 -; GCN-IR-NEXT: s_sub_i32 s0, s0, s3 +; GCN-IR-NEXT: v_mul_hi_u32 v0, s3, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-IR-NEXT: s_mul_i32 s1, s1, s8 +; GCN-IR-NEXT: s_sub_i32 s1, s3, s1 +; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 +; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 +; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 +; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 ; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 @@ -750,37 +744,34 @@ define amdgpu_kernel void @s_test_srem31_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_ashr_i64 s[2:3], s[2:3], 33 -; GCN-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-NEXT: s_add_i32 s2, s2, s3 -; GCN-NEXT: s_xor_b32 s8, s2, s3 +; GCN-NEXT: s_abs_i32 s8, s2 ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 ; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_sub_i32 s2, 0, s8 ; GCN-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GCN-NEXT: s_mov_b32 s4, s0 ; GCN-NEXT: s_mov_b32 s5, s1 +; GCN-NEXT: s_mov_b32 s4, s0 ; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 ; GCN-NEXT: s_ashr_i64 s[2:3], s[2:3], 33 -; GCN-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-NEXT: s_add_i32 s2, s2, s3 +; GCN-NEXT: s_abs_i32 s3, s2 +; GCN-NEXT: s_ashr_i32 s0, s2, 31 ; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 -; GCN-NEXT: s_xor_b32 s2, s2, s3 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-NEXT: v_mul_hi_u32 v0, s2, v0 -; GCN-NEXT: v_readfirstlane_b32 s0, v0 -; GCN-NEXT: s_mul_i32 s0, s0, s8 -; GCN-NEXT: s_sub_i32 s0, s2, s0 -; GCN-NEXT: s_sub_i32 s1, s0, s8 -; GCN-NEXT: s_cmp_ge_u32 s0, s8 -; GCN-NEXT: s_cselect_b32 s0, s1, s0 -; GCN-NEXT: s_sub_i32 s1, s0, s8 -; GCN-NEXT: s_cmp_ge_u32 s0, s8 -; GCN-NEXT: s_cselect_b32 s0, s1, s0 -; GCN-NEXT: s_xor_b32 s0, s0, s3 -; GCN-NEXT: s_sub_i32 s0, s0, s3 +; GCN-NEXT: v_mul_hi_u32 v0, s3, v0 +; GCN-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-NEXT: s_mul_i32 s1, s1, s8 +; GCN-NEXT: s_sub_i32 s1, s3, s1 +; GCN-NEXT: s_sub_i32 s2, s1, s8 +; GCN-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-NEXT: s_sub_i32 s2, s1, s8 +; GCN-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-NEXT: s_xor_b32 s1, s1, s0 +; GCN-NEXT: s_sub_i32 s0, s1, s0 ; GCN-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-NEXT: v_mov_b32_e32 v0, s0 ; GCN-NEXT: v_mov_b32_e32 v1, s1 @@ -794,37 +785,34 @@ define amdgpu_kernel void @s_test_srem31_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-IR-NEXT: s_mov_b32 s6, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_ashr_i64 s[2:3], s[2:3], 33 -; GCN-IR-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-IR-NEXT: s_add_i32 s2, s2, s3 -; GCN-IR-NEXT: s_xor_b32 s8, s2, s3 +; GCN-IR-NEXT: s_abs_i32 s8, s2 ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s8 ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_sub_i32 s2, 0, s8 ; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GCN-IR-NEXT: s_mov_b32 s4, s0 ; GCN-IR-NEXT: s_mov_b32 s5, s1 +; GCN-IR-NEXT: s_mov_b32 s4, s0 ; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 ; GCN-IR-NEXT: s_ashr_i64 s[2:3], s[2:3], 33 -; GCN-IR-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-IR-NEXT: s_add_i32 s2, s2, s3 +; GCN-IR-NEXT: s_abs_i32 s3, s2 +; GCN-IR-NEXT: s_ashr_i32 s0, s2, 31 ; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 -; GCN-IR-NEXT: s_xor_b32 s2, s2, s3 ; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-IR-NEXT: v_mul_hi_u32 v0, s2, v0 -; GCN-IR-NEXT: v_readfirstlane_b32 s0, v0 -; GCN-IR-NEXT: s_mul_i32 s0, s0, s8 -; GCN-IR-NEXT: s_sub_i32 s0, s2, s0 -; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 -; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 -; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 -; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 -; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 -; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 -; GCN-IR-NEXT: s_xor_b32 s0, s0, s3 -; GCN-IR-NEXT: s_sub_i32 s0, s0, s3 +; GCN-IR-NEXT: v_mul_hi_u32 v0, s3, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-IR-NEXT: s_mul_i32 s1, s1, s8 +; GCN-IR-NEXT: s_sub_i32 s1, s3, s1 +; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 +; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 +; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 +; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 ; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 @@ -845,36 +833,33 @@ define amdgpu_kernel void @s_test_srem32_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-NEXT: s_add_i32 s2, s2, s3 -; GCN-NEXT: s_xor_b32 s8, s2, s3 +; GCN-NEXT: s_abs_i32 s8, s2 ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 -; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 -; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_sub_i32 s2, 0, s8 ; GCN-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GCN-NEXT: s_mov_b32 s4, s0 -; GCN-NEXT: s_mov_b32 s5, s1 ; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 -; GCN-NEXT: s_ashr_i32 s2, s3, 31 -; GCN-NEXT: s_add_i32 s3, s3, s2 -; GCN-NEXT: s_xor_b32 s3, s3, s2 +; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_abs_i32 s2, s3 +; GCN-NEXT: s_mov_b32 s5, s1 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-NEXT: v_mul_hi_u32 v0, s3, v0 -; GCN-NEXT: v_readfirstlane_b32 s0, v0 -; GCN-NEXT: s_mul_i32 s0, s0, s8 -; GCN-NEXT: s_sub_i32 s0, s3, s0 -; GCN-NEXT: s_sub_i32 s1, s0, s8 -; GCN-NEXT: s_cmp_ge_u32 s0, s8 -; GCN-NEXT: s_cselect_b32 s0, s1, s0 -; GCN-NEXT: s_sub_i32 s1, s0, s8 -; GCN-NEXT: s_cmp_ge_u32 s0, s8 -; GCN-NEXT: s_cselect_b32 s0, s1, s0 -; GCN-NEXT: s_xor_b32 s0, s0, s2 -; GCN-NEXT: s_sub_i32 s0, s0, s2 +; GCN-NEXT: v_mul_hi_u32 v0, s2, v0 +; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: s_ashr_i32 s0, s3, 31 +; GCN-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-NEXT: s_mul_i32 s1, s1, s8 +; GCN-NEXT: s_sub_i32 s1, s2, s1 +; GCN-NEXT: s_sub_i32 s2, s1, s8 +; GCN-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-NEXT: s_sub_i32 s2, s1, s8 +; GCN-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-NEXT: s_xor_b32 s1, s1, s0 +; GCN-NEXT: s_sub_i32 s0, s1, s0 ; GCN-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-NEXT: v_mov_b32_e32 v0, s0 ; GCN-NEXT: v_mov_b32_e32 v1, s1 @@ -887,36 +872,33 @@ define amdgpu_kernel void @s_test_srem32_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s6, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_ashr_i32 s3, s2, 31 -; GCN-IR-NEXT: s_add_i32 s2, s2, s3 -; GCN-IR-NEXT: s_xor_b32 s8, s2, s3 +; GCN-IR-NEXT: s_abs_i32 s8, s2 ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s8 -; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 -; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_sub_i32 s2, 0, s8 ; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GCN-IR-NEXT: s_mov_b32 s4, s0 -; GCN-IR-NEXT: s_mov_b32 s5, s1 ; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 -; GCN-IR-NEXT: s_ashr_i32 s2, s3, 31 -; GCN-IR-NEXT: s_add_i32 s3, s3, s2 -; GCN-IR-NEXT: s_xor_b32 s3, s3, s2 +; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +; GCN-IR-NEXT: s_abs_i32 s2, s3 +; GCN-IR-NEXT: s_mov_b32 s5, s1 ; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-IR-NEXT: v_mul_hi_u32 v0, s3, v0 -; GCN-IR-NEXT: v_readfirstlane_b32 s0, v0 -; GCN-IR-NEXT: s_mul_i32 s0, s0, s8 -; GCN-IR-NEXT: s_sub_i32 s0, s3, s0 -; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 -; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 -; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 -; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 -; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 -; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 -; GCN-IR-NEXT: s_xor_b32 s0, s0, s2 -; GCN-IR-NEXT: s_sub_i32 s0, s0, s2 +; GCN-IR-NEXT: v_mul_hi_u32 v0, s2, v0 +; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: s_ashr_i32 s0, s3, 31 +; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-IR-NEXT: s_mul_i32 s1, s1, s8 +; GCN-IR-NEXT: s_sub_i32 s1, s2, s1 +; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 +; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 +; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 +; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 ; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 @@ -1074,19 +1056,19 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-IR-NEXT: s_ashr_i64 s[2:3], s[6:7], 31 ; GCN-IR-NEXT: s_ashr_i64 s[6:7], s[0:1], 31 ; GCN-IR-NEXT: s_ashr_i32 s0, s3, 31 -; GCN-IR-NEXT: s_ashr_i32 s10, s7, 31 ; GCN-IR-NEXT: s_mov_b32 s1, s0 -; GCN-IR-NEXT: s_mov_b32 s11, s10 ; GCN-IR-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1] -; GCN-IR-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11] ; GCN-IR-NEXT: s_sub_u32 s2, s2, s0 ; GCN-IR-NEXT: s_subb_u32 s3, s3, s0 +; GCN-IR-NEXT: s_ashr_i32 s10, s7, 31 +; GCN-IR-NEXT: s_mov_b32 s11, s10 +; GCN-IR-NEXT: s_xor_b64 s[6:7], s[6:7], s[10:11] ; GCN-IR-NEXT: s_sub_u32 s8, s6, s10 ; GCN-IR-NEXT: s_subb_u32 s9, s7, s10 +; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[14:15], s[2:3], 0 ; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[6:7], s[8:9], 0 -; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[10:11], s[2:3], 0 ; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[8:9] -; GCN-IR-NEXT: s_or_b64 s[10:11], s[6:7], s[10:11] +; GCN-IR-NEXT: s_or_b64 s[10:11], s[6:7], s[14:15] ; GCN-IR-NEXT: s_flbit_i32_b64 s20, s[2:3] ; GCN-IR-NEXT: s_sub_u32 s14, s12, s20 ; GCN-IR-NEXT: s_subb_u32 s15, 0, 0 @@ -1215,17 +1197,17 @@ define amdgpu_kernel void @s_test_srem24_48(ptr addrspace(1) %out, i48 %x, i48 % ; GCN-IR-NEXT: s_ashr_i64 s[2:3], s[4:5], 24 ; GCN-IR-NEXT: s_ashr_i64 s[4:5], s[6:7], 24 ; GCN-IR-NEXT: s_lshl_b64 s[2:3], s[2:3], 16 -; GCN-IR-NEXT: s_lshl_b64 s[4:5], s[4:5], 16 -; GCN-IR-NEXT: s_ashr_i64 s[6:7], s[2:3], 16 +; GCN-IR-NEXT: s_lshl_b64 s[6:7], s[4:5], 16 +; GCN-IR-NEXT: s_ashr_i64 s[4:5], s[2:3], 16 ; GCN-IR-NEXT: s_ashr_i32 s2, s3, 31 -; GCN-IR-NEXT: s_ashr_i32 s10, s5, 31 -; GCN-IR-NEXT: s_ashr_i64 s[8:9], s[4:5], 16 ; GCN-IR-NEXT: s_mov_b32 s3, s2 -; GCN-IR-NEXT: s_mov_b32 s11, s10 -; GCN-IR-NEXT: s_xor_b64 s[4:5], s[6:7], s[2:3] -; GCN-IR-NEXT: s_xor_b64 s[6:7], s[8:9], s[10:11] +; GCN-IR-NEXT: s_ashr_i64 s[8:9], s[6:7], 16 +; GCN-IR-NEXT: s_xor_b64 s[4:5], s[4:5], s[2:3] ; GCN-IR-NEXT: s_sub_u32 s4, s4, s2 ; GCN-IR-NEXT: s_subb_u32 s5, s5, s2 +; GCN-IR-NEXT: s_ashr_i32 s10, s7, 31 +; GCN-IR-NEXT: s_mov_b32 s11, s10 +; GCN-IR-NEXT: s_xor_b64 s[6:7], s[8:9], s[10:11] ; GCN-IR-NEXT: s_sub_u32 s6, s6, s10 ; GCN-IR-NEXT: s_subb_u32 s7, s7, s10 ; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[8:9], s[6:7], 0 diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll index 33cc8e96f663..1c303de55c95 100644 --- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll +++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll @@ -177,56 +177,55 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: pushl %ebx ; X86-NEXT: pushl %edi ; X86-NEXT: pushl %esi -; X86-NEXT: subl $152, %esp -; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: subl $156, %esp ; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: movl %ecx, %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, %eax ; X86-NEXT: sarl $31, %eax -; X86-NEXT: movl %edx, %ebx -; X86-NEXT: sarl $31, %ebx -; X86-NEXT: movl %eax, %esi -; X86-NEXT: xorl %ecx, %esi -; X86-NEXT: movl %esi, %ebp -; X86-NEXT: movl %eax, %ecx -; X86-NEXT: xorl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: movl %ecx, %edi -; X86-NEXT: movl %eax, %ecx -; X86-NEXT: xorl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: movl %eax, %esi -; X86-NEXT: xorl {{[0-9]+}}(%esp), %esi -; X86-NEXT: subl %eax, %esi -; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: sbbl %eax, %ecx +; X86-NEXT: xorl %eax, %esi +; X86-NEXT: movl %esi, %edi +; X86-NEXT: xorl %eax, %edx +; X86-NEXT: movl %edx, %esi +; X86-NEXT: movl %ecx, %edx +; X86-NEXT: xorl %eax, %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: xorl %eax, %ecx +; X86-NEXT: subl %eax, %ecx ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: sbbl %eax, %edx +; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: sbbl %eax, %esi +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: sbbl %eax, %edi ; X86-NEXT: movl %edi, (%esp) # 4-byte Spill -; X86-NEXT: sbbl %eax, %ebp -; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %ebx, %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl %ecx, %edx +; X86-NEXT: sarl $31, %edx +; X86-NEXT: movl %ecx, %esi ; X86-NEXT: xorl %edx, %esi -; X86-NEXT: movl %ebx, %edx -; X86-NEXT: xorl {{[0-9]+}}(%esp), %edx -; X86-NEXT: movl %ebx, %edi -; X86-NEXT: xorl {{[0-9]+}}(%esp), %edi -; X86-NEXT: movl %ebx, %ebp -; X86-NEXT: xorl {{[0-9]+}}(%esp), %ebp -; X86-NEXT: subl %ebx, %ebp -; X86-NEXT: sbbl %ebx, %edi -; X86-NEXT: sbbl %ebx, %edx -; X86-NEXT: sbbl %ebx, %esi -; X86-NEXT: xorl %eax, %ebx -; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %edi, %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx +; X86-NEXT: xorl %edx, %ebx +; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp +; X86-NEXT: xorl %edx, %ebp +; X86-NEXT: movl {{[0-9]+}}(%esp), %edi +; X86-NEXT: xorl %edx, %edi +; X86-NEXT: subl %edx, %edi +; X86-NEXT: sbbl %edx, %ebp +; X86-NEXT: sbbl %edx, %ebx +; X86-NEXT: sbbl %edx, %esi +; X86-NEXT: xorl %eax, %edx +; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %ebp, %eax ; X86-NEXT: orl %esi, %eax -; X86-NEXT: movl %ebp, %ecx -; X86-NEXT: orl %edx, %ecx -; X86-NEXT: movl %edx, %ebx +; X86-NEXT: movl %edi, %ecx +; X86-NEXT: orl %ebx, %ecx ; X86-NEXT: orl %eax, %ecx ; X86-NEXT: sete %cl ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload +; X86-NEXT: orl (%esp), %eax # 4-byte Folded Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X86-NEXT: orl (%esp), %edx # 4-byte Folded Reload +; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload ; X86-NEXT: orl %eax, %edx ; X86-NEXT: sete %al ; X86-NEXT: orb %cl, %al @@ -238,76 +237,78 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: addl $32, %ecx ; X86-NEXT: testl %esi, %esi ; X86-NEXT: cmovnel %edx, %ecx -; X86-NEXT: bsrl %edi, %edx +; X86-NEXT: bsrl %ebp, %edx ; X86-NEXT: xorl $31, %edx -; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: bsrl %ebp, %ebp -; X86-NEXT: xorl $31, %ebp -; X86-NEXT: addl $32, %ebp ; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: testl %edi, %edi -; X86-NEXT: cmovnel %edx, %ebp -; X86-NEXT: addl $64, %ebp +; X86-NEXT: bsrl %edi, %edi +; X86-NEXT: xorl $31, %edi +; X86-NEXT: addl $32, %edi +; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: testl %ebp, %ebp +; X86-NEXT: cmovnel %edx, %edi +; X86-NEXT: addl $64, %edi ; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %ebx, %edx ; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: orl %esi, %ebx -; X86-NEXT: cmovnel %ecx, %ebp -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload +; X86-NEXT: orl %esi, %edx +; X86-NEXT: cmovnel %ecx, %edi +; X86-NEXT: movl (%esp), %ebx # 4-byte Reload ; X86-NEXT: bsrl %ebx, %edx ; X86-NEXT: xorl $31, %edx -; X86-NEXT: movl (%esp), %eax # 4-byte Reload -; X86-NEXT: bsrl %eax, %ecx +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload +; X86-NEXT: bsrl %ebp, %ecx ; X86-NEXT: xorl $31, %ecx ; X86-NEXT: addl $32, %ecx ; X86-NEXT: testl %ebx, %ebx ; X86-NEXT: cmovnel %edx, %ecx -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload -; X86-NEXT: bsrl %edi, %esi +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload +; X86-NEXT: bsrl %eax, %esi ; X86-NEXT: xorl $31, %esi ; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload ; X86-NEXT: xorl $31, %edx ; X86-NEXT: addl $32, %edx -; X86-NEXT: testl %edi, %edi +; X86-NEXT: testl %eax, %eax ; X86-NEXT: cmovnel %esi, %edx ; X86-NEXT: addl $64, %edx -; X86-NEXT: movl %eax, %esi +; X86-NEXT: movl %ebp, %esi ; X86-NEXT: orl %ebx, %esi ; X86-NEXT: cmovnel %ecx, %edx -; X86-NEXT: xorl %esi, %esi -; X86-NEXT: subl %edx, %ebp -; X86-NEXT: movl $0, %edi -; X86-NEXT: sbbl %edi, %edi +; X86-NEXT: xorl %ebx, %ebx +; X86-NEXT: subl %edx, %edi ; X86-NEXT: movl $0, %edx ; X86-NEXT: sbbl %edx, %edx ; X86-NEXT: movl $0, %eax ; X86-NEXT: sbbl %eax, %eax +; X86-NEXT: movl $0, %esi +; X86-NEXT: sbbl %esi, %esi ; X86-NEXT: movl $127, %ecx -; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: cmpl %ebp, %ecx -; X86-NEXT: movl %edx, %ebp -; X86-NEXT: movl $0, %ecx -; X86-NEXT: sbbl %edi, %ecx +; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: cmpl %edi, %ecx ; X86-NEXT: movl $0, %ecx +; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: sbbl %edx, %ecx ; X86-NEXT: movl $0, %ecx ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: sbbl %eax, %ecx +; X86-NEXT: movl $0, %ecx +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: sbbl %esi, %ecx ; X86-NEXT: setb %cl ; X86-NEXT: orb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Folded Reload -; X86-NEXT: movl %ebx, %edx -; X86-NEXT: cmovnel %esi, %edx -; X86-NEXT: movl (%esp), %ebx # 4-byte Reload -; X86-NEXT: cmovnel %esi, %ebx +; X86-NEXT: movl (%esp), %edx # 4-byte Reload +; X86-NEXT: cmovnel %ebx, %edx +; X86-NEXT: cmovnel %ebx, %ebp ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: cmovnel %esi, %eax -; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload +; X86-NEXT: cmovnel %ebx, %eax +; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload +; X86-NEXT: movl %ebx, %esi ; X86-NEXT: jne .LBB4_8 ; X86-NEXT: # %bb.1: # %_udiv-special-cases -; X86-NEXT: movl %edi, %ecx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload ; X86-NEXT: xorl $127, %edi -; X86-NEXT: orl %ebp, %edi -; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload +; X86-NEXT: movl %ebx, %ecx ; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload ; X86-NEXT: orl %edi, %ecx ; X86-NEXT: je .LBB4_8 @@ -316,10 +317,10 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) -; X86-NEXT: movl (%esp), %eax # 4-byte Reload -; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) +; X86-NEXT: movl (%esp), %eax # 4-byte Reload +; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) @@ -333,27 +334,27 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: andb $15, %al ; X86-NEXT: negb %al ; X86-NEXT: movsbl %al, %edi -; X86-NEXT: movl 144(%esp,%edi), %edx -; X86-NEXT: movl 148(%esp,%edi), %esi +; X86-NEXT: movl 148(%esp,%edi), %edx +; X86-NEXT: movl 152(%esp,%edi), %esi ; X86-NEXT: movb %ch, %cl ; X86-NEXT: shldl %cl, %edx, %esi ; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: shll %cl, %edx ; X86-NEXT: notb %cl -; X86-NEXT: movl 140(%esp,%edi), %eax -; X86-NEXT: movl %eax, %ebx -; X86-NEXT: shrl %ebx -; X86-NEXT: shrl %cl, %ebx -; X86-NEXT: orl %edx, %ebx -; X86-NEXT: movl 136(%esp,%edi), %edx +; X86-NEXT: movl 144(%esp,%edi), %eax +; X86-NEXT: movl %eax, %ebp +; X86-NEXT: shrl %ebp +; X86-NEXT: shrl %cl, %ebp +; X86-NEXT: orl %edx, %ebp +; X86-NEXT: movl 140(%esp,%edi), %edx ; X86-NEXT: movb %ch, %cl ; X86-NEXT: shldl %cl, %edx, %eax ; X86-NEXT: shll %cl, %edx ; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: addl $1, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: adcl $0, %ecx -; X86-NEXT: adcl $0, %ebp +; X86-NEXT: adcl $0, %ebx +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload +; X86-NEXT: adcl $0, %edi ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload ; X86-NEXT: adcl $0, %edx ; X86-NEXT: jae .LBB4_3 @@ -363,38 +364,37 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload ; X86-NEXT: jmp .LBB4_7 ; X86-NEXT: .LBB4_3: # %udiv-preheader -; X86-NEXT: movl %ecx, %esi -; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; X86-NEXT: movl %esi, {{[0-9]+}}(%esp) ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; X86-NEXT: movl %esi, {{[0-9]+}}(%esp) -; X86-NEXT: movl (%esp), %esi # 4-byte Reload -; X86-NEXT: movl %esi, {{[0-9]+}}(%esp) ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; X86-NEXT: movl %esi, {{[0-9]+}}(%esp) +; X86-NEXT: movl (%esp), %esi # 4-byte Reload +; X86-NEXT: movl %esi, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movb %dl, %ch ; X86-NEXT: andb $7, %ch ; X86-NEXT: movb %dl, %cl ; X86-NEXT: shrb $3, %cl ; X86-NEXT: andb $15, %cl ; X86-NEXT: movzbl %cl, %edx -; X86-NEXT: movl 100(%esp,%edx), %esi -; X86-NEXT: movl %esi, (%esp) # 4-byte Spill -; X86-NEXT: movl 96(%esp,%edx), %edi -; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl 104(%esp,%edx), %ebx +; X86-NEXT: movl 100(%esp,%edx), %edi +; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl %edi, %ebp ; X86-NEXT: movb %ch, %cl -; X86-NEXT: shrdl %cl, %esi, %ebp -; X86-NEXT: movl 88(%esp,%edx), %ebx +; X86-NEXT: shrdl %cl, %ebx, %ebp ; X86-NEXT: movl 92(%esp,%edx), %esi +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl 96(%esp,%edx), %esi ; X86-NEXT: movl %esi, %edx ; X86-NEXT: shrl %cl, %edx ; X86-NEXT: notb %cl @@ -403,9 +403,9 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: orl %edx, %edi ; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movb %ch, %cl -; X86-NEXT: shrl %cl, (%esp) # 4-byte Folded Spill -; X86-NEXT: shrdl %cl, %esi, %ebx +; X86-NEXT: shrl %cl, %ebx ; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: shrdl %cl, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: addl $-1, %ecx ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill @@ -415,7 +415,6 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: adcl $-1, %ecx ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: adcl $-1, %ecx ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill @@ -424,26 +423,26 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: .p2align 4, 0x90 ; X86-NEXT: .LBB4_4: # %udiv-do-while ; X86-NEXT: # =>This Inner Loop Header: Depth=1 -; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %ebp, %esi -; X86-NEXT: shldl $1, %ebp, (%esp) # 4-byte Folded Spill +; X86-NEXT: movl %ebp, (%esp) # 4-byte Spill +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload +; X86-NEXT: shldl $1, %ebp, %ebx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload -; X86-NEXT: shldl $1, %ebp, %esi +; X86-NEXT: shldl $1, %ebp, (%esp) # 4-byte Folded Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload ; X86-NEXT: shldl $1, %edx, %ebp ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload ; X86-NEXT: shldl $1, %edi, %edx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: shldl $1, %ecx, %edi -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload -; X86-NEXT: orl %ebx, %edi +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload +; X86-NEXT: orl %esi, %edi ; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: shldl $1, %eax, %ecx -; X86-NEXT: orl %ebx, %ecx +; X86-NEXT: orl %esi, %ecx ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: shldl $1, %ecx, %eax -; X86-NEXT: orl %ebx, %eax +; X86-NEXT: orl %esi, %eax ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: addl %ecx, %ecx ; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload @@ -452,15 +451,15 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: sbbl %ebp, %ecx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: sbbl %esi, %ecx -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: sbbl (%esp), %ecx # 4-byte Folded Reload +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload +; X86-NEXT: sbbl %ebx, %ecx ; X86-NEXT: sarl $31, %ecx ; X86-NEXT: movl %ecx, %eax ; X86-NEXT: andl $1, %eax ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %ecx, %ebx -; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload +; X86-NEXT: movl %ecx, %esi +; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload ; X86-NEXT: movl %ecx, %edi ; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload ; X86-NEXT: movl %ecx, %eax @@ -468,21 +467,22 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload ; X86-NEXT: subl %ecx, %edx ; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload ; X86-NEXT: sbbl %eax, %ebp ; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: sbbl %edi, %esi -; X86-NEXT: movl %esi, %ebp +; X86-NEXT: movl (%esp), %ebp # 4-byte Reload +; X86-NEXT: sbbl %edi, %ebp ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: sbbl %ebx, (%esp) # 4-byte Folded Spill +; X86-NEXT: sbbl %esi, %ebx +; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: addl $-1, %ecx -; X86-NEXT: adcl $-1, %edx +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload +; X86-NEXT: adcl $-1, %edi ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload ; X86-NEXT: adcl $-1, %ebx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; X86-NEXT: adcl $-1, %esi -; X86-NEXT: movl %edx, %edi +; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: orl %esi, %edi ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill @@ -491,15 +491,15 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: orl %edi, %ecx ; X86-NEXT: jne .LBB4_4 ; X86-NEXT: # %bb.5: -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload ; X86-NEXT: .LBB4_7: # %udiv-loop-exit -; X86-NEXT: shldl $1, %ebx, %edx +; X86-NEXT: shldl $1, %ebp, %edx ; X86-NEXT: orl %ecx, %edx -; X86-NEXT: shldl $1, %eax, %ebx -; X86-NEXT: orl %ecx, %ebx +; X86-NEXT: shldl $1, %eax, %ebp +; X86-NEXT: orl %ecx, %ebp ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; X86-NEXT: shldl $1, %esi, %eax ; X86-NEXT: orl %ecx, %eax @@ -508,36 +508,35 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: .LBB4_8: # %udiv-end ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: xorl %ecx, %edx -; X86-NEXT: xorl %ecx, %ebx +; X86-NEXT: xorl %ecx, %ebp ; X86-NEXT: xorl %ecx, %eax ; X86-NEXT: xorl %ecx, %esi ; X86-NEXT: subl %ecx, %esi ; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: sbbl %ecx, %eax ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: sbbl %ecx, %ebx +; X86-NEXT: sbbl %ecx, %ebp ; X86-NEXT: sbbl %ecx, %edx ; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-NEXT: movl %esi, (%ecx) ; X86-NEXT: movl %eax, 4(%ecx) -; X86-NEXT: movl %ebx, 8(%ecx) +; X86-NEXT: movl %ebp, 8(%ecx) ; X86-NEXT: movl %edx, 12(%ecx) ; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp -; X86-NEXT: movl %ebx, %edi +; X86-NEXT: movl %ebp, %edi ; X86-NEXT: mull %ecx ; X86-NEXT: movl %edx, %ebx -; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %eax, %ebp ; X86-NEXT: movl %esi, %eax ; X86-NEXT: mull %ecx ; X86-NEXT: movl %eax, (%esp) # 4-byte Spill ; X86-NEXT: movl %edx, %ecx -; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload +; X86-NEXT: addl %ebp, %ecx ; X86-NEXT: adcl $0, %ebx ; X86-NEXT: movl %esi, %eax -; X86-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NEXT: mull %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp +; X86-NEXT: mull %ebp ; X86-NEXT: addl %ecx, %eax ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: adcl %ebx, %edx @@ -545,7 +544,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: setb %cl ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; X86-NEXT: movl %esi, %eax -; X86-NEXT: mull {{[0-9]+}}(%esp) +; X86-NEXT: mull %ebp ; X86-NEXT: addl %ebx, %eax ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movzbl %cl, %eax @@ -556,35 +555,36 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: imull %eax, %ecx ; X86-NEXT: mull %edi ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: imull {{[0-9]+}}(%esp), %edi +; X86-NEXT: imull %ebp, %edi ; X86-NEXT: addl %edx, %edi ; X86-NEXT: addl %ecx, %edi ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: movl %eax, %ecx ; X86-NEXT: imull %esi, %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X86-NEXT: imull %edx, %ebp +; X86-NEXT: imull %edx, %esi ; X86-NEXT: mull %edx -; X86-NEXT: addl %edx, %ebp -; X86-NEXT: addl %ecx, %ebp +; X86-NEXT: addl %edx, %esi +; X86-NEXT: addl %ecx, %esi ; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload -; X86-NEXT: adcl %edi, %ebp +; X86-NEXT: adcl %edi, %esi ; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload -; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload -; X86-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NEXT: subl (%esp), %edx # 4-byte Folded Reload +; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload ; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload -; X86-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NEXT: sbbl %eax, %esi +; X86-NEXT: subl (%esp), %ecx # 4-byte Folded Reload +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload +; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx +; X86-NEXT: sbbl %eax, %ebx ; X86-NEXT: movl {{[0-9]+}}(%esp), %edi -; X86-NEXT: sbbl %ebp, %edi +; X86-NEXT: sbbl %esi, %edi ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: movl %edx, (%eax) -; X86-NEXT: movl %ecx, 4(%eax) -; X86-NEXT: movl %esi, 8(%eax) +; X86-NEXT: movl %ecx, (%eax) +; X86-NEXT: movl %edx, 4(%eax) +; X86-NEXT: movl %ebx, 8(%eax) ; X86-NEXT: movl %edi, 12(%eax) -; X86-NEXT: addl $152, %esp +; X86-NEXT: addl $156, %esp ; X86-NEXT: popl %esi ; X86-NEXT: popl %edi ; X86-NEXT: popl %ebx diff --git a/llvm/test/CodeGen/X86/pr38539.ll b/llvm/test/CodeGen/X86/pr38539.ll index ace78b38d53e..fbc363f77ec4 100644 --- a/llvm/test/CodeGen/X86/pr38539.ll +++ b/llvm/test/CodeGen/X86/pr38539.ll @@ -36,9 +36,9 @@ define void @f() nounwind { ; X86-NEXT: movl %eax, %ecx ; X86-NEXT: sarl $30, %ecx ; X86-NEXT: sarl $31, %eax -; X86-NEXT: shrdl $1, %eax, %ecx ; X86-NEXT: xorl %eax, %edx ; X86-NEXT: xorl %eax, %edi +; X86-NEXT: shrdl $1, %eax, %ecx ; X86-NEXT: xorl %ecx, %esi ; X86-NEXT: subl %ecx, %esi ; X86-NEXT: sbbl %eax, %edi -- GitLab From fd45dcca26d6031fcbaa907d8d6c0d9755b36699 Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Fri, 7 Jun 2024 19:03:07 +0900 Subject: [PATCH 227/462] fix(mlir/**.py): fix comparison to None (#94019) from PEP8 (https://peps.python.org/pep-0008/#programming-recommendations): > Comparisons to singletons like None should always be done with is or is not, never the equality operators. Co-authored-by: Eisuke Kawashima --- mlir/test/python/ir/affine_expr.py | 2 +- mlir/test/python/ir/attributes.py | 8 ++++---- mlir/test/python/ir/builtin_types.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mlir/test/python/ir/affine_expr.py b/mlir/test/python/ir/affine_expr.py index 63564303e831..c7861c1acfe1 100644 --- a/mlir/test/python/ir/affine_expr.py +++ b/mlir/test/python/ir/affine_expr.py @@ -42,7 +42,7 @@ def testAffineExprEq(): # CHECK: False print(a1 == a3) # CHECK: False - print(a1 == None) + print(a1 is None) # CHECK: False print(a1 == "foo") diff --git a/mlir/test/python/ir/attributes.py b/mlir/test/python/ir/attributes.py index dbd6bad05e01..0f2c8e7b7252 100644 --- a/mlir/test/python/ir/attributes.py +++ b/mlir/test/python/ir/attributes.py @@ -56,8 +56,8 @@ def testAttrEq(): print("a1 == a2:", a1 == a2) # CHECK: a1 == a3: True print("a1 == a3:", a1 == a3) - # CHECK: a1 == None: False - print("a1 == None:", a1 == None) + # CHECK: a1 is None: False + print("a1 is None:", a1 is None) # CHECK-LABEL: TEST: testAttrHash @@ -109,9 +109,9 @@ def testAttrEqDoesNotRaise(): # CHECK: False print(a1 == not_an_attr) # CHECK: False - print(a1 == None) + print(a1 is None) # CHECK: True - print(a1 != None) + print(a1 is not None) # CHECK-LABEL: TEST: testAttrCapsule diff --git a/mlir/test/python/ir/builtin_types.py b/mlir/test/python/ir/builtin_types.py index 4eea1a9c372e..cfe377c70371 100644 --- a/mlir/test/python/ir/builtin_types.py +++ b/mlir/test/python/ir/builtin_types.py @@ -57,8 +57,8 @@ def testTypeEq(): print("t1 == t2:", t1 == t2) # CHECK: t1 == t3: True print("t1 == t3:", t1 == t3) - # CHECK: t1 == None: False - print("t1 == None:", t1 == None) + # CHECK: t1 is None: False + print("t1 is None:", t1 is None) # CHECK-LABEL: TEST: testTypeHash @@ -143,9 +143,9 @@ def testTypeEqDoesNotRaise(): # CHECK: False print(t1 == not_a_type) # CHECK: False - print(t1 == None) + print(t1 is None) # CHECK: True - print(t1 != None) + print(t1 is not None) # CHECK-LABEL: TEST: testTypeCapsule -- GitLab From 917afa883258757575ac6448e83a9233d7877333 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray Date: Fri, 7 Jun 2024 11:03:32 +0100 Subject: [PATCH 228/462] [ARM] Add support for Cortex-R52+ (#94633) Cortex-R52+ is an Armv8-R AArch32 CPU. Technical Reference Manual for Cortex-R52+: https://developer.arm.com/documentation/102199/latest/ --- clang/docs/ReleaseNotes.rst | 1 + clang/test/Misc/target-invalid-cpu-note.c | 2 +- llvm/docs/ReleaseNotes.rst | 1 + llvm/include/llvm/TargetParser/ARMTargetParser.def | 1 + llvm/lib/Target/ARM/ARMProcessors.td | 8 ++++++++ llvm/lib/Target/ARM/ARMSubtarget.cpp | 1 + llvm/lib/TargetParser/Host.cpp | 1 + llvm/test/CodeGen/ARM/build-attributes.ll | 4 ++++ llvm/test/CodeGen/ARM/cortexr52-misched-basic.ll | 1 + llvm/test/CodeGen/ARM/lsr-scale-addr-mode.ll | 1 + llvm/test/CodeGen/ARM/misched-fp-basic.ll | 2 ++ llvm/test/CodeGen/ARM/misched-int-basic-thumb2.mir | 2 ++ llvm/test/CodeGen/ARM/misched-int-basic.mir | 2 ++ llvm/test/CodeGen/ARM/proc-resource-sched.ll | 1 + llvm/test/CodeGen/ARM/single-issue-r52.mir | 2 ++ llvm/test/CodeGen/ARM/useaa.ll | 1 + llvm/test/MC/ARM/dfb-neg.s | 2 ++ llvm/test/MC/ARM/dfb.s | 2 ++ llvm/test/MC/ARM/invalid-armv8r.s | 6 ++++++ llvm/test/MC/ARM/thumb-hints.s | 2 ++ llvm/test/MC/Disassembler/ARM/dfb-thumb.txt | 2 ++ llvm/unittests/TargetParser/TargetParserTest.cpp | 9 ++++++++- 22 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 llvm/test/MC/ARM/invalid-armv8r.s diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index b9c9070fcb22..cf1ba02cbc4b 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -907,6 +907,7 @@ Arm and AArch64 Support * Arm Cortex-A520AE (cortex-a520ae). * Arm Cortex-A720AE (cortex-a720ae). * Arm Cortex-R82AE (cortex-r82ae). + * Arm Cortex-R52+ (cortex-r52plus). * Arm Neoverse-N3 (neoverse-n3). * Arm Neoverse-V3 (neoverse-v3). * Arm Neoverse-V3AE (neoverse-v3ae). diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c index cb5b6752850c..2439025609b9 100644 --- a/clang/test/Misc/target-invalid-cpu-note.c +++ b/clang/test/Misc/target-invalid-cpu-note.c @@ -1,7 +1,7 @@ // Use CHECK-NEXT instead of multiple CHECK-SAME to ensure we will fail if there is anything extra in the output. // RUN: not %clang_cc1 -triple armv5--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix ARM // ARM: error: unknown target CPU 'not-a-cpu' -// ARM-NEXT: note: valid target CPU values are: arm8, arm810, strongarm, strongarm110, strongarm1100, strongarm1110, arm7tdmi, arm7tdmi-s, arm710t, arm720t, arm9, arm9tdmi, arm920, arm920t, arm922t, arm940t, ep9312, arm10tdmi, arm1020t, arm9e, arm946e-s, arm966e-s, arm968e-s, arm10e, arm1020e, arm1022e, arm926ej-s, arm1136j-s, arm1136jf-s, mpcore, mpcorenovfp, arm1176jz-s, arm1176jzf-s, arm1156t2-s, arm1156t2f-s, cortex-m0, cortex-m0plus, cortex-m1, sc000, cortex-a5, cortex-a7, cortex-a8, cortex-a9, cortex-a12, cortex-a15, cortex-a17, krait, cortex-r4, cortex-r4f, cortex-r5, cortex-r7, cortex-r8, cortex-r52, sc300, cortex-m3, cortex-m4, cortex-m7, cortex-m23, cortex-m33, cortex-m35p, cortex-m55, cortex-m85, cortex-m52, cortex-a32, cortex-a35, cortex-a53, cortex-a55, cortex-a57, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-x1, cortex-x1c, neoverse-n1, neoverse-n2, neoverse-v1, cyclone, exynos-m3, exynos-m4, exynos-m5, kryo, iwmmxt, xscale, swift{{$}} +// ARM-NEXT: note: valid target CPU values are: arm8, arm810, strongarm, strongarm110, strongarm1100, strongarm1110, arm7tdmi, arm7tdmi-s, arm710t, arm720t, arm9, arm9tdmi, arm920, arm920t, arm922t, arm940t, ep9312, arm10tdmi, arm1020t, arm9e, arm946e-s, arm966e-s, arm968e-s, arm10e, arm1020e, arm1022e, arm926ej-s, arm1136j-s, arm1136jf-s, mpcore, mpcorenovfp, arm1176jz-s, arm1176jzf-s, arm1156t2-s, arm1156t2f-s, cortex-m0, cortex-m0plus, cortex-m1, sc000, cortex-a5, cortex-a7, cortex-a8, cortex-a9, cortex-a12, cortex-a15, cortex-a17, krait, cortex-r4, cortex-r4f, cortex-r5, cortex-r7, cortex-r8, cortex-r52, cortex-r52plus, sc300, cortex-m3, cortex-m4, cortex-m7, cortex-m23, cortex-m33, cortex-m35p, cortex-m55, cortex-m85, cortex-m52, cortex-a32, cortex-a35, cortex-a53, cortex-a55, cortex-a57, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-x1, cortex-x1c, neoverse-n1, neoverse-n2, neoverse-v1, cyclone, exynos-m3, exynos-m4, exynos-m5, kryo, iwmmxt, xscale, swift{{$}} // RUN: not %clang_cc1 -triple arm64--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix AARCH64 // AARCH64: error: unknown target CPU 'not-a-cpu' diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst index c4a85620fc3e..28dd567a6108 100644 --- a/llvm/docs/ReleaseNotes.rst +++ b/llvm/docs/ReleaseNotes.rst @@ -101,6 +101,7 @@ Changes to the AMDGPU Backend Changes to the ARM Backend -------------------------- +* Added support for Cortex-R52+ CPU. * FEAT_F32MM is no longer activated by default when using `+sve` on v8.6-A or greater. The feature is still available and can be used by adding `+f32mm` to the command line options. * armv8-r now implies only fp-armv8d16sp, rather than neon and full fp-armv8. These features are still included by default for cortex-r52. The default cpu for armv8-r is now "generic", for compatibility with variants that do not include neon, fp64, and d32. diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.def b/llvm/include/llvm/TargetParser/ARMTargetParser.def index d7b77a6ef5b6..e5a1ce54fd46 100644 --- a/llvm/include/llvm/TargetParser/ARMTargetParser.def +++ b/llvm/include/llvm/TargetParser/ARMTargetParser.def @@ -330,6 +330,7 @@ ARM_CPU_NAME("cortex-r7", ARMV7R, FK_VFPV3_D16_FP16, false, ARM_CPU_NAME("cortex-r8", ARMV7R, FK_VFPV3_D16_FP16, false, (ARM::AEK_MP | ARM::AEK_HWDIVARM)) ARM_CPU_NAME("cortex-r52", ARMV8R, FK_NEON_FP_ARMV8, false, ARM::AEK_NONE) +ARM_CPU_NAME("cortex-r52plus", ARMV8R, FK_NEON_FP_ARMV8, false, ARM::AEK_NONE) ARM_CPU_NAME("sc300", ARMV7M, FK_NONE, false, ARM::AEK_NONE) ARM_CPU_NAME("cortex-m3", ARMV7M, FK_NONE, true, ARM::AEK_NONE) ARM_CPU_NAME("cortex-m4", ARMV7EM, FK_FPV4_SP_D16, true, ARM::AEK_NONE) diff --git a/llvm/lib/Target/ARM/ARMProcessors.td b/llvm/lib/Target/ARM/ARMProcessors.td index eb5ed41ae8a1..e4e122a0d133 100644 --- a/llvm/lib/Target/ARM/ARMProcessors.td +++ b/llvm/lib/Target/ARM/ARMProcessors.td @@ -90,6 +90,8 @@ def ProcR7 : SubtargetFeature<"r7", "ARMProcFamily", "CortexR7", "Cortex-R7 ARM processors", []>; def ProcR52 : SubtargetFeature<"r52", "ARMProcFamily", "CortexR52", "Cortex-R52 ARM processors", []>; +def ProcR52plus : SubtargetFeature<"r52plus", "ARMProcFamily", "CortexR52plus", + "Cortex-R52plus ARM processors", []>; def ProcM3 : SubtargetFeature<"m3", "ARMProcFamily", "CortexM3", "Cortex-M3 ARM processors", []>; @@ -577,3 +579,9 @@ def : ProcessorModel<"cortex-r52", CortexR52Model, [ARMv8r, ProcR52, FeatureNEON, FeatureUseMISched, FeatureFPAO]>; + +def : ProcessorModel<"cortex-r52plus", CortexR52Model, [ARMv8r, ProcR52plus, + FeatureFPARMv8, + FeatureNEON, + FeatureUseMISched, + FeatureFPAO]>; diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp index 5e13d8fabe04..a8c6cd99633f 100644 --- a/llvm/lib/Target/ARM/ARMSubtarget.cpp +++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp @@ -298,6 +298,7 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) { case CortexM3: case CortexM7: case CortexR52: + case CortexR52plus: case CortexX1: case CortexX1C: break; diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp index b3b8486c604b..369287e2d203 100644 --- a/llvm/lib/TargetParser/Host.cpp +++ b/llvm/lib/TargetParser/Host.cpp @@ -213,6 +213,7 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) { .Case("0xd23", "cortex-m85") .Case("0xc18", "cortex-r8") .Case("0xd13", "cortex-r52") + .Case("0xd16", "cortex-r52plus") .Case("0xd15", "cortex-r82") .Case("0xd14", "cortex-r82ae") .Case("0xd02", "cortex-a34") diff --git a/llvm/test/CodeGen/ARM/build-attributes.ll b/llvm/test/CodeGen/ARM/build-attributes.ll index e8c83b92f3f9..9220f73d94f8 100644 --- a/llvm/test/CodeGen/ARM/build-attributes.ll +++ b/llvm/test/CodeGen/ARM/build-attributes.ll @@ -224,6 +224,10 @@ ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52 -mattr=-neon,-fp64,-d32 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-SP ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-NEON +; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52plus -mattr=-vfp2sp,-fp16 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-NOFPU +; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52plus -mattr=-neon,-fp64,-d32 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-SP +; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52plus | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-NEON + ; ARMv8-M ; RUN: llc < %s -mtriple=thumbv8-none-none-eabi -mcpu=cortex-m23 | FileCheck %s --check-prefix=STRICT-ALIGN ; RUN: llc < %s -mtriple=thumbv8-none-none-eabi -mcpu=cortex-m33 | FileCheck %s --check-prefix=NO-STRICT-ALIGN diff --git a/llvm/test/CodeGen/ARM/cortexr52-misched-basic.ll b/llvm/test/CodeGen/ARM/cortexr52-misched-basic.ll index c6bff0c9d5ad..0cefecd66c71 100644 --- a/llvm/test/CodeGen/ARM/cortexr52-misched-basic.ll +++ b/llvm/test/CodeGen/ARM/cortexr52-misched-basic.ll @@ -1,5 +1,6 @@ ; REQUIRES: asserts ; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-r52 -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=R52_SCHED +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-r52plus -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=R52_SCHED ; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=generic -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC ; ; Check the latency for instructions for both generic and cortex-r52. diff --git a/llvm/test/CodeGen/ARM/lsr-scale-addr-mode.ll b/llvm/test/CodeGen/ARM/lsr-scale-addr-mode.ll index 0c519f6285dc..ea84a3762ec8 100644 --- a/llvm/test/CodeGen/ARM/lsr-scale-addr-mode.ll +++ b/llvm/test/CodeGen/ARM/lsr-scale-addr-mode.ll @@ -4,6 +4,7 @@ ; RUN: llc -mtriple=arm-eabi -mcpu=cortex-a53 %s -o - | FileCheck %s -check-prefix CHECK-NONEGOFF ; RUN: llc -mtriple=arm-eabi -mcpu=cortex-a57 %s -o - | FileCheck %s -check-prefix CHECK-NONEGOFF ; RUN: llc -mtriple=arm-eabi -mcpu=cortex-r52 %s -o - | FileCheck %s -check-prefix CHECK-NONEGOFF +; RUN: llc -mtriple=arm-eabi -mcpu=cortex-r52plus %s -o - | FileCheck %s -check-prefix CHECK-NONEGOFF ; Should not generate negated register offset define void @sintzero(ptr %a) nounwind { diff --git a/llvm/test/CodeGen/ARM/misched-fp-basic.ll b/llvm/test/CodeGen/ARM/misched-fp-basic.ll index 2f672b0cb540..eabede386f1f 100644 --- a/llvm/test/CodeGen/ARM/misched-fp-basic.ll +++ b/llvm/test/CodeGen/ARM/misched-fp-basic.ll @@ -5,6 +5,8 @@ ; RUN: /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_SWIFT ; RUN: llc < %s -mtriple=arm-eabi -mcpu=cortex-r52 -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > \ ; RUN: /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_R52 +; RUN: llc < %s -mtriple=arm-eabi -mcpu=cortex-r52plus -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > \ +; RUN: /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_R52 ; ; Check the latency of instructions for processors with sched-models ; diff --git a/llvm/test/CodeGen/ARM/misched-int-basic-thumb2.mir b/llvm/test/CodeGen/ARM/misched-int-basic-thumb2.mir index be5340bd51ab..0cd20a9f1ff5 100644 --- a/llvm/test/CodeGen/ARM/misched-int-basic-thumb2.mir +++ b/llvm/test/CodeGen/ARM/misched-int-basic-thumb2.mir @@ -5,6 +5,8 @@ # RUN: -debug-only=machine-scheduler 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_A9 # RUN: llc -o /dev/null %s -mtriple=thumbv8r-eabi -mcpu=cortex-r52 -run-pass machine-scheduler -enable-misched -verify-misched \ # RUN: -debug-only=machine-scheduler 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_R52 +# RUN: llc -o /dev/null %s -mtriple=thumbv8r-eabi -mcpu=cortex-r52plus -run-pass machine-scheduler -enable-misched -verify-misched \ +# RUN: -debug-only=machine-scheduler 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_R52 # REQUIRES: asserts --- | ; ModuleID = 'foo.ll' diff --git a/llvm/test/CodeGen/ARM/misched-int-basic.mir b/llvm/test/CodeGen/ARM/misched-int-basic.mir index 4a52af194247..1a0ee9b19bc7 100644 --- a/llvm/test/CodeGen/ARM/misched-int-basic.mir +++ b/llvm/test/CodeGen/ARM/misched-int-basic.mir @@ -4,6 +4,8 @@ # RUN: -debug-only=machine-scheduler 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_A9 # RUN: llc -o /dev/null %s -mtriple=arm-eabi -mcpu=cortex-r52 -run-pass machine-scheduler -enable-misched -verify-misched \ # RUN: -debug-only=machine-scheduler 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_R52 +# RUN: llc -o /dev/null %s -mtriple=arm-eabi -mcpu=cortex-r52plus -run-pass machine-scheduler -enable-misched -verify-misched \ +# RUN: -debug-only=machine-scheduler 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK_R52 # REQUIRES: asserts --- | ; ModuleID = 'foo.ll' diff --git a/llvm/test/CodeGen/ARM/proc-resource-sched.ll b/llvm/test/CodeGen/ARM/proc-resource-sched.ll index 2dfadda755c6..3bd3bff847ee 100644 --- a/llvm/test/CodeGen/ARM/proc-resource-sched.ll +++ b/llvm/test/CodeGen/ARM/proc-resource-sched.ll @@ -1,4 +1,5 @@ ; RUN: llc -mtriple=arm-eabi -mcpu=cortex-r52 -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-R52 +; RUN: llc -mtriple=arm-eabi -mcpu=cortex-r52plus -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-R52 ; REQUIRES: asserts ; source_filename = "sched-2.c" diff --git a/llvm/test/CodeGen/ARM/single-issue-r52.mir b/llvm/test/CodeGen/ARM/single-issue-r52.mir index 05b167316ca4..084afb6f666c 100644 --- a/llvm/test/CodeGen/ARM/single-issue-r52.mir +++ b/llvm/test/CodeGen/ARM/single-issue-r52.mir @@ -1,5 +1,7 @@ # RUN: llc -o /dev/null %s -mtriple=arm-eabi -mcpu=cortex-r52 -run-pass machine-scheduler -enable-misched -debug-only=machine-scheduler -misched-topdown 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=TOPDOWN # RUN: llc -o /dev/null %s -mtriple=arm-eabi -mcpu=cortex-r52 -run-pass machine-scheduler -enable-misched -debug-only=machine-scheduler -misched-bottomup 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOTTOMUP +# RUN: llc -o /dev/null %s -mtriple=arm-eabi -mcpu=cortex-r52plus -run-pass machine-scheduler -enable-misched -debug-only=machine-scheduler -misched-topdown 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=TOPDOWN +# RUN: llc -o /dev/null %s -mtriple=arm-eabi -mcpu=cortex-r52plus -run-pass machine-scheduler -enable-misched -debug-only=machine-scheduler -misched-bottomup 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOTTOMUP # REQUIRES: asserts --- | ; ModuleID = 'foo.ll' diff --git a/llvm/test/CodeGen/ARM/useaa.ll b/llvm/test/CodeGen/ARM/useaa.ll index d70d24b3fd34..9be0b88b3598 100644 --- a/llvm/test/CodeGen/ARM/useaa.ll +++ b/llvm/test/CodeGen/ARM/useaa.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-r52 | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-r52plus | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA ; RUN: llc < %s -mtriple=armv7m-eabi -mcpu=cortex-m4 | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA ; RUN: llc < %s -mtriple=armv8m-eabi -mcpu=cortex-m33 | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA ; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=generic | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA diff --git a/llvm/test/MC/ARM/dfb-neg.s b/llvm/test/MC/ARM/dfb-neg.s index 15c44877fa6f..ec9192bca3cc 100644 --- a/llvm/test/MC/ARM/dfb-neg.s +++ b/llvm/test/MC/ARM/dfb-neg.s @@ -1,5 +1,7 @@ @ RUN: not llvm-mc -triple armv8-none-eabi -mcpu=cortex-r52 -mattr=-dfb -show-encoding < %s 2>&1 | FileCheck %s @ RUN: not llvm-mc -triple thumbv8-none-eabi -mcpu=cortex-r52 -mattr=-dfb -show-encoding < %s 2>&1 | FileCheck %s +@ RUN: not llvm-mc -triple armv8-none-eabi -mcpu=cortex-r52plus -mattr=-dfb -show-encoding < %s 2>&1 | FileCheck %s +@ RUN: not llvm-mc -triple thumbv8-none-eabi -mcpu=cortex-r52plus -mattr=-dfb -show-encoding < %s 2>&1 | FileCheck %s dfb @ CHECK: error: instruction requires: full-data-barrier diff --git a/llvm/test/MC/ARM/dfb.s b/llvm/test/MC/ARM/dfb.s index 58477749807f..9c4768f6e245 100644 --- a/llvm/test/MC/ARM/dfb.s +++ b/llvm/test/MC/ARM/dfb.s @@ -1,5 +1,7 @@ @ RUN: llvm-mc -triple armv8-none-eabi -mcpu=cortex-r52 -show-encoding < %s | FileCheck %s --check-prefix=CHECK-ARM @ RUN: llvm-mc -triple thumbv8-none-eabi -mcpu=cortex-r52 -show-encoding < %s | FileCheck %s --check-prefix=CHECK-THUMB +@ RUN: llvm-mc -triple armv8-none-eabi -mcpu=cortex-r52plus -show-encoding < %s | FileCheck %s --check-prefix=CHECK-ARM +@ RUN: llvm-mc -triple thumbv8-none-eabi -mcpu=cortex-r52plus -show-encoding < %s | FileCheck %s --check-prefix=CHECK-THUMB dfb @ CHECK-ARM: dfb @ encoding: [0x4c,0xf0,0x7f,0xf5] diff --git a/llvm/test/MC/ARM/invalid-armv8r.s b/llvm/test/MC/ARM/invalid-armv8r.s new file mode 100644 index 000000000000..c87875a22f14 --- /dev/null +++ b/llvm/test/MC/ARM/invalid-armv8r.s @@ -0,0 +1,6 @@ +@ RUN: not llvm-mc -triple=armv8r-linux-gnu -mcpu=cortex-r52 -show-encoding < %s 2>&1 | FileCheck %s -check-prefix=CHECK-NOTZ +@ RUN: not llvm-mc -triple=armv8r-linux-gnu -mcpu=cortex-r52plus -show-encoding < %s 2>&1 | FileCheck %s -check-prefix=CHECK-NOTZ + + smc #0xf + +@ CHECK-NOTZ: error: instruction requires: TrustZone diff --git a/llvm/test/MC/ARM/thumb-hints.s b/llvm/test/MC/ARM/thumb-hints.s index b3c4cee40bbd..53b7b91237cc 100644 --- a/llvm/test/MC/ARM/thumb-hints.s +++ b/llvm/test/MC/ARM/thumb-hints.s @@ -1,3 +1,5 @@ +@ RUN: llvm-mc -triple=thumbv8r-apple-darwin -mcpu=cortex-r52 -show-encoding < %s | FileCheck %s +@ RUN: llvm-mc -triple=thumbv8r-apple-darwin -mcpu=cortex-r52plus -show-encoding < %s | FileCheck %s @ RUN: llvm-mc -triple=thumbv7-apple-darwin -show-encoding < %s | FileCheck %s @ RUN: llvm-mc -triple=thumbv6-apple-darwin -mcpu=cortex-m0 -show-encoding < %s | FileCheck %s @ RUN: not llvm-mc -triple=thumbv6-apple-darwin -show-encoding < %s > %t 2> %t2 diff --git a/llvm/test/MC/Disassembler/ARM/dfb-thumb.txt b/llvm/test/MC/Disassembler/ARM/dfb-thumb.txt index aa8adc83c1f4..ae620cb458eb 100644 --- a/llvm/test/MC/Disassembler/ARM/dfb-thumb.txt +++ b/llvm/test/MC/Disassembler/ARM/dfb-thumb.txt @@ -1,5 +1,7 @@ # RUN: llvm-mc -disassemble -triple thumbv8-none-eabi -mcpu=cortex-r52 -show-encoding < %s | FileCheck %s --check-prefix=CHECK-DFB # RUN: llvm-mc -disassemble -triple thumbv8-none-eabi -mcpu=cortex-r52 -mattr=-dfb -show-encoding < %s | FileCheck %s --check-prefix=CHECK-NODFB +# RUN: llvm-mc -disassemble -triple thumbv8-none-eabi -mcpu=cortex-r52plus -show-encoding < %s | FileCheck %s --check-prefix=CHECK-DFB +# RUN: llvm-mc -disassemble -triple thumbv8-none-eabi -mcpu=cortex-r52plus -mattr=-dfb -show-encoding < %s | FileCheck %s --check-prefix=CHECK-NODFB # CHECK-DFB: dfb @ encoding: [0xbf,0xf3,0x4c,0x8f] # CHECK-NODFB: dsb #0xc @ encoding: [0xbf,0xf3,0x4c,0x8f] diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp index 571031d07fcc..b2e57f2cca13 100644 --- a/llvm/unittests/TargetParser/TargetParserTest.cpp +++ b/llvm/unittests/TargetParser/TargetParserTest.cpp @@ -356,6 +356,11 @@ INSTANTIATE_TEST_SUITE_P( ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP, "8-R"), + ARMCPUTestParams("cortex-r52plus", "armv8-r", "neon-fp-armv8", + ARM::AEK_NONE | ARM::AEK_CRC | ARM::AEK_MP | + ARM::AEK_VIRT | ARM::AEK_HWDIVARM | + ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP, + "8-R"), ARMCPUTestParams("sc300", "armv7-m", "none", ARM::AEK_NONE | ARM::AEK_HWDIVTHUMB, "7-M"), ARMCPUTestParams("cortex-m3", "armv7-m", "none", @@ -548,7 +553,7 @@ INSTANTIATE_TEST_SUITE_P( "7-S")), ARMCPUTestParams::PrintToStringParamName); -static constexpr unsigned NumARMCPUArchs = 91; +static constexpr unsigned NumARMCPUArchs = 92; TEST(TargetParserTest, testARMCPUArchList) { SmallVector List; @@ -692,6 +697,8 @@ TEST(TargetParserTest, testARMExtension) { EXPECT_FALSE( testARMExtension("cortex-a75", ARM::ArchKind::INVALID, "fp16fml")); EXPECT_FALSE(testARMExtension("cortex-r52", ARM::ArchKind::INVALID, "ras")); + EXPECT_FALSE( + testARMExtension("cortex-r52plus", ARM::ArchKind::INVALID, "ras")); EXPECT_FALSE(testARMExtension("iwmmxt", ARM::ArchKind::INVALID, "crc")); EXPECT_FALSE(testARMExtension("xscale", ARM::ArchKind::INVALID, "crc")); EXPECT_FALSE(testARMExtension("swift", ARM::ArchKind::INVALID, "crc")); -- GitLab From 537165bb02df8e9f4dc4c21725d50e77e2908a5f Mon Sep 17 00:00:00 2001 From: WANG Rui Date: Fri, 7 Jun 2024 18:29:30 +0800 Subject: [PATCH 229/462] [NFC][LoongArch] Update test for #94590 --- llvm/test/CodeGen/LoongArch/opt-pipeline.ll | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll index 615cb5700ebf..b0c77155c095 100644 --- a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll +++ b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll @@ -119,6 +119,7 @@ ; LAXX-NEXT: Register Coalescer ; LAXX-NEXT: Rename Disconnected Subregister Components ; LAXX-NEXT: Machine Instruction Scheduler +; LAXX-NEXT: LoongArch Dead register definitions ; LAXX-NEXT: Machine Block Frequency Analysis ; LAXX-NEXT: Debug Variable Analysis ; LAXX-NEXT: Live Stack Slot Analysis -- GitLab From 54c5dbe7c3812461decbccb6ed122e41777e02bd Mon Sep 17 00:00:00 2001 From: David Spickett Date: Fri, 7 Jun 2024 10:36:20 +0000 Subject: [PATCH 230/462] [clang][test] Skip interpreter value test on Arm 32 bit https://github.com/llvm/llvm-project/pull/89811 caused this test to fail, somehow. I think it may not be at fault, but actually be exposing some existing undefined behaviour, see https://github.com/llvm/llvm-project/issues/94741. Skipping this for now to get the bots green again. --- clang/unittests/Interpreter/InterpreterTest.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp index ab9b7a31094f..72b34dae378e 100644 --- a/clang/unittests/Interpreter/InterpreterTest.cpp +++ b/clang/unittests/Interpreter/InterpreterTest.cpp @@ -282,6 +282,9 @@ TEST_F(InterpreterTest, InstantiateTemplate) { EXPECT_EQ(42, fn(NewA.getPtr())); } +// This test exposes an ARM specific problem in the interpreter, see +// https://github.com/llvm/llvm-project/issues/94741. +#ifndef __arm__ TEST_F(InterpreterTest, Value) { std::unique_ptr Interp = createInterpreter(); @@ -379,5 +382,6 @@ TEST_F(InterpreterTest, Value) { EXPECT_EQ(V9.getKind(), Value::K_PtrOrObj); EXPECT_TRUE(V9.isManuallyAlloc()); } +#endif /* ifndef __arm__ */ } // end anonymous namespace -- GitLab From d3e531cf37ed5334aa873e4e46aff693efac9d77 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Fri, 7 Jun 2024 10:42:32 +0000 Subject: [PATCH 231/462] [gn build] Port e622996eddfb --- .../gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn index 3ae50b214eb1..f12aaa7dd4fa 100644 --- a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn @@ -83,6 +83,7 @@ static_library("Checkers") { "MoveChecker.cpp", "NSAutoreleasePoolChecker.cpp", "NSErrorChecker.cpp", + "NoOwnershipChangeVisitor.cpp", "NoReturnFunctionChecker.cpp", "NonNullParamChecker.cpp", "NonnullGlobalConstantsChecker.cpp", -- GitLab From 6fe5428ecbd18aa263417a244c0850b1271617c0 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Fri, 7 Jun 2024 12:49:01 +0200 Subject: [PATCH 232/462] [Flang] Handle the newly-added "Reserved" FramePointerKind for 1a5239251ead73ee57f4e2f7fc93433ac7cf18b1 --- clang/lib/Driver/ToolChains/Flang.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp index 9609a1dc65c0..42b45dba2bd3 100644 --- a/clang/lib/Driver/ToolChains/Flang.cpp +++ b/clang/lib/Driver/ToolChains/Flang.cpp @@ -802,6 +802,9 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA, case CodeGenOptions::FramePointerKind::None: FPKeepKindStr = "-mframe-pointer=none"; break; + case CodeGenOptions::FramePointerKind::Reserved: + FPKeepKindStr = "-mframe-pointer=reserved"; + break; case CodeGenOptions::FramePointerKind::NonLeaf: FPKeepKindStr = "-mframe-pointer=non-leaf"; break; -- GitLab From 88e2bb40921f35663e16e45514de20018615c36f Mon Sep 17 00:00:00 2001 From: Alex Voicu Date: Fri, 7 Jun 2024 13:50:23 +0300 Subject: [PATCH 233/462] [clang][SPIR-V] Add support for AMDGCN flavoured SPIRV (#89796) This change seeks to add support for vendor flavoured SPIRV - more specifically, AMDGCN flavoured SPIRV. The aim is to generate SPIRV that carries some extra bits of information that are only usable by AMDGCN targets, forfeiting absolute genericity to obtain greater expressiveness for target features: - AMDGCN inline ASM is allowed/supported, under the assumption that the [SPV_INTEL_inline_assembly](https://github.com/intel/llvm/blob/sycl/sycl/doc/design/spirv-extensions/SPV_INTEL_inline_assembly.asciidoc) extension is enabled/used - AMDGCN target specific builtins are allowed/supported, under the assumption that e.g. the `--spirv-allow-unknown-intrinsics` option is enabled when using the downstream translator - the featureset matches the union of AMDGCN targets' features - the datalayout string is overspecified to affix both the program address space and the alloca address space, the latter under the assumption that the [SPV_INTEL_function_pointers](https://github.com/intel/llvm/blob/sycl/sycl/doc/design/spirv-extensions/SPV_INTEL_function_pointers.asciidoc) extension is enabled/used, case in which the extant SPIRV datalayout string would lead to pointers to function pointing to the private address space, which would be wrong. Existing AMDGCN tests are extended to cover this new target. It is currently dormant / will require some additional changes, but I thought I'd rather put it up for review to get feedback as early as possible. I will note that an alternative option is to place this under AMDGPU, but that seems slightly less natural, since this is still SPIRV, albeit relaxed in terms of preconditions & constrained in terms of postconditions, and only guaranteed to be usable on AMDGCN targets (it is still possible to obtain pristine portable SPIRV through usage of the flavoured target, though). --- clang/lib/Basic/Targets.cpp | 5 +- clang/lib/Basic/Targets/SPIR.cpp | 75 +++++ clang/lib/Basic/Targets/SPIR.h | 51 +++ clang/lib/CodeGen/CGBuiltin.cpp | 7 + clang/test/CodeGen/target-data.c | 4 + .../test/CodeGenCUDA/builtins-spirv-amdgcn.cu | 294 ++++++++++++++++++ ...tins-unsafe-atomics-spirv-amdgcn-gfx90a.cu | 31 ++ clang/test/CodeGenCUDA/long-double.cu | 4 + clang/test/CodeGenCUDA/spirv-amdgcn-bf16.cu | 129 ++++++++ .../test/CodeGenCXX/spirv-amdgcn-float16.cpp | 38 +++ clang/test/CodeGenHIP/hipspv-addr-spaces.cpp | 2 + clang/test/CodeGenHIP/spirv-amdgcn-ballot.cpp | 27 ++ .../spirv-amdgcn-dpp-const-fold.hip | 46 +++ clang/test/CodeGenHIP/spirv-amdgcn-half.hip | 15 + .../CodeGenOpenCL/amdgcn-flat-scratch-name.cl | 7 +- .../CodeGenOpenCL/builtins-amdgcn-gfx10.cl | 13 +- .../CodeGenOpenCL/builtins-amdgcn-gfx11.cl | 13 +- .../test/CodeGenOpenCL/builtins-amdgcn-vi.cl | 55 ++-- clang/test/CodeGenOpenCL/builtins-amdgcn.cl | 294 ++++++++++-------- clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl | 13 +- clang/test/Preprocessor/hash_builtin.cpp | 3 + .../predefined-macros-no-warnings.c | 1 + clang/test/Preprocessor/predefined-macros.c | 10 + ...in-spirv-amdgcn-atomic-inc-dec-failure.cpp | 25 ++ .../Sema/inline-asm-validate-spirv-amdgcn.cl | 111 +++++++ clang/test/SemaCUDA/allow-int128.cu | 3 + clang/test/SemaCUDA/amdgpu-f128.cu | 1 + clang/test/SemaCUDA/float16.cu | 1 + clang/test/SemaCUDA/fp16-arg-return.cu | 1 + .../test/SemaCUDA/spirv-amdgcn-atomic-ops.cu | 86 +++++ .../builtins-amdgcn-error-gfx908-param.cl | 1 + .../builtins-amdgcn-error-gfx90a-param.cl | 1 + .../builtins-amdgcn-error-gfx940-param.cl | 1 + llvm/docs/SPIRVUsage.rst | 94 +++--- llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp | 4 + llvm/lib/TargetParser/TargetParser.cpp | 42 ++- .../subgroup-rotate.ll | 2 +- ...insics-no-divergent-spv_assign_ptr_type.ll | 5 +- ...Intrinsics-no-duplicate-spv_assign_type.ll | 5 +- .../transcoding/builtin_vars_arithmetics.ll | 6 +- .../transcoding/sub_group_non_uniform_vote.ll | 2 +- 41 files changed, 1298 insertions(+), 230 deletions(-) create mode 100644 clang/test/CodeGenCUDA/builtins-spirv-amdgcn.cu create mode 100644 clang/test/CodeGenCUDA/builtins-unsafe-atomics-spirv-amdgcn-gfx90a.cu create mode 100644 clang/test/CodeGenCUDA/spirv-amdgcn-bf16.cu create mode 100644 clang/test/CodeGenCXX/spirv-amdgcn-float16.cpp create mode 100644 clang/test/CodeGenHIP/spirv-amdgcn-ballot.cpp create mode 100644 clang/test/CodeGenHIP/spirv-amdgcn-dpp-const-fold.hip create mode 100644 clang/test/CodeGenHIP/spirv-amdgcn-half.hip create mode 100644 clang/test/Sema/builtin-spirv-amdgcn-atomic-inc-dec-failure.cpp create mode 100644 clang/test/Sema/inline-asm-validate-spirv-amdgcn.cl create mode 100644 clang/test/SemaCUDA/spirv-amdgcn-atomic-ops.cu diff --git a/clang/lib/Basic/Targets.cpp b/clang/lib/Basic/Targets.cpp index dc1792b3471e..29133f9ee8fc 100644 --- a/clang/lib/Basic/Targets.cpp +++ b/clang/lib/Basic/Targets.cpp @@ -673,8 +673,11 @@ std::unique_ptr AllocateTarget(const llvm::Triple &Triple, } case llvm::Triple::spirv64: { if (os != llvm::Triple::UnknownOS || - Triple.getEnvironment() != llvm::Triple::UnknownEnvironment) + Triple.getEnvironment() != llvm::Triple::UnknownEnvironment) { + if (os == llvm::Triple::OSType::AMDHSA) + return std::make_unique(Triple, Opts); return nullptr; + } return std::make_unique(Triple, Opts); } case llvm::Triple::wasm32: diff --git a/clang/lib/Basic/Targets/SPIR.cpp b/clang/lib/Basic/Targets/SPIR.cpp index dc920177d3a9..040303983594 100644 --- a/clang/lib/Basic/Targets/SPIR.cpp +++ b/clang/lib/Basic/Targets/SPIR.cpp @@ -11,7 +11,9 @@ //===----------------------------------------------------------------------===// #include "SPIR.h" +#include "AMDGPU.h" #include "Targets.h" +#include "llvm/TargetParser/TargetParser.h" using namespace clang; using namespace clang::targets; @@ -54,3 +56,76 @@ void SPIRV64TargetInfo::getTargetDefines(const LangOptions &Opts, BaseSPIRVTargetInfo::getTargetDefines(Opts, Builder); DefineStd(Builder, "SPIRV64", Opts); } + +static const AMDGPUTargetInfo AMDGPUTI(llvm::Triple("amdgcn-amd-amdhsa"), {}); + +ArrayRef SPIRV64AMDGCNTargetInfo::getGCCRegNames() const { + return AMDGPUTI.getGCCRegNames(); +} + +bool SPIRV64AMDGCNTargetInfo::initFeatureMap( + llvm::StringMap &Features, DiagnosticsEngine &Diags, StringRef, + const std::vector &FeatureVec) const { + llvm::AMDGPU::fillAMDGPUFeatureMap({}, getTriple(), Features); + + return TargetInfo::initFeatureMap(Features, Diags, {}, FeatureVec); +} + +bool SPIRV64AMDGCNTargetInfo::validateAsmConstraint( + const char *&Name, TargetInfo::ConstraintInfo &Info) const { + return AMDGPUTI.validateAsmConstraint(Name, Info); +} + +std::string +SPIRV64AMDGCNTargetInfo::convertConstraint(const char *&Constraint) const { + return AMDGPUTI.convertConstraint(Constraint); +} + +ArrayRef SPIRV64AMDGCNTargetInfo::getTargetBuiltins() const { + return AMDGPUTI.getTargetBuiltins(); +} + +void SPIRV64AMDGCNTargetInfo::getTargetDefines(const LangOptions &Opts, + MacroBuilder &Builder) const { + BaseSPIRVTargetInfo::getTargetDefines(Opts, Builder); + DefineStd(Builder, "SPIRV64", Opts); + + Builder.defineMacro("__AMD__"); + Builder.defineMacro("__AMDGPU__"); + Builder.defineMacro("__AMDGCN__"); +} + +void SPIRV64AMDGCNTargetInfo::setAuxTarget(const TargetInfo *Aux) { + assert(Aux && "Cannot invoke setAuxTarget without a valid auxiliary target!"); + + // This is a 1:1 copy of AMDGPUTargetInfo::setAuxTarget() + assert(HalfFormat == Aux->HalfFormat); + assert(FloatFormat == Aux->FloatFormat); + assert(DoubleFormat == Aux->DoubleFormat); + + // On x86_64 long double is 80-bit extended precision format, which is + // not supported by AMDGPU. 128-bit floating point format is also not + // supported by AMDGPU. Therefore keep its own format for these two types. + auto SaveLongDoubleFormat = LongDoubleFormat; + auto SaveFloat128Format = Float128Format; + auto SaveLongDoubleWidth = LongDoubleWidth; + auto SaveLongDoubleAlign = LongDoubleAlign; + copyAuxTarget(Aux); + LongDoubleFormat = SaveLongDoubleFormat; + Float128Format = SaveFloat128Format; + LongDoubleWidth = SaveLongDoubleWidth; + LongDoubleAlign = SaveLongDoubleAlign; + // For certain builtin types support on the host target, claim they are + // supported to pass the compilation of the host code during the device-side + // compilation. + // FIXME: As the side effect, we also accept `__float128` uses in the device + // code. To reject these builtin types supported in the host target but not in + // the device target, one approach would support `device_builtin` attribute + // so that we could tell the device builtin types from the host ones. This + // also solves the different representations of the same builtin type, such + // as `size_t` in the MSVC environment. + if (Aux->hasFloat128Type()) { + HasFloat128 = true; + Float128Format = DoubleFormat; + } +} diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h index 44265445ff00..37cf9d7921ba 100644 --- a/clang/lib/Basic/Targets/SPIR.h +++ b/clang/lib/Basic/Targets/SPIR.h @@ -364,6 +364,57 @@ public: MacroBuilder &Builder) const override; }; +class LLVM_LIBRARY_VISIBILITY SPIRV64AMDGCNTargetInfo final + : public BaseSPIRVTargetInfo { +public: + SPIRV64AMDGCNTargetInfo(const llvm::Triple &Triple, const TargetOptions &Opts) + : BaseSPIRVTargetInfo(Triple, Opts) { + assert(Triple.getArch() == llvm::Triple::spirv64 && + "Invalid architecture for 64-bit AMDGCN SPIR-V."); + assert(Triple.getVendor() == llvm::Triple::VendorType::AMD && + "64-bit AMDGCN SPIR-V target must use AMD vendor"); + assert(getTriple().getOS() == llvm::Triple::OSType::AMDHSA && + "64-bit AMDGCN SPIR-V target must use AMDHSA OS"); + assert(getTriple().getEnvironment() == llvm::Triple::UnknownEnvironment && + "64-bit SPIR-V target must use unknown environment type"); + PointerWidth = PointerAlign = 64; + SizeType = TargetInfo::UnsignedLong; + PtrDiffType = IntPtrType = TargetInfo::SignedLong; + + resetDataLayout("e-i64:64-v16:16-v24:32-v32:32-v48:64-" + "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1-P4-A0"); + + BFloat16Width = BFloat16Align = 16; + BFloat16Format = &llvm::APFloat::BFloat(); + + HasLegalHalfType = true; + HasFloat16 = true; + HalfArgsAndReturns = true; + } + + bool hasBFloat16Type() const override { return true; } + + ArrayRef getGCCRegNames() const override; + + bool initFeatureMap(llvm::StringMap &Features, DiagnosticsEngine &Diags, + StringRef, + const std::vector &) const override; + + bool validateAsmConstraint(const char *&Name, + TargetInfo::ConstraintInfo &Info) const override; + + std::string convertConstraint(const char *&Constraint) const override; + + ArrayRef getTargetBuiltins() const override; + + void getTargetDefines(const LangOptions &Opts, + MacroBuilder &Builder) const override; + + void setAuxTarget(const TargetInfo *Aux) override; + + bool hasInt128Type() const override { return TargetInfo::hasInt128Type(); } +}; + } // namespace targets } // namespace clang #endif // LLVM_CLANG_LIB_BASIC_TARGETS_SPIR_H diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 37d0c478e033..c16b69ba8756 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -6012,6 +6012,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch()); if (!Prefix.empty()) { IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name); + if (IntrinsicID == Intrinsic::not_intrinsic && Prefix == "spv" && + getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA) + IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin("amdgcn", Name); // NOTE we don't need to perform a compatibility flag check here since the // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the // MS builtins via ALL_MS_LANGUAGES and are filtered earlier. @@ -6182,6 +6185,10 @@ static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, case llvm::Triple::riscv32: case llvm::Triple::riscv64: return CGF->EmitRISCVBuiltinExpr(BuiltinID, E, ReturnValue); + case llvm::Triple::spirv64: + if (CGF->getTarget().getTriple().getOS() != llvm::Triple::OSType::AMDHSA) + return nullptr; + return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E); default: return nullptr; } diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c index 9d86880d6513..7f7005d21b99 100644 --- a/clang/test/CodeGen/target-data.c +++ b/clang/test/CodeGen/target-data.c @@ -268,3 +268,7 @@ // RUN: %clang_cc1 -triple ve -o - -emit-llvm %s | \ // RUN: FileCheck %s -check-prefix=VE // VE: target datalayout = "e-m:e-i64:64-n32:64-S128-v64:64:64-v128:64:64-v256:64:64-v512:64:64-v1024:64:64-v2048:64:64-v4096:64:64-v8192:64:64-v16384:64:64" + +// RUN: %clang_cc1 -triple spirv64-amd -o - -emit-llvm %s | \ +// RUN: FileCheck %s -check-prefix=SPIR64 +// AMDGPUSPIRV64: target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1-P4-A0" diff --git a/clang/test/CodeGenCUDA/builtins-spirv-amdgcn.cu b/clang/test/CodeGenCUDA/builtins-spirv-amdgcn.cu new file mode 100644 index 000000000000..8dbb8c538ddc --- /dev/null +++ b/clang/test/CodeGenCUDA/builtins-spirv-amdgcn.cu @@ -0,0 +1,294 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -x hip \ +// RUN: -aux-triple x86_64-unknown-linux-gnu -fcuda-is-device -emit-llvm %s \ +// RUN: -o - | FileCheck %s + +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -x hip \ +// RUN: -aux-triple x86_64-pc-windows-msvc -fcuda-is-device -emit-llvm %s \ +// RUN: -o - | FileCheck %s + +#include "Inputs/cuda.h" + +// CHECK-LABEL: @_Z16use_dispatch_ptrPi( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[OUT:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[DISPATCH_PTR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[OUT_ASCAST:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(4) +// CHECK-NEXT: [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[DISPATCH_PTR_ASCAST:%.*]] = addrspacecast ptr [[DISPATCH_PTR]] to ptr addrspace(4) +// CHECK-NEXT: [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[OUT_COERCE:%.*]] to ptr addrspace(4) +// CHECK-NEXT: store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[OUT_ASCAST]], align 8 +// CHECK-NEXT: [[OUT1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ASCAST]], align 8 +// CHECK-NEXT: store ptr addrspace(4) [[OUT1]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = call align 4 dereferenceable(64) addrspace(4) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr() +// CHECK-NEXT: store ptr addrspace(4) [[TMP1]], ptr addrspace(4) [[DISPATCH_PTR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP2:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[DISPATCH_PTR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(4) [[TMP2]], align 4 +// CHECK-NEXT: [[TMP4:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store i32 [[TMP3]], ptr addrspace(4) [[TMP4]], align 4 +// CHECK-NEXT: ret void +// +__global__ void use_dispatch_ptr(int* out) { + const int* dispatch_ptr = (const int*)__builtin_amdgcn_dispatch_ptr(); + *out = *dispatch_ptr; +} + +// CHECK-LABEL: @_Z13use_queue_ptrPi( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[OUT:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[QUEUE_PTR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[OUT_ASCAST:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(4) +// CHECK-NEXT: [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[QUEUE_PTR_ASCAST:%.*]] = addrspacecast ptr [[QUEUE_PTR]] to ptr addrspace(4) +// CHECK-NEXT: [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[OUT_COERCE:%.*]] to ptr addrspace(4) +// CHECK-NEXT: store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[OUT_ASCAST]], align 8 +// CHECK-NEXT: [[OUT1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ASCAST]], align 8 +// CHECK-NEXT: store ptr addrspace(4) [[OUT1]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = call addrspace(4) ptr addrspace(4) @llvm.amdgcn.queue.ptr() +// CHECK-NEXT: store ptr addrspace(4) [[TMP1]], ptr addrspace(4) [[QUEUE_PTR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP2:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[QUEUE_PTR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(4) [[TMP2]], align 4 +// CHECK-NEXT: [[TMP4:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store i32 [[TMP3]], ptr addrspace(4) [[TMP4]], align 4 +// CHECK-NEXT: ret void +// +__global__ void use_queue_ptr(int* out) { + const int* queue_ptr = (const int*)__builtin_amdgcn_queue_ptr(); + *out = *queue_ptr; +} + +// CHECK-LABEL: @_Z19use_implicitarg_ptrPi( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[OUT:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[IMPLICITARG_PTR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[OUT_ASCAST:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(4) +// CHECK-NEXT: [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[IMPLICITARG_PTR_ASCAST:%.*]] = addrspacecast ptr [[IMPLICITARG_PTR]] to ptr addrspace(4) +// CHECK-NEXT: [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[OUT_COERCE:%.*]] to ptr addrspace(4) +// CHECK-NEXT: store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[OUT_ASCAST]], align 8 +// CHECK-NEXT: [[OUT1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ASCAST]], align 8 +// CHECK-NEXT: store ptr addrspace(4) [[OUT1]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = call addrspace(4) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() +// CHECK-NEXT: store ptr addrspace(4) [[TMP1]], ptr addrspace(4) [[IMPLICITARG_PTR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP2:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[IMPLICITARG_PTR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(4) [[TMP2]], align 4 +// CHECK-NEXT: [[TMP4:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store i32 [[TMP3]], ptr addrspace(4) [[TMP4]], align 4 +// CHECK-NEXT: ret void +// +__global__ void use_implicitarg_ptr(int* out) { + const int* implicitarg_ptr = (const int*)__builtin_amdgcn_implicitarg_ptr(); + *out = *implicitarg_ptr; +} + +__global__ + // + void +// CHECK-LABEL: @_Z12test_ds_fmaxf( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[SRC_ADDR:%.*]] = alloca float, align 4 +// CHECK-NEXT: [[X:%.*]] = alloca float, align 4 +// CHECK-NEXT: [[SRC_ADDR_ASCAST:%.*]] = addrspacecast ptr [[SRC_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[X_ASCAST:%.*]] = addrspacecast ptr [[X]] to ptr addrspace(4) +// CHECK-NEXT: store float [[SRC:%.*]], ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = call contract addrspace(4) float @llvm.amdgcn.ds.fmax.f32(ptr addrspace(3) @_ZZ12test_ds_fmaxfE6shared, float [[TMP0]], i32 0, i32 0, i1 false) +// CHECK-NEXT: store volatile float [[TMP1]], ptr addrspace(4) [[X_ASCAST]], align 4 +// CHECK-NEXT: ret void +// + test_ds_fmax(float src) { + __shared__ float shared; + volatile float x = __builtin_amdgcn_ds_fmaxf(&shared, src, 0, 0, false); +} + +// CHECK-LABEL: @_Z12test_ds_faddf( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[SRC_ADDR:%.*]] = alloca float, align 4 +// CHECK-NEXT: [[X:%.*]] = alloca float, align 4 +// CHECK-NEXT: [[SRC_ADDR_ASCAST:%.*]] = addrspacecast ptr [[SRC_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[X_ASCAST:%.*]] = addrspacecast ptr [[X]] to ptr addrspace(4) +// CHECK-NEXT: store float [[SRC:%.*]], ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = call contract addrspace(4) float @llvm.amdgcn.ds.fadd.f32(ptr addrspace(3) @_ZZ12test_ds_faddfE6shared, float [[TMP0]], i32 0, i32 0, i1 false) +// CHECK-NEXT: store volatile float [[TMP1]], ptr addrspace(4) [[X_ASCAST]], align 4 +// CHECK-NEXT: ret void +// +__global__ void test_ds_fadd(float src) { + __shared__ float shared; + volatile float x = __builtin_amdgcn_ds_faddf(&shared, src, 0, 0, false); +} + +// CHECK-LABEL: @_Z12test_ds_fminfPf( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[SHARED:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[SRC_ADDR:%.*]] = alloca float, align 4 +// CHECK-NEXT: [[SHARED_ADDR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[X:%.*]] = alloca float, align 4 +// CHECK-NEXT: [[SHARED_ASCAST:%.*]] = addrspacecast ptr [[SHARED]] to ptr addrspace(4) +// CHECK-NEXT: [[SRC_ADDR_ASCAST:%.*]] = addrspacecast ptr [[SRC_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[SHARED_ADDR_ASCAST:%.*]] = addrspacecast ptr [[SHARED_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[X_ASCAST:%.*]] = addrspacecast ptr [[X]] to ptr addrspace(4) +// CHECK-NEXT: [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[SHARED_COERCE:%.*]] to ptr addrspace(4) +// CHECK-NEXT: store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[SHARED_ASCAST]], align 8 +// CHECK-NEXT: [[SHARED1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[SHARED_ASCAST]], align 8 +// CHECK-NEXT: store float [[SRC:%.*]], ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4 +// CHECK-NEXT: store ptr addrspace(4) [[SHARED1]], ptr addrspace(4) [[SHARED_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[SHARED_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr addrspace(4) [[TMP1]] to ptr addrspace(3) +// CHECK-NEXT: [[TMP3:%.*]] = load float, ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP4:%.*]] = call contract addrspace(4) float @llvm.amdgcn.ds.fmin.f32(ptr addrspace(3) [[TMP2]], float [[TMP3]], i32 0, i32 0, i1 false) +// CHECK-NEXT: store volatile float [[TMP4]], ptr addrspace(4) [[X_ASCAST]], align 4 +// CHECK-NEXT: ret void +// +__global__ void test_ds_fmin(float src, float *shared) { + volatile float x = __builtin_amdgcn_ds_fminf(shared, src, 0, 0, false); +} + +#if 0 // FIXME: returning a pointer to AS4 explicitly is wrong for AMDGPU SPIRV +// +__device__ void test_ret_builtin_nondef_addrspace() { + void *x = __builtin_amdgcn_dispatch_ptr(); +} +#endif + +// CHECK-LABEL: @_Z6endpgmv( +// CHECK-NEXT: entry: +// CHECK-NEXT: call addrspace(4) void @llvm.amdgcn.endpgm() +// CHECK-NEXT: ret void +// +__global__ void endpgm() { + __builtin_amdgcn_endpgm(); +} + +// Check the 64 bit argument is correctly passed to the intrinsic without truncation or assertion. + +// CHECK-LABEL: @_Z14test_uicmp_i64Pyyy( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[OUT:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[A_ADDR:%.*]] = alloca i64, align 8 +// CHECK-NEXT: [[B_ADDR:%.*]] = alloca i64, align 8 +// CHECK-NEXT: [[OUT_ASCAST:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(4) +// CHECK-NEXT: [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[OUT_COERCE:%.*]] to ptr addrspace(4) +// CHECK-NEXT: store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[OUT_ASCAST]], align 8 +// CHECK-NEXT: [[OUT1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ASCAST]], align 8 +// CHECK-NEXT: store ptr addrspace(4) [[OUT1]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store i64 [[A:%.*]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store i64 [[B:%.*]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr addrspace(4) [[B_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP3:%.*]] = call addrspace(4) i64 @llvm.amdgcn.icmp.i64.i64(i64 [[TMP1]], i64 [[TMP2]], i32 35) +// CHECK-NEXT: [[TMP4:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store i64 [[TMP3]], ptr addrspace(4) [[TMP4]], align 8 +// CHECK-NEXT: ret void +// +__global__ void test_uicmp_i64(unsigned long long *out, unsigned long long a, unsigned long long b) +{ + *out = __builtin_amdgcn_uicmpl(a, b, 30+5); +} + +// Check the 64 bit return value is correctly returned without truncation or assertion. + +// CHECK-LABEL: @_Z14test_s_memtimePy( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[OUT:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[OUT_ASCAST:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(4) +// CHECK-NEXT: [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[OUT_COERCE:%.*]] to ptr addrspace(4) +// CHECK-NEXT: store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[OUT_ASCAST]], align 8 +// CHECK-NEXT: [[OUT1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ASCAST]], align 8 +// CHECK-NEXT: store ptr addrspace(4) [[OUT1]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = call addrspace(4) i64 @llvm.amdgcn.s.memtime() +// CHECK-NEXT: [[TMP2:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store i64 [[TMP1]], ptr addrspace(4) [[TMP2]], align 8 +// CHECK-NEXT: ret void +// +__global__ void test_s_memtime(unsigned long long* out) +{ + *out = __builtin_amdgcn_s_memtime(); +} + +// Check a generic pointer can be passed as a shared pointer and a generic pointer. +__device__ void func(float *x); + +// CHECK-LABEL: @_Z17test_ds_fmin_funcfPf( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[SHARED:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[SRC_ADDR:%.*]] = alloca float, align 4 +// CHECK-NEXT: [[SHARED_ADDR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[X:%.*]] = alloca float, align 4 +// CHECK-NEXT: [[SHARED_ASCAST:%.*]] = addrspacecast ptr [[SHARED]] to ptr addrspace(4) +// CHECK-NEXT: [[SRC_ADDR_ASCAST:%.*]] = addrspacecast ptr [[SRC_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[SHARED_ADDR_ASCAST:%.*]] = addrspacecast ptr [[SHARED_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[X_ASCAST:%.*]] = addrspacecast ptr [[X]] to ptr addrspace(4) +// CHECK-NEXT: [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[SHARED_COERCE:%.*]] to ptr addrspace(4) +// CHECK-NEXT: store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[SHARED_ASCAST]], align 8 +// CHECK-NEXT: [[SHARED1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[SHARED_ASCAST]], align 8 +// CHECK-NEXT: store float [[SRC:%.*]], ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4 +// CHECK-NEXT: store ptr addrspace(4) [[SHARED1]], ptr addrspace(4) [[SHARED_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[SHARED_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr addrspace(4) [[TMP1]] to ptr addrspace(3) +// CHECK-NEXT: [[TMP3:%.*]] = load float, ptr addrspace(4) [[SRC_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP4:%.*]] = call contract addrspace(4) float @llvm.amdgcn.ds.fmin.f32(ptr addrspace(3) [[TMP2]], float [[TMP3]], i32 0, i32 0, i1 false) +// CHECK-NEXT: store volatile float [[TMP4]], ptr addrspace(4) [[X_ASCAST]], align 4 +// CHECK-NEXT: [[TMP5:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[SHARED_ADDR_ASCAST]], align 8 +// CHECK-NEXT: call spir_func addrspace(4) void @_Z4funcPf(ptr addrspace(4) noundef [[TMP5]]) #[[ATTR7:[0-9]+]] +// CHECK-NEXT: ret void +// +__global__ void test_ds_fmin_func(float src, float *__restrict shared) { + volatile float x = __builtin_amdgcn_ds_fminf(shared, src, 0, 0, false); + func(shared); +} + +// CHECK-LABEL: @_Z14test_is_sharedPf( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[X:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[X_ADDR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[RET:%.*]] = alloca i8, align 1 +// CHECK-NEXT: [[X_ASCAST:%.*]] = addrspacecast ptr [[X]] to ptr addrspace(4) +// CHECK-NEXT: [[X_ADDR_ASCAST:%.*]] = addrspacecast ptr [[X_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[RET_ASCAST:%.*]] = addrspacecast ptr [[RET]] to ptr addrspace(4) +// CHECK-NEXT: [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[X_COERCE:%.*]] to ptr addrspace(4) +// CHECK-NEXT: store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[X_ASCAST]], align 8 +// CHECK-NEXT: [[X1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[X_ASCAST]], align 8 +// CHECK-NEXT: store ptr addrspace(4) [[X1]], ptr addrspace(4) [[X_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[X_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr addrspace(4) [[TMP1]] to ptr +// CHECK-NEXT: [[TMP3:%.*]] = call addrspace(4) i1 @llvm.amdgcn.is.shared(ptr [[TMP2]]) +// CHECK-NEXT: [[FROMBOOL:%.*]] = zext i1 [[TMP3]] to i8 +// CHECK-NEXT: store i8 [[FROMBOOL]], ptr addrspace(4) [[RET_ASCAST]], align 1 +// CHECK-NEXT: ret void +// +__global__ void test_is_shared(float *x){ + bool ret = __builtin_amdgcn_is_shared(x); +} + +// CHECK-LABEL: @_Z15test_is_privatePi( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[X:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[X_ADDR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[RET:%.*]] = alloca i8, align 1 +// CHECK-NEXT: [[X_ASCAST:%.*]] = addrspacecast ptr [[X]] to ptr addrspace(4) +// CHECK-NEXT: [[X_ADDR_ASCAST:%.*]] = addrspacecast ptr [[X_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[RET_ASCAST:%.*]] = addrspacecast ptr [[RET]] to ptr addrspace(4) +// CHECK-NEXT: [[TMP0:%.*]] = addrspacecast ptr addrspace(1) [[X_COERCE:%.*]] to ptr addrspace(4) +// CHECK-NEXT: store ptr addrspace(4) [[TMP0]], ptr addrspace(4) [[X_ASCAST]], align 8 +// CHECK-NEXT: [[X1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[X_ASCAST]], align 8 +// CHECK-NEXT: store ptr addrspace(4) [[X1]], ptr addrspace(4) [[X_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[X_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr addrspace(4) [[TMP1]] to ptr +// CHECK-NEXT: [[TMP3:%.*]] = call addrspace(4) i1 @llvm.amdgcn.is.private(ptr [[TMP2]]) +// CHECK-NEXT: [[FROMBOOL:%.*]] = zext i1 [[TMP3]] to i8 +// CHECK-NEXT: store i8 [[FROMBOOL]], ptr addrspace(4) [[RET_ASCAST]], align 1 +// CHECK-NEXT: ret void +// +__global__ void test_is_private(int *x){ + bool ret = __builtin_amdgcn_is_private(x); +} diff --git a/clang/test/CodeGenCUDA/builtins-unsafe-atomics-spirv-amdgcn-gfx90a.cu b/clang/test/CodeGenCUDA/builtins-unsafe-atomics-spirv-amdgcn-gfx90a.cu new file mode 100644 index 000000000000..1ea1d5f45476 --- /dev/null +++ b/clang/test/CodeGenCUDA/builtins-unsafe-atomics-spirv-amdgcn-gfx90a.cu @@ -0,0 +1,31 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4 +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -x hip \ +// RUN: -aux-triple x86_64-unknown-linux-gnu -fcuda-is-device -emit-llvm %s \ +// RUN: -o - | FileCheck %s + +#define __device__ __attribute__((device)) +typedef __attribute__((address_space(3))) float *LP; + +// CHECK-LABEL: define spir_func void @_Z22test_ds_atomic_add_f32Pff( +// CHECK-SAME: ptr addrspace(4) noundef [[ADDR:%.*]], float noundef [[VAL:%.*]]) addrspace(4) #[[ATTR0:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[ADDR_ADDR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[VAL_ADDR:%.*]] = alloca float, align 4 +// CHECK-NEXT: [[RTN:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[ADDR_ADDR_ASCAST:%.*]] = addrspacecast ptr [[ADDR_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[VAL_ADDR_ASCAST:%.*]] = addrspacecast ptr [[VAL_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[RTN_ASCAST:%.*]] = addrspacecast ptr [[RTN]] to ptr addrspace(4) +// CHECK-NEXT: store ptr addrspace(4) [[ADDR]], ptr addrspace(4) [[ADDR_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store float [[VAL]], ptr addrspace(4) [[VAL_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[ADDR_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(4) [[TMP0]] to ptr addrspace(3) +// CHECK-NEXT: [[TMP2:%.*]] = load float, ptr addrspace(4) [[VAL_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP3:%.*]] = call contract addrspace(4) float @llvm.amdgcn.ds.fadd.f32(ptr addrspace(3) [[TMP1]], float [[TMP2]], i32 0, i32 0, i1 false) +// CHECK-NEXT: [[TMP4:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[RTN_ASCAST]], align 8 +// CHECK-NEXT: store float [[TMP3]], ptr addrspace(4) [[TMP4]], align 4 +// CHECK-NEXT: ret void +// +__device__ void test_ds_atomic_add_f32(float *addr, float val) { + float *rtn; + *rtn = __builtin_amdgcn_ds_faddf((LP)addr, val, 0, 0, 0); +} diff --git a/clang/test/CodeGenCUDA/long-double.cu b/clang/test/CodeGenCUDA/long-double.cu index d52de972ea3d..898afcac124b 100644 --- a/clang/test/CodeGenCUDA/long-double.cu +++ b/clang/test/CodeGenCUDA/long-double.cu @@ -2,6 +2,10 @@ // RUN: -aux-triple x86_64-unknown-gnu-linux -fcuda-is-device \ // RUN: -emit-llvm -o - -x hip %s 2>&1 | FileCheck %s +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa \ +// RUN: -aux-triple x86_64-unknown-gnu-linux -fcuda-is-device \ +// RUN: -emit-llvm -o - -x hip %s 2>&1 | FileCheck %s + // RUN: %clang_cc1 -triple nvptx \ // RUN: -aux-triple x86_64-unknown-gnu-linux -fcuda-is-device \ // RUN: -emit-llvm -o - %s 2>&1 | FileCheck %s diff --git a/clang/test/CodeGenCUDA/spirv-amdgcn-bf16.cu b/clang/test/CodeGenCUDA/spirv-amdgcn-bf16.cu new file mode 100644 index 000000000000..2a0f84d1daa7 --- /dev/null +++ b/clang/test/CodeGenCUDA/spirv-amdgcn-bf16.cu @@ -0,0 +1,129 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: amdgpu-registered-target +// REQUIRES: x86-registered-target + +// RUN: %clang_cc1 "-aux-triple" "x86_64-unknown-linux-gnu" "-triple" "spirv64-amd-amdhsa" \ +// RUN: -fcuda-is-device "-aux-target-cpu" "x86-64" -emit-llvm -o - %s | FileCheck %s + +#include "Inputs/cuda.h" + +// CHECK-LABEL: @_Z8test_argPDF16bDF16b( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[IN_ADDR:%.*]] = alloca bfloat, align 2 +// CHECK-NEXT: [[BF16:%.*]] = alloca bfloat, align 2 +// CHECK-NEXT: [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[IN_ADDR_ASCAST:%.*]] = addrspacecast ptr [[IN_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[BF16_ASCAST:%.*]] = addrspacecast ptr [[BF16]] to ptr addrspace(4) +// CHECK-NEXT: store ptr addrspace(4) [[OUT:%.*]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store bfloat [[IN:%.*]], ptr addrspace(4) [[IN_ADDR_ASCAST]], align 2 +// CHECK-NEXT: [[TMP0:%.*]] = load bfloat, ptr addrspace(4) [[IN_ADDR_ASCAST]], align 2 +// CHECK-NEXT: store bfloat [[TMP0]], ptr addrspace(4) [[BF16_ASCAST]], align 2 +// CHECK-NEXT: [[TMP1:%.*]] = load bfloat, ptr addrspace(4) [[BF16_ASCAST]], align 2 +// CHECK-NEXT: [[TMP2:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store bfloat [[TMP1]], ptr addrspace(4) [[TMP2]], align 2 +// CHECK-NEXT: ret void +// +__device__ void test_arg(__bf16 *out, __bf16 in) { + __bf16 bf16 = in; + *out = bf16; +} + +// CHECK-LABEL: @_Z9test_loadPDF16bS_( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[IN_ADDR:%.*]] = alloca ptr addrspace(4), align 8 +// CHECK-NEXT: [[BF16:%.*]] = alloca bfloat, align 2 +// CHECK-NEXT: [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr [[OUT_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[IN_ADDR_ASCAST:%.*]] = addrspacecast ptr [[IN_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: [[BF16_ASCAST:%.*]] = addrspacecast ptr [[BF16]] to ptr addrspace(4) +// CHECK-NEXT: store ptr addrspace(4) [[OUT:%.*]], ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store ptr addrspace(4) [[IN:%.*]], ptr addrspace(4) [[IN_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[IN_ADDR_ASCAST]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = load bfloat, ptr addrspace(4) [[TMP0]], align 2 +// CHECK-NEXT: store bfloat [[TMP1]], ptr addrspace(4) [[BF16_ASCAST]], align 2 +// CHECK-NEXT: [[TMP2:%.*]] = load bfloat, ptr addrspace(4) [[BF16_ASCAST]], align 2 +// CHECK-NEXT: [[TMP3:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[OUT_ADDR_ASCAST]], align 8 +// CHECK-NEXT: store bfloat [[TMP2]], ptr addrspace(4) [[TMP3]], align 2 +// CHECK-NEXT: ret void +// +__device__ void test_load(__bf16 *out, __bf16 *in) { + __bf16 bf16 = *in; + *out = bf16; +} + +// CHECK-LABEL: @_Z8test_retDF16b( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[RETVAL:%.*]] = alloca bfloat, align 2 +// CHECK-NEXT: [[IN_ADDR:%.*]] = alloca bfloat, align 2 +// CHECK-NEXT: [[RETVAL_ASCAST:%.*]] = addrspacecast ptr [[RETVAL]] to ptr addrspace(4) +// CHECK-NEXT: [[IN_ADDR_ASCAST:%.*]] = addrspacecast ptr [[IN_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: store bfloat [[IN:%.*]], ptr addrspace(4) [[IN_ADDR_ASCAST]], align 2 +// CHECK-NEXT: [[TMP0:%.*]] = load bfloat, ptr addrspace(4) [[IN_ADDR_ASCAST]], align 2 +// CHECK-NEXT: ret bfloat [[TMP0]] +// +__device__ __bf16 test_ret( __bf16 in) { + return in; +} + +// CHECK-LABEL: @_Z9test_callDF16b( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[RETVAL:%.*]] = alloca bfloat, align 2 +// CHECK-NEXT: [[IN_ADDR:%.*]] = alloca bfloat, align 2 +// CHECK-NEXT: [[RETVAL_ASCAST:%.*]] = addrspacecast ptr [[RETVAL]] to ptr addrspace(4) +// CHECK-NEXT: [[IN_ADDR_ASCAST:%.*]] = addrspacecast ptr [[IN_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: store bfloat [[IN:%.*]], ptr addrspace(4) [[IN_ADDR_ASCAST]], align 2 +// CHECK-NEXT: [[TMP0:%.*]] = load bfloat, ptr addrspace(4) [[IN_ADDR_ASCAST]], align 2 +// CHECK-NEXT: [[CALL:%.*]] = call contract spir_func noundef addrspace(4) bfloat @_Z8test_retDF16b(bfloat noundef [[TMP0]]) #[[ATTR1:[0-9]+]] +// CHECK-NEXT: ret bfloat [[CALL]] +// +__device__ __bf16 test_call( __bf16 in) { + return test_ret(in); +} + + +// CHECK-LABEL: @_Z15test_vec_assignv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[VEC2_A:%.*]] = alloca <2 x bfloat>, align 4 +// CHECK-NEXT: [[VEC2_B:%.*]] = alloca <2 x bfloat>, align 4 +// CHECK-NEXT: [[VEC4_A:%.*]] = alloca <4 x bfloat>, align 8 +// CHECK-NEXT: [[VEC4_B:%.*]] = alloca <4 x bfloat>, align 8 +// CHECK-NEXT: [[VEC8_A:%.*]] = alloca <8 x bfloat>, align 16 +// CHECK-NEXT: [[VEC8_B:%.*]] = alloca <8 x bfloat>, align 16 +// CHECK-NEXT: [[VEC16_A:%.*]] = alloca <16 x bfloat>, align 32 +// CHECK-NEXT: [[VEC16_B:%.*]] = alloca <16 x bfloat>, align 32 +// CHECK-NEXT: [[VEC2_A_ASCAST:%.*]] = addrspacecast ptr [[VEC2_A]] to ptr addrspace(4) +// CHECK-NEXT: [[VEC2_B_ASCAST:%.*]] = addrspacecast ptr [[VEC2_B]] to ptr addrspace(4) +// CHECK-NEXT: [[VEC4_A_ASCAST:%.*]] = addrspacecast ptr [[VEC4_A]] to ptr addrspace(4) +// CHECK-NEXT: [[VEC4_B_ASCAST:%.*]] = addrspacecast ptr [[VEC4_B]] to ptr addrspace(4) +// CHECK-NEXT: [[VEC8_A_ASCAST:%.*]] = addrspacecast ptr [[VEC8_A]] to ptr addrspace(4) +// CHECK-NEXT: [[VEC8_B_ASCAST:%.*]] = addrspacecast ptr [[VEC8_B]] to ptr addrspace(4) +// CHECK-NEXT: [[VEC16_A_ASCAST:%.*]] = addrspacecast ptr [[VEC16_A]] to ptr addrspace(4) +// CHECK-NEXT: [[VEC16_B_ASCAST:%.*]] = addrspacecast ptr [[VEC16_B]] to ptr addrspace(4) +// CHECK-NEXT: [[TMP0:%.*]] = load <2 x bfloat>, ptr addrspace(4) [[VEC2_B_ASCAST]], align 4 +// CHECK-NEXT: store <2 x bfloat> [[TMP0]], ptr addrspace(4) [[VEC2_A_ASCAST]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load <4 x bfloat>, ptr addrspace(4) [[VEC4_B_ASCAST]], align 8 +// CHECK-NEXT: store <4 x bfloat> [[TMP1]], ptr addrspace(4) [[VEC4_A_ASCAST]], align 8 +// CHECK-NEXT: [[TMP2:%.*]] = load <8 x bfloat>, ptr addrspace(4) [[VEC8_B_ASCAST]], align 16 +// CHECK-NEXT: store <8 x bfloat> [[TMP2]], ptr addrspace(4) [[VEC8_A_ASCAST]], align 16 +// CHECK-NEXT: [[TMP3:%.*]] = load <16 x bfloat>, ptr addrspace(4) [[VEC16_B_ASCAST]], align 32 +// CHECK-NEXT: store <16 x bfloat> [[TMP3]], ptr addrspace(4) [[VEC16_A_ASCAST]], align 32 +// CHECK-NEXT: ret void +// +__device__ void test_vec_assign() { + typedef __attribute__((ext_vector_type(2))) __bf16 bf16_x2; + bf16_x2 vec2_a, vec2_b; + vec2_a = vec2_b; + + typedef __attribute__((ext_vector_type(4))) __bf16 bf16_x4; + bf16_x4 vec4_a, vec4_b; + vec4_a = vec4_b; + + typedef __attribute__((ext_vector_type(8))) __bf16 bf16_x8; + bf16_x8 vec8_a, vec8_b; + vec8_a = vec8_b; + + typedef __attribute__((ext_vector_type(16))) __bf16 bf16_x16; + bf16_x16 vec16_a, vec16_b; + vec16_a = vec16_b; +} diff --git a/clang/test/CodeGenCXX/spirv-amdgcn-float16.cpp b/clang/test/CodeGenCXX/spirv-amdgcn-float16.cpp new file mode 100644 index 000000000000..2487e0fcd434 --- /dev/null +++ b/clang/test/CodeGenCXX/spirv-amdgcn-float16.cpp @@ -0,0 +1,38 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4 +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm -o - %s | FileCheck %s + +// CHECK-LABEL: define spir_func void @_Z1fv( +// CHECK-SAME: ) addrspace(4) #[[ATTR0:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[X:%.*]] = alloca half, align 2 +// CHECK-NEXT: [[Y:%.*]] = alloca half, align 2 +// CHECK-NEXT: [[Z:%.*]] = alloca half, align 2 +// CHECK-NEXT: [[TMP0:%.*]] = load half, ptr [[X]], align 2 +// CHECK-NEXT: [[TMP1:%.*]] = load half, ptr [[Y]], align 2 +// CHECK-NEXT: [[ADD:%.*]] = fadd half [[TMP0]], [[TMP1]] +// CHECK-NEXT: store half [[ADD]], ptr [[Z]], align 2 +// CHECK-NEXT: [[TMP2:%.*]] = load half, ptr [[X]], align 2 +// CHECK-NEXT: [[TMP3:%.*]] = load half, ptr [[Y]], align 2 +// CHECK-NEXT: [[SUB:%.*]] = fsub half [[TMP2]], [[TMP3]] +// CHECK-NEXT: store half [[SUB]], ptr [[Z]], align 2 +// CHECK-NEXT: [[TMP4:%.*]] = load half, ptr [[X]], align 2 +// CHECK-NEXT: [[TMP5:%.*]] = load half, ptr [[Y]], align 2 +// CHECK-NEXT: [[MUL:%.*]] = fmul half [[TMP4]], [[TMP5]] +// CHECK-NEXT: store half [[MUL]], ptr [[Z]], align 2 +// CHECK-NEXT: [[TMP6:%.*]] = load half, ptr [[X]], align 2 +// CHECK-NEXT: [[TMP7:%.*]] = load half, ptr [[Y]], align 2 +// CHECK-NEXT: [[DIV:%.*]] = fdiv half [[TMP6]], [[TMP7]] +// CHECK-NEXT: store half [[DIV]], ptr [[Z]], align 2 +// CHECK-NEXT: ret void +// +void f() { + _Float16 x, y, z; + + z = x + y; + + z = x - y; + + z = x * y; + + z = x / y; +} diff --git a/clang/test/CodeGenHIP/hipspv-addr-spaces.cpp b/clang/test/CodeGenHIP/hipspv-addr-spaces.cpp index 9fcdc460482e..c575f49ff697 100644 --- a/clang/test/CodeGenHIP/hipspv-addr-spaces.cpp +++ b/clang/test/CodeGenHIP/hipspv-addr-spaces.cpp @@ -1,5 +1,7 @@ // RUN: %clang_cc1 -triple spirv64 -x hip -emit-llvm -fcuda-is-device \ // RUN: -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -x hip -emit-llvm -fcuda-is-device \ +// RUN: -o - %s | FileCheck %s #define __device__ __attribute__((device)) #define __shared__ __attribute__((shared)) diff --git a/clang/test/CodeGenHIP/spirv-amdgcn-ballot.cpp b/clang/test/CodeGenHIP/spirv-amdgcn-ballot.cpp new file mode 100644 index 000000000000..8226a109d8b8 --- /dev/null +++ b/clang/test/CodeGenHIP/spirv-amdgcn-ballot.cpp @@ -0,0 +1,27 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4 +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -aux-triple x86_64-pc-windows-msvc -x hip -emit-llvm -fcuda-is-device -o - %s | FileCheck %s + +// Unlike OpenCL, HIP depends on the C++ interpration of "unsigned long", which +// is 64 bits long on Linux and 32 bits long on Windows. The return type of the +// ballot intrinsic needs to be a 64 bit integer on both platforms. This test +// cross-compiles to Windows to confirm that the return type is indeed 64 bits +// on Windows. + +#define __device__ __attribute__((device)) + +// CHECK-LABEL: define spir_func noundef i64 @_Z3fooi( +// CHECK-SAME: i32 noundef [[P:%.*]]) addrspace(4) #[[ATTR0:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[RETVAL:%.*]] = alloca i64, align 8 +// CHECK-NEXT: [[P_ADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: [[RETVAL_ASCAST:%.*]] = addrspacecast ptr [[RETVAL]] to ptr addrspace(4) +// CHECK-NEXT: [[P_ADDR_ASCAST:%.*]] = addrspacecast ptr [[P_ADDR]] to ptr addrspace(4) +// CHECK-NEXT: store i32 [[P]], ptr addrspace(4) [[P_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr addrspace(4) [[P_ADDR_ASCAST]], align 4 +// CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0 +// CHECK-NEXT: [[TMP1:%.*]] = call addrspace(4) i64 @llvm.amdgcn.ballot.i64(i1 [[TOBOOL]]) +// CHECK-NEXT: ret i64 [[TMP1]] +// +__device__ unsigned long long foo(int p) { + return __builtin_amdgcn_ballot_w64(p); +} diff --git a/clang/test/CodeGenHIP/spirv-amdgcn-dpp-const-fold.hip b/clang/test/CodeGenHIP/spirv-amdgcn-dpp-const-fold.hip new file mode 100644 index 000000000000..2b785200e8ee --- /dev/null +++ b/clang/test/CodeGenHIP/spirv-amdgcn-dpp-const-fold.hip @@ -0,0 +1,46 @@ +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -x hip -fcuda-is-device -emit-llvm %s \ +// RUN: -o - | FileCheck %s + +constexpr static int OpCtrl() +{ + return 15 + 1; +} + +constexpr static int RowMask() +{ + return 3 + 1; +} + +constexpr static int BankMask() +{ + return 2 + 1; +} + +constexpr static bool BountCtrl() +{ + return true & false; +} + +// CHECK: call{{.*}} i32 @llvm.amdgcn.update.dpp.i32(i32 %1, i32 %2, i32 16, i32 0, i32 0, i1 false) +__attribute__((global)) void test_update_dpp_const_fold_imm_operand_2(int* out, int a, int b) +{ + *out = __builtin_amdgcn_update_dpp(a, b, OpCtrl(), 0, 0, false); +} + +// CHECK: call{{.*}} i32 @llvm.amdgcn.update.dpp.i32(i32 %1, i32 %2, i32 0, i32 4, i32 0, i1 false) +__attribute__((global)) void test_update_dpp_const_fold_imm_operand_3(int* out, int a, int b) +{ + *out = __builtin_amdgcn_update_dpp(a, b, 0, RowMask(), 0, false); +} + +// CHECK: call{{.*}} i32 @llvm.amdgcn.update.dpp.i32(i32 %1, i32 %2, i32 0, i32 0, i32 3, i1 false) +__attribute__((global)) void test_update_dpp_const_fold_imm_operand_4(int* out, int a, int b) +{ + *out = __builtin_amdgcn_update_dpp(a, b, 0, 0, BankMask(), false); +} + +// CHECK: call{{.*}} i32 @llvm.amdgcn.update.dpp.i32(i32 %1, i32 %2, i32 0, i32 0, i32 0, i1 false) +__attribute__((global)) void test_update_dpp_const_fold_imm_operand_5(int* out, int a, int b) +{ + *out = __builtin_amdgcn_update_dpp(a, b, 0, 0, 0, BountCtrl()); +} diff --git a/clang/test/CodeGenHIP/spirv-amdgcn-half.hip b/clang/test/CodeGenHIP/spirv-amdgcn-half.hip new file mode 100644 index 000000000000..2caf766d943b --- /dev/null +++ b/clang/test/CodeGenHIP/spirv-amdgcn-half.hip @@ -0,0 +1,15 @@ +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -x hip -emit-llvm -fcuda-is-device -o - %s | FileCheck %s + +#define __device__ __attribute__((device)) + +// CHECK-LABEL: @_Z2d0DF16_ +// CHECK: fpext +__device__ float d0(_Float16 x) { + return x; +} + +// CHECK-LABEL: @_Z2d1f +// CHECK: fptrunc +__device__ _Float16 d1(float x) { + return x; +} diff --git a/clang/test/CodeGenOpenCL/amdgcn-flat-scratch-name.cl b/clang/test/CodeGenOpenCL/amdgcn-flat-scratch-name.cl index 619a9a99568e..40a523f0aa0b 100644 --- a/clang/test/CodeGenOpenCL/amdgcn-flat-scratch-name.cl +++ b/clang/test/CodeGenOpenCL/amdgcn-flat-scratch-name.cl @@ -1,15 +1,16 @@ // REQUIRES: amdgpu-registered-target // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm -o - %s | FileCheck %s // CHECK-LABEL: @use_flat_scratch_name kernel void use_flat_scratch_name() { -// CHECK: tail call void asm sideeffect "s_mov_b64 flat_scratch, 0", "~{flat_scratch}"() +// CHECK: tail call{{.*}} void asm sideeffect "s_mov_b64 flat_scratch, 0", "~{flat_scratch}"() __asm__ volatile("s_mov_b64 flat_scratch, 0" : : : "flat_scratch"); -// CHECK: tail call void asm sideeffect "s_mov_b32 flat_scratch_lo, 0", "~{flat_scratch_lo}"() +// CHECK: tail call{{.*}} void asm sideeffect "s_mov_b32 flat_scratch_lo, 0", "~{flat_scratch_lo}"() __asm__ volatile("s_mov_b32 flat_scratch_lo, 0" : : : "flat_scratch_lo"); -// CHECK: tail call void asm sideeffect "s_mov_b32 flat_scratch_hi, 0", "~{flat_scratch_hi}"() +// CHECK: tail call{{.*}} void asm sideeffect "s_mov_b32 flat_scratch_hi, 0", "~{flat_scratch_hi}"() __asm__ volatile("s_mov_b32 flat_scratch_hi, 0" : : : "flat_scratch_hi"); } diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx10.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx10.cl index 3c40370e7f10..f30776a8bb85 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx10.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx10.cl @@ -2,44 +2,45 @@ // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1010 -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1011 -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1012 -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm -o - %s | FileCheck %s typedef unsigned int uint; typedef unsigned long ulong; // CHECK-LABEL: @test_permlane16( -// CHECK: call i32 @llvm.amdgcn.permlane16(i32 %a, i32 %b, i32 %c, i32 %d, i1 false, i1 false) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.permlane16(i32 %a, i32 %b, i32 %c, i32 %d, i1 false, i1 false) void test_permlane16(global uint* out, uint a, uint b, uint c, uint d) { *out = __builtin_amdgcn_permlane16(a, b, c, d, 0, 0); } // CHECK-LABEL: @test_permlanex16( -// CHECK: call i32 @llvm.amdgcn.permlanex16(i32 %a, i32 %b, i32 %c, i32 %d, i1 false, i1 false) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.permlanex16(i32 %a, i32 %b, i32 %c, i32 %d, i1 false, i1 false) void test_permlanex16(global uint* out, uint a, uint b, uint c, uint d) { *out = __builtin_amdgcn_permlanex16(a, b, c, d, 0, 0); } // CHECK-LABEL: @test_mov_dpp8( -// CHECK: call i32 @llvm.amdgcn.mov.dpp8.i32(i32 %a, i32 1) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.mov.dpp8.i32(i32 %a, i32 1) void test_mov_dpp8(global uint* out, uint a) { *out = __builtin_amdgcn_mov_dpp8(a, 1); } // CHECK-LABEL: @test_s_memtime -// CHECK: call i64 @llvm.amdgcn.s.memtime() +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.s.memtime() void test_s_memtime(global ulong* out) { *out = __builtin_amdgcn_s_memtime(); } // CHECK-LABEL: @test_groupstaticsize -// CHECK: call i32 @llvm.amdgcn.groupstaticsize() +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.groupstaticsize() void test_groupstaticsize(global uint* out) { *out = __builtin_amdgcn_groupstaticsize(); } // CHECK-LABEL: @test_ballot_wave32( -// CHECK: call i32 @llvm.amdgcn.ballot.i32(i1 %{{.+}}) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ballot.i32(i1 %{{.+}}) void test_ballot_wave32(global uint* out, int a, int b) { *out = __builtin_amdgcn_ballot_w32(a == b); diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl index 66061786cca6..868b5bed0c95 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl @@ -6,6 +6,7 @@ // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1150 -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1151 -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1152 -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm -o - %s | FileCheck %s typedef unsigned int uint; typedef unsigned long ulong; @@ -13,19 +14,19 @@ typedef uint uint2 __attribute__((ext_vector_type(2))); typedef uint uint4 __attribute__((ext_vector_type(4))); // CHECK-LABEL: @test_s_sendmsg_rtn( -// CHECK: call i32 @llvm.amdgcn.s.sendmsg.rtn.i32(i32 0) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.s.sendmsg.rtn.i32(i32 0) void test_s_sendmsg_rtn(global uint* out) { *out = __builtin_amdgcn_s_sendmsg_rtn(0); } // CHECK-LABEL: @test_s_sendmsg_rtnl( -// CHECK: call i64 @llvm.amdgcn.s.sendmsg.rtn.i64(i32 0) +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.s.sendmsg.rtn.i64(i32 0) void test_s_sendmsg_rtnl(global ulong* out) { *out = __builtin_amdgcn_s_sendmsg_rtnl(0); } // CHECK-LABEL: @test_ds_bvh_stack_rtn( -// CHECK: %0 = tail call { i32, i32 } @llvm.amdgcn.ds.bvh.stack.rtn(i32 %addr, i32 %data, <4 x i32> %data1, i32 128) +// CHECK: %0 = tail call{{.*}} { i32, i32 } @llvm.amdgcn.ds.bvh.stack.rtn(i32 %addr, i32 %data, <4 x i32> %data1, i32 128) // CHECK: %1 = extractvalue { i32, i32 } %0, 0 // CHECK: %2 = extractvalue { i32, i32 } %0, 1 // CHECK: %3 = insertelement <2 x i32> poison, i32 %1, i64 0 @@ -36,19 +37,19 @@ void test_ds_bvh_stack_rtn(global uint2* out, uint addr, uint data, uint4 data1) } // CHECK-LABEL: @test_permlane64( -// CHECK: call i32 @llvm.amdgcn.permlane64(i32 %a) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.permlane64(i32 %a) void test_permlane64(global uint* out, uint a) { *out = __builtin_amdgcn_permlane64(a); } // CHECK-LABEL: @test_s_wait_event_export_ready -// CHECK: call void @llvm.amdgcn.s.wait.event.export.ready +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.wait.event.export.ready void test_s_wait_event_export_ready() { __builtin_amdgcn_s_wait_event_export_ready(); } // CHECK-LABEL: @test_global_add_f32 -// CHECK: call float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %{{.*}}, float %{{.*}}) +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %{{.*}}, float %{{.*}}) void test_global_add_f32(float *rtn, global float *addr, float x) { *rtn = __builtin_amdgcn_global_atomic_fadd_f32(addr, x); } diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-vi.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-vi.cl index d135d33d7dec..ea2aedf8d44a 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-vi.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-vi.cl @@ -3,6 +3,7 @@ // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx900 -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1010 -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1012 -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm -o - %s | FileCheck %s #pragma OPENCL EXTENSION cl_khr_fp16 : enable @@ -10,42 +11,42 @@ typedef unsigned long ulong; typedef unsigned int uint; // CHECK-LABEL: @test_div_fixup_f16 -// CHECK: call half @llvm.amdgcn.div.fixup.f16 +// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.div.fixup.f16 void test_div_fixup_f16(global half* out, half a, half b, half c) { *out = __builtin_amdgcn_div_fixuph(a, b, c); } // CHECK-LABEL: @test_rcp_f16 -// CHECK: call half @llvm.amdgcn.rcp.f16 +// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.rcp.f16 void test_rcp_f16(global half* out, half a) { *out = __builtin_amdgcn_rcph(a); } // CHECK-LABEL: @test_sqrt_f16 -// CHECK: call half @llvm.sqrt.f16 +// CHECK: {{.*}}call{{.*}} half @llvm.{{((amdgcn.){0,1})}}sqrt.f16 void test_sqrt_f16(global half* out, half a) { *out = __builtin_amdgcn_sqrth(a); } // CHECK-LABEL: @test_rsq_f16 -// CHECK: call half @llvm.amdgcn.rsq.f16 +// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.rsq.f16 void test_rsq_f16(global half* out, half a) { *out = __builtin_amdgcn_rsqh(a); } // CHECK-LABEL: @test_sin_f16 -// CHECK: call half @llvm.amdgcn.sin.f16 +// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.sin.f16 void test_sin_f16(global half* out, half a) { *out = __builtin_amdgcn_sinh(a); } // CHECK-LABEL: @test_cos_f16 -// CHECK: call half @llvm.amdgcn.cos.f16 +// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.cos.f16 void test_cos_f16(global half* out, half a) { *out = __builtin_amdgcn_cosh(a); @@ -53,102 +54,114 @@ void test_cos_f16(global half* out, half a) // CHECK-LABEL: @test_ldexp_f16 // CHECK: [[TRUNC:%[0-9a-z]+]] = trunc i32 -// CHECK: call half @llvm.ldexp.f16.i16(half %a, i16 [[TRUNC]]) +// CHECK: {{.*}}call{{.*}} half @llvm.ldexp.f16.i16(half %a, i16 [[TRUNC]]) void test_ldexp_f16(global half* out, half a, int b) { *out = __builtin_amdgcn_ldexph(a, b); } // CHECK-LABEL: @test_frexp_mant_f16 -// CHECK: call half @llvm.amdgcn.frexp.mant.f16 +// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.frexp.mant.f16 void test_frexp_mant_f16(global half* out, half a) { *out = __builtin_amdgcn_frexp_manth(a); } // CHECK-LABEL: @test_frexp_exp_f16 -// CHECK: call i16 @llvm.amdgcn.frexp.exp.i16.f16 +// CHECK: {{.*}}call{{.*}} i16 @llvm.amdgcn.frexp.exp.i16.f16 void test_frexp_exp_f16(global short* out, half a) { *out = __builtin_amdgcn_frexp_exph(a); } // CHECK-LABEL: @test_fract_f16 -// CHECK: call half @llvm.amdgcn.fract.f16 +// CHECK: {{.*}}call{{.*}} half @llvm.amdgcn.fract.f16 void test_fract_f16(global half* out, half a) { *out = __builtin_amdgcn_fracth(a); } // CHECK-LABEL: @test_class_f16 -// CHECK: call i1 @llvm.amdgcn.class.f16 +// CHECK: {{.*}}call{{.*}} i1 @llvm.amdgcn.class.f16 void test_class_f16(global half* out, half a, int b) { *out = __builtin_amdgcn_classh(a, b); } // CHECK-LABEL: @test_s_memrealtime -// CHECK: call i64 @llvm.amdgcn.s.memrealtime() +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.s.memrealtime() void test_s_memrealtime(global ulong* out) { *out = __builtin_amdgcn_s_memrealtime(); } // CHECK-LABEL: @test_s_dcache_wb() -// CHECK: call void @llvm.amdgcn.s.dcache.wb() +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.dcache.wb() void test_s_dcache_wb() { __builtin_amdgcn_s_dcache_wb(); } // CHECK-LABEL: @test_mov_dpp -// CHECK: call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 %src, i32 0, i32 0, i32 0, i1 false) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 %src, i32 0, i32 0, i32 0, i1 false) void test_mov_dpp(global int* out, int src) { *out = __builtin_amdgcn_mov_dpp(src, 0, 0, 0, false); } // CHECK-LABEL: @test_update_dpp -// CHECK: call i32 @llvm.amdgcn.update.dpp.i32(i32 %arg1, i32 %arg2, i32 0, i32 0, i32 0, i1 false) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.update.dpp.i32(i32 %arg1, i32 %arg2, i32 0, i32 0, i32 0, i1 false) void test_update_dpp(global int* out, int arg1, int arg2) { *out = __builtin_amdgcn_update_dpp(arg1, arg2, 0, 0, 0, false); } // CHECK-LABEL: @test_ds_fadd -// CHECK: call float @llvm.amdgcn.ds.fadd.f32(ptr addrspace(3) %out, float %src, i32 0, i32 0, i1 false) +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.ds.fadd.f32(ptr addrspace(3) %out, float %src, i32 0, i32 0, i1 false) +#if !defined(__SPIRV__) void test_ds_faddf(local float *out, float src) { +#else +void test_ds_faddf(__attribute__((address_space(3))) float *out, float src) { +#endif *out = __builtin_amdgcn_ds_faddf(out, src, 0, 0, false); } // CHECK-LABEL: @test_ds_fmin -// CHECK: call float @llvm.amdgcn.ds.fmin.f32(ptr addrspace(3) %out, float %src, i32 0, i32 0, i1 false) +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.ds.fmin.f32(ptr addrspace(3) %out, float %src, i32 0, i32 0, i1 false) +#if !defined(__SPIRV__) void test_ds_fminf(local float *out, float src) { +#else +void test_ds_fminf(__attribute__((address_space(3))) float *out, float src) { +#endif *out = __builtin_amdgcn_ds_fminf(out, src, 0, 0, false); } // CHECK-LABEL: @test_ds_fmax -// CHECK: call float @llvm.amdgcn.ds.fmax.f32(ptr addrspace(3) %out, float %src, i32 0, i32 0, i1 false) +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.ds.fmax.f32(ptr addrspace(3) %out, float %src, i32 0, i32 0, i1 false) +#if !defined(__SPIRV__) void test_ds_fmaxf(local float *out, float src) { +#else +void test_ds_fmaxf(__attribute__((address_space(3))) float *out, float src) { +#endif *out = __builtin_amdgcn_ds_fmaxf(out, src, 0, 0, false); } // CHECK-LABEL: @test_s_memtime -// CHECK: call i64 @llvm.amdgcn.s.memtime() +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.s.memtime() void test_s_memtime(global ulong* out) { *out = __builtin_amdgcn_s_memtime(); } // CHECK-LABEL: @test_perm -// CHECK: call i32 @llvm.amdgcn.perm(i32 %a, i32 %b, i32 %s) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.perm(i32 %a, i32 %b, i32 %s) void test_perm(global uint* out, uint a, uint b, uint s) { *out = __builtin_amdgcn_perm(a, b, s); } // CHECK-LABEL: @test_groupstaticsize -// CHECK: call i32 @llvm.amdgcn.groupstaticsize() +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.groupstaticsize() void test_groupstaticsize(global uint* out) { *out = __builtin_amdgcn_groupstaticsize(); diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl index c2ef9ea947e9..ffc190b76db9 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl @@ -1,5 +1,7 @@ // REQUIRES: amdgpu-registered-target -// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgcn-unknown-unknown -target-cpu tahiti -emit-llvm -o - %s | FileCheck -enable-var-scope %s +// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgcn-unknown-unknown -target-cpu tahiti -emit-llvm -o - %s | FileCheck -enable-var-scope --check-prefixes=CHECK-AMDGCN,CHECK %s +// RUN: %clang_cc1 -cl-std=CL2.0 -triple spirv64-amd-amdhsa -emit-llvm -o - %s | FileCheck -enable-var-scope --check-prefix=CHECK %s + #pragma OPENCL EXTENSION cl_khr_fp64 : enable @@ -12,7 +14,7 @@ typedef ushort __attribute__((ext_vector_type(2))) ushort2; typedef uint __attribute__((ext_vector_type(4))) uint4; // CHECK-LABEL: @test_div_scale_f64 -// CHECK: call { double, i1 } @llvm.amdgcn.div.scale.f64(double %a, double %b, i1 true) +// CHECK: {{.*}}call{{.*}} { double, i1 } @llvm.amdgcn.div.scale.f64(double %a, double %b, i1 true) // CHECK-DAG: [[FLAG:%.+]] = extractvalue { double, i1 } %{{.+}}, 1 // CHECK-DAG: [[VAL:%.+]] = extractvalue { double, i1 } %{{.+}}, 0 // CHECK: [[FLAGEXT:%.+]] = zext i1 [[FLAG]] to i32 @@ -25,7 +27,7 @@ void test_div_scale_f64(global double* out, global int* flagout, double a, doubl } // CHECK-LABEL: @test_div_scale_f32( -// CHECK: call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true) +// CHECK: {{.*}}call{{.*}} { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true) // CHECK-DAG: [[FLAG:%.+]] = extractvalue { float, i1 } %{{.+}}, 1 // CHECK-DAG: [[VAL:%.+]] = extractvalue { float, i1 } %{{.+}}, 0 // CHECK: [[FLAGEXT:%.+]] = zext i1 [[FLAG]] to i8 @@ -38,7 +40,7 @@ void test_div_scale_f32(global float* out, global bool* flagout, float a, float } // CHECK-LABEL: @test_div_scale_f32_global_ptr( -// CHECK: call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true) +// CHECK: {{.*}}call{{.*}} { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true) // CHECK-DAG: [[FLAG:%.+]] = extractvalue { float, i1 } %{{.+}}, 1 // CHECK-DAG: [[VAL:%.+]] = extractvalue { float, i1 } %{{.+}}, 0 // CHECK: [[FLAGEXT:%.+]] = zext i1 [[FLAG]] to i8 @@ -49,7 +51,7 @@ void test_div_scale_f32_global_ptr(global float* out, global int* flagout, float } // CHECK-LABEL: @test_div_scale_f32_generic_ptr( -// CHECK: call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true) +// CHECK: {{.*}}call{{.*}} { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true) // CHECK-DAG: [[FLAG:%.+]] = extractvalue { float, i1 } %{{.+}}, 1 // CHECK-DAG: [[VAL:%.+]] = extractvalue { float, i1 } %{{.+}}, 0 // CHECK: [[FLAGEXT:%.+]] = zext i1 [[FLAG]] to i8 @@ -61,360 +63,360 @@ void test_div_scale_f32_generic_ptr(global float* out, global int* flagout, floa } // CHECK-LABEL: @test_div_fmas_f32 -// CHECK: call float @llvm.amdgcn.div.fmas.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.div.fmas.f32 void test_div_fmas_f32(global float* out, float a, float b, float c, int d) { *out = __builtin_amdgcn_div_fmasf(a, b, c, d); } // CHECK-LABEL: @test_div_fmas_f64 -// CHECK: call double @llvm.amdgcn.div.fmas.f64 +// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.div.fmas.f64 void test_div_fmas_f64(global double* out, double a, double b, double c, int d) { *out = __builtin_amdgcn_div_fmas(a, b, c, d); } // CHECK-LABEL: @test_div_fixup_f32 -// CHECK: call float @llvm.amdgcn.div.fixup.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.div.fixup.f32 void test_div_fixup_f32(global float* out, float a, float b, float c) { *out = __builtin_amdgcn_div_fixupf(a, b, c); } // CHECK-LABEL: @test_div_fixup_f64 -// CHECK: call double @llvm.amdgcn.div.fixup.f64 +// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.div.fixup.f64 void test_div_fixup_f64(global double* out, double a, double b, double c) { *out = __builtin_amdgcn_div_fixup(a, b, c); } // CHECK-LABEL: @test_trig_preop_f32 -// CHECK: call float @llvm.amdgcn.trig.preop.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.trig.preop.f32 void test_trig_preop_f32(global float* out, float a, int b) { *out = __builtin_amdgcn_trig_preopf(a, b); } // CHECK-LABEL: @test_trig_preop_f64 -// CHECK: call double @llvm.amdgcn.trig.preop.f64 +// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.trig.preop.f64 void test_trig_preop_f64(global double* out, double a, int b) { *out = __builtin_amdgcn_trig_preop(a, b); } // CHECK-LABEL: @test_rcp_f32 -// CHECK: call float @llvm.amdgcn.rcp.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.rcp.f32 void test_rcp_f32(global float* out, float a) { *out = __builtin_amdgcn_rcpf(a); } // CHECK-LABEL: @test_rcp_f64 -// CHECK: call double @llvm.amdgcn.rcp.f64 +// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.rcp.f64 void test_rcp_f64(global double* out, double a) { *out = __builtin_amdgcn_rcp(a); } // CHECK-LABEL: @test_sqrt_f32 -// CHECK: call float @llvm.amdgcn.sqrt.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.{{((amdgcn.){0,1})}}sqrt.f32 void test_sqrt_f32(global float* out, float a) { *out = __builtin_amdgcn_sqrtf(a); } // CHECK-LABEL: @test_sqrt_f64 -// CHECK: call double @llvm.amdgcn.sqrt.f64 +// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.sqrt.f64 void test_sqrt_f64(global double* out, double a) { *out = __builtin_amdgcn_sqrt(a); } // CHECK-LABEL: @test_rsq_f32 -// CHECK: call float @llvm.amdgcn.rsq.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.rsq.f32 void test_rsq_f32(global float* out, float a) { *out = __builtin_amdgcn_rsqf(a); } // CHECK-LABEL: @test_rsq_f64 -// CHECK: call double @llvm.amdgcn.rsq.f64 +// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.rsq.f64 void test_rsq_f64(global double* out, double a) { *out = __builtin_amdgcn_rsq(a); } // CHECK-LABEL: @test_rsq_clamp_f32 -// CHECK: call float @llvm.amdgcn.rsq.clamp.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.rsq.clamp.f32 void test_rsq_clamp_f32(global float* out, float a) { *out = __builtin_amdgcn_rsq_clampf(a); } // CHECK-LABEL: @test_rsq_clamp_f64 -// CHECK: call double @llvm.amdgcn.rsq.clamp.f64 +// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.rsq.clamp.f64 void test_rsq_clamp_f64(global double* out, double a) { *out = __builtin_amdgcn_rsq_clamp(a); } // CHECK-LABEL: @test_sin_f32 -// CHECK: call float @llvm.amdgcn.sin.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.sin.f32 void test_sin_f32(global float* out, float a) { *out = __builtin_amdgcn_sinf(a); } // CHECK-LABEL: @test_cos_f32 -// CHECK: call float @llvm.amdgcn.cos.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.cos.f32 void test_cos_f32(global float* out, float a) { *out = __builtin_amdgcn_cosf(a); } // CHECK-LABEL: @test_log_f32 -// CHECK: call float @llvm.amdgcn.log.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.log.f32 void test_log_f32(global float* out, float a) { *out = __builtin_amdgcn_logf(a); } // CHECK-LABEL: @test_exp2_f32 -// CHECK: call float @llvm.amdgcn.exp2.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.exp2.f32 void test_exp2_f32(global float* out, float a) { *out = __builtin_amdgcn_exp2f(a); } // CHECK-LABEL: @test_log_clamp_f32 -// CHECK: call float @llvm.amdgcn.log.clamp.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.log.clamp.f32 void test_log_clamp_f32(global float* out, float a) { *out = __builtin_amdgcn_log_clampf(a); } // CHECK-LABEL: @test_ldexp_f32 -// CHECK: call float @llvm.ldexp.f32.i32 +// CHECK: {{.*}}call{{.*}} float @llvm.ldexp.f32.i32 void test_ldexp_f32(global float* out, float a, int b) { *out = __builtin_amdgcn_ldexpf(a, b); } // CHECK-LABEL: @test_ldexp_f64 -// CHECK: call double @llvm.ldexp.f64.i32 +// CHECK: {{.*}}call{{.*}} double @llvm.ldexp.f64.i32 void test_ldexp_f64(global double* out, double a, int b) { *out = __builtin_amdgcn_ldexp(a, b); } // CHECK-LABEL: @test_frexp_mant_f32 -// CHECK: call float @llvm.amdgcn.frexp.mant.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.frexp.mant.f32 void test_frexp_mant_f32(global float* out, float a) { *out = __builtin_amdgcn_frexp_mantf(a); } // CHECK-LABEL: @test_frexp_mant_f64 -// CHECK: call double @llvm.amdgcn.frexp.mant.f64 +// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.frexp.mant.f64 void test_frexp_mant_f64(global double* out, double a) { *out = __builtin_amdgcn_frexp_mant(a); } // CHECK-LABEL: @test_frexp_exp_f32 -// CHECK: call i32 @llvm.amdgcn.frexp.exp.i32.f32 +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.frexp.exp.i32.f32 void test_frexp_exp_f32(global int* out, float a) { *out = __builtin_amdgcn_frexp_expf(a); } // CHECK-LABEL: @test_frexp_exp_f64 -// CHECK: call i32 @llvm.amdgcn.frexp.exp.i32.f64 +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.frexp.exp.i32.f64 void test_frexp_exp_f64(global int* out, double a) { *out = __builtin_amdgcn_frexp_exp(a); } // CHECK-LABEL: @test_fract_f32 -// CHECK: call float @llvm.amdgcn.fract.f32 +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.fract.f32 void test_fract_f32(global int* out, float a) { *out = __builtin_amdgcn_fractf(a); } // CHECK-LABEL: @test_fract_f64 -// CHECK: call double @llvm.amdgcn.fract.f64 +// CHECK: {{.*}}call{{.*}} double @llvm.amdgcn.fract.f64 void test_fract_f64(global int* out, double a) { *out = __builtin_amdgcn_fract(a); } // CHECK-LABEL: @test_lerp -// CHECK: call i32 @llvm.amdgcn.lerp +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.lerp void test_lerp(global int* out, int a, int b, int c) { *out = __builtin_amdgcn_lerp(a, b, c); } // CHECK-LABEL: @test_sicmp_i32 -// CHECK: call i64 @llvm.amdgcn.icmp.i64.i32(i32 %a, i32 %b, i32 32) +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.icmp.i64.i32(i32 %a, i32 %b, i32 32) void test_sicmp_i32(global ulong* out, int a, int b) { *out = __builtin_amdgcn_sicmp(a, b, 32); } // CHECK-LABEL: @test_uicmp_i32 -// CHECK: call i64 @llvm.amdgcn.icmp.i64.i32(i32 %a, i32 %b, i32 32) +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.icmp.i64.i32(i32 %a, i32 %b, i32 32) void test_uicmp_i32(global ulong* out, uint a, uint b) { *out = __builtin_amdgcn_uicmp(a, b, 32); } // CHECK-LABEL: @test_sicmp_i64 -// CHECK: call i64 @llvm.amdgcn.icmp.i64.i64(i64 %a, i64 %b, i32 38) +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.icmp.i64.i64(i64 %a, i64 %b, i32 38) void test_sicmp_i64(global ulong* out, long a, long b) { *out = __builtin_amdgcn_sicmpl(a, b, 39-1); } // CHECK-LABEL: @test_uicmp_i64 -// CHECK: call i64 @llvm.amdgcn.icmp.i64.i64(i64 %a, i64 %b, i32 35) +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.icmp.i64.i64(i64 %a, i64 %b, i32 35) void test_uicmp_i64(global ulong* out, ulong a, ulong b) { *out = __builtin_amdgcn_uicmpl(a, b, 30+5); } // CHECK-LABEL: @test_ds_swizzle -// CHECK: call i32 @llvm.amdgcn.ds.swizzle(i32 %a, i32 32) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.swizzle(i32 %a, i32 32) void test_ds_swizzle(global int* out, int a) { *out = __builtin_amdgcn_ds_swizzle(a, 32); } // CHECK-LABEL: @test_ds_permute -// CHECK: call i32 @llvm.amdgcn.ds.permute(i32 %a, i32 %b) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.permute(i32 %a, i32 %b) void test_ds_permute(global int* out, int a, int b) { out[0] = __builtin_amdgcn_ds_permute(a, b); } // CHECK-LABEL: @test_ds_bpermute -// CHECK: call i32 @llvm.amdgcn.ds.bpermute(i32 %a, i32 %b) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.bpermute(i32 %a, i32 %b) void test_ds_bpermute(global int* out, int a, int b) { *out = __builtin_amdgcn_ds_bpermute(a, b); } // CHECK-LABEL: @test_readfirstlane -// CHECK: call i32 @llvm.amdgcn.readfirstlane(i32 %a) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.readfirstlane(i32 %a) void test_readfirstlane(global int* out, int a) { *out = __builtin_amdgcn_readfirstlane(a); } // CHECK-LABEL: @test_readlane -// CHECK: call i32 @llvm.amdgcn.readlane(i32 %a, i32 %b) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.readlane(i32 %a, i32 %b) void test_readlane(global int* out, int a, int b) { *out = __builtin_amdgcn_readlane(a, b); } // CHECK-LABEL: @test_fcmp_f32 -// CHECK: call i64 @llvm.amdgcn.fcmp.i64.f32(float %a, float %b, i32 5) +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.fcmp.i64.f32(float %a, float %b, i32 5) void test_fcmp_f32(global ulong* out, float a, float b) { *out = __builtin_amdgcn_fcmpf(a, b, 5); } // CHECK-LABEL: @test_fcmp_f64 -// CHECK: call i64 @llvm.amdgcn.fcmp.i64.f64(double %a, double %b, i32 6) +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.fcmp.i64.f64(double %a, double %b, i32 6) void test_fcmp_f64(global ulong* out, double a, double b) { *out = __builtin_amdgcn_fcmp(a, b, 3+3); } // CHECK-LABEL: @test_class_f32 -// CHECK: call i1 @llvm.amdgcn.class.f32 +// CHECK: {{.*}}call{{.*}} i1 @llvm.amdgcn.class.f32 void test_class_f32(global float* out, float a, int b) { *out = __builtin_amdgcn_classf(a, b); } // CHECK-LABEL: @test_class_f64 -// CHECK: call i1 @llvm.amdgcn.class.f64 +// CHECK: {{.*}}call{{.*}} i1 @llvm.amdgcn.class.f64 void test_class_f64(global double* out, double a, int b) { *out = __builtin_amdgcn_class(a, b); } // CHECK-LABEL: @test_buffer_wbinvl1 -// CHECK: call void @llvm.amdgcn.buffer.wbinvl1( +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.buffer.wbinvl1( void test_buffer_wbinvl1() { __builtin_amdgcn_buffer_wbinvl1(); } // CHECK-LABEL: @test_s_dcache_inv -// CHECK: call void @llvm.amdgcn.s.dcache.inv( +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.dcache.inv( void test_s_dcache_inv() { __builtin_amdgcn_s_dcache_inv(); } // CHECK-LABEL: @test_s_waitcnt -// CHECK: call void @llvm.amdgcn.s.waitcnt( +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.waitcnt( void test_s_waitcnt() { __builtin_amdgcn_s_waitcnt(0); } // CHECK-LABEL: @test_s_sendmsg -// CHECK: call void @llvm.amdgcn.s.sendmsg( +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.sendmsg( void test_s_sendmsg() { __builtin_amdgcn_s_sendmsg(1, 0); } // CHECK-LABEL: @test_s_sendmsg_var -// CHECK: call void @llvm.amdgcn.s.sendmsg( +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.sendmsg( void test_s_sendmsg_var(int in) { __builtin_amdgcn_s_sendmsg(1, in); } // CHECK-LABEL: @test_s_sendmsghalt -// CHECK: call void @llvm.amdgcn.s.sendmsghalt( +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.sendmsghalt( void test_s_sendmsghalt() { __builtin_amdgcn_s_sendmsghalt(1, 0); } // CHECK-LABEL: @test_s_sendmsghalt -// CHECK: call void @llvm.amdgcn.s.sendmsghalt( +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.sendmsghalt( void test_s_sendmsghalt_var(int in) { __builtin_amdgcn_s_sendmsghalt(1, in); } // CHECK-LABEL: @test_s_barrier -// CHECK: call void @llvm.amdgcn.s.barrier( +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.barrier( void test_s_barrier() { __builtin_amdgcn_s_barrier(); } // CHECK-LABEL: @test_wave_barrier -// CHECK: call void @llvm.amdgcn.wave.barrier( +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.wave.barrier( void test_wave_barrier() { __builtin_amdgcn_wave_barrier(); } // CHECK-LABEL: @test_sched_barrier -// CHECK: call void @llvm.amdgcn.sched.barrier(i32 0) -// CHECK: call void @llvm.amdgcn.sched.barrier(i32 1) -// CHECK: call void @llvm.amdgcn.sched.barrier(i32 4) -// CHECK: call void @llvm.amdgcn.sched.barrier(i32 15) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.barrier(i32 0) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.barrier(i32 1) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.barrier(i32 4) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.barrier(i32 15) void test_sched_barrier() { __builtin_amdgcn_sched_barrier(0); @@ -424,10 +426,10 @@ void test_sched_barrier() } // CHECK-LABEL: @test_sched_group_barrier -// CHECK: call void @llvm.amdgcn.sched.group.barrier(i32 0, i32 1, i32 2) -// CHECK: call void @llvm.amdgcn.sched.group.barrier(i32 1, i32 2, i32 4) -// CHECK: call void @llvm.amdgcn.sched.group.barrier(i32 4, i32 8, i32 16) -// CHECK: call void @llvm.amdgcn.sched.group.barrier(i32 15, i32 10000, i32 -1) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.group.barrier(i32 0, i32 1, i32 2) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.group.barrier(i32 1, i32 2, i32 4) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.group.barrier(i32 4, i32 8, i32 16) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.sched.group.barrier(i32 15, i32 10000, i32 -1) void test_sched_group_barrier() { __builtin_amdgcn_sched_group_barrier(0, 1, 2); @@ -437,10 +439,10 @@ void test_sched_group_barrier() } // CHECK-LABEL: @test_iglp_opt -// CHECK: call void @llvm.amdgcn.iglp.opt(i32 0) -// CHECK: call void @llvm.amdgcn.iglp.opt(i32 1) -// CHECK: call void @llvm.amdgcn.iglp.opt(i32 4) -// CHECK: call void @llvm.amdgcn.iglp.opt(i32 15) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.iglp.opt(i32 0) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.iglp.opt(i32 1) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.iglp.opt(i32 4) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.iglp.opt(i32 15) void test_iglp_opt() { __builtin_amdgcn_iglp_opt(0); @@ -450,8 +452,8 @@ void test_iglp_opt() } // CHECK-LABEL: @test_s_sleep -// CHECK: call void @llvm.amdgcn.s.sleep(i32 1) -// CHECK: call void @llvm.amdgcn.s.sleep(i32 15) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.sleep(i32 1) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.sleep(i32 15) void test_s_sleep() { __builtin_amdgcn_s_sleep(1); @@ -459,8 +461,8 @@ void test_s_sleep() } // CHECK-LABEL: @test_s_incperflevel -// CHECK: call void @llvm.amdgcn.s.incperflevel(i32 1) -// CHECK: call void @llvm.amdgcn.s.incperflevel(i32 15) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.incperflevel(i32 1) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.incperflevel(i32 15) void test_s_incperflevel() { __builtin_amdgcn_s_incperflevel(1); @@ -468,8 +470,8 @@ void test_s_incperflevel() } // CHECK-LABEL: @test_s_decperflevel -// CHECK: call void @llvm.amdgcn.s.decperflevel(i32 1) -// CHECK: call void @llvm.amdgcn.s.decperflevel(i32 15) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.decperflevel(i32 1) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.decperflevel(i32 15) void test_s_decperflevel() { __builtin_amdgcn_s_decperflevel(1); @@ -477,8 +479,8 @@ void test_s_decperflevel() } // CHECK-LABEL: @test_s_setprio -// CHECK: call void @llvm.amdgcn.s.setprio(i16 0) -// CHECK: call void @llvm.amdgcn.s.setprio(i16 3) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.setprio(i16 0) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.setprio(i16 3) void test_s_setprio() { __builtin_amdgcn_s_setprio(0); @@ -486,47 +488,47 @@ void test_s_setprio() } // CHECK-LABEL: @test_cubeid( -// CHECK: call float @llvm.amdgcn.cubeid(float %a, float %b, float %c) +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.cubeid(float %a, float %b, float %c) void test_cubeid(global float* out, float a, float b, float c) { *out = __builtin_amdgcn_cubeid(a, b, c); } // CHECK-LABEL: @test_cubesc( -// CHECK: call float @llvm.amdgcn.cubesc(float %a, float %b, float %c) +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.cubesc(float %a, float %b, float %c) void test_cubesc(global float* out, float a, float b, float c) { *out = __builtin_amdgcn_cubesc(a, b, c); } // CHECK-LABEL: @test_cubetc( -// CHECK: call float @llvm.amdgcn.cubetc(float %a, float %b, float %c) +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.cubetc(float %a, float %b, float %c) void test_cubetc(global float* out, float a, float b, float c) { *out = __builtin_amdgcn_cubetc(a, b, c); } // CHECK-LABEL: @test_cubema( -// CHECK: call float @llvm.amdgcn.cubema(float %a, float %b, float %c) +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.cubema(float %a, float %b, float %c) void test_cubema(global float* out, float a, float b, float c) { *out = __builtin_amdgcn_cubema(a, b, c); } // CHECK-LABEL: @test_read_exec( -// CHECK: call i64 @llvm.amdgcn.ballot.i64(i1 true) +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.ballot.i64(i1 true) void test_read_exec(global ulong* out) { *out = __builtin_amdgcn_read_exec(); } -// CHECK: declare i64 @llvm.amdgcn.ballot.i64(i1) #[[$NOUNWIND_READONLY:[0-9]+]] +// CHECK: declare i64 @llvm.amdgcn.ballot.i64(i1){{.*}} #[[$NOUNWIND_READONLY:[0-9]+]] // CHECK-LABEL: @test_read_exec_lo( -// CHECK: call i32 @llvm.amdgcn.ballot.i32(i1 true) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ballot.i32(i1 true) void test_read_exec_lo(global uint* out) { *out = __builtin_amdgcn_read_exec_lo(); } -// CHECK: declare i32 @llvm.amdgcn.ballot.i32(i1) #[[$NOUNWIND_READONLY:[0-9]+]] +// CHECK: declare i32 @llvm.amdgcn.ballot.i32(i1){{.*}} #[[$NOUNWIND_READONLY:[0-9]+]] // CHECK-LABEL: @test_read_exec_hi( -// CHECK: call i64 @llvm.amdgcn.ballot.i64(i1 true) +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.ballot.i64(i1 true) // CHECK: lshr i64 [[A:%.*]], 32 // CHECK: trunc nuw i64 [[B:%.*]] to i32 void test_read_exec_hi(global uint* out) { @@ -534,37 +536,53 @@ void test_read_exec_hi(global uint* out) { } // CHECK-LABEL: @test_dispatch_ptr -// CHECK: call align 4 dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr() +// CHECK: {{.*}}call align 4 dereferenceable(64){{.*}} ptr addrspace(4) @llvm.amdgcn.dispatch.ptr() +#if !defined(__SPIRV__) void test_dispatch_ptr(__constant unsigned char ** out) +#else +void test_dispatch_ptr(__attribute__((address_space(4))) unsigned char ** out) +#endif { *out = __builtin_amdgcn_dispatch_ptr(); } // CHECK-LABEL: @test_queue_ptr -// CHECK: call ptr addrspace(4) @llvm.amdgcn.queue.ptr() +// CHECK: {{.*}}call{{.*}} ptr addrspace(4) @llvm.amdgcn.queue.ptr() +#if !defined(__SPIRV__) void test_queue_ptr(__constant unsigned char ** out) +#else +void test_queue_ptr(__attribute__((address_space(4))) unsigned char ** out) +#endif { *out = __builtin_amdgcn_queue_ptr(); } // CHECK-LABEL: @test_kernarg_segment_ptr -// CHECK: call ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr() +// CHECK: {{.*}}call{{.*}} ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr() +#if !defined(__SPIRV__) void test_kernarg_segment_ptr(__constant unsigned char ** out) +#else +void test_kernarg_segment_ptr(__attribute__((address_space(4))) unsigned char ** out) +#endif { *out = __builtin_amdgcn_kernarg_segment_ptr(); } // CHECK-LABEL: @test_implicitarg_ptr -// CHECK: call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() +// CHECK: {{.*}}call{{.*}} ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() +#if !defined(__SPIRV__) void test_implicitarg_ptr(__constant unsigned char ** out) +#else +void test_implicitarg_ptr(__attribute__((address_space(4))) unsigned char ** out) +#endif { *out = __builtin_amdgcn_implicitarg_ptr(); } // CHECK-LABEL: @test_get_group_id( -// CHECK: tail call i32 @llvm.amdgcn.workgroup.id.x() -// CHECK: tail call i32 @llvm.amdgcn.workgroup.id.y() -// CHECK: tail call i32 @llvm.amdgcn.workgroup.id.z() +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workgroup.id.x() +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workgroup.id.y() +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workgroup.id.z() void test_get_group_id(int d, global int *out) { switch (d) { @@ -576,9 +594,9 @@ void test_get_group_id(int d, global int *out) } // CHECK-LABEL: @test_s_getreg( -// CHECK: tail call i32 @llvm.amdgcn.s.getreg(i32 0) -// CHECK: tail call i32 @llvm.amdgcn.s.getreg(i32 1) -// CHECK: tail call i32 @llvm.amdgcn.s.getreg(i32 65535) +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.s.getreg(i32 0) +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.s.getreg(i32 1) +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.s.getreg(i32 65535) void test_s_getreg(volatile global uint *out) { *out = __builtin_amdgcn_s_getreg(0); @@ -587,9 +605,9 @@ void test_s_getreg(volatile global uint *out) } // CHECK-LABEL: @test_get_local_id( -// CHECK: tail call i32 @llvm.amdgcn.workitem.id.x(), !range [[$WI_RANGE:![0-9]*]], !noundef -// CHECK: tail call i32 @llvm.amdgcn.workitem.id.y(), !range [[$WI_RANGE]], !noundef -// CHECK: tail call i32 @llvm.amdgcn.workitem.id.z(), !range [[$WI_RANGE]], !noundef +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workitem.id.x(), !range [[$WI_RANGE:![0-9]*]], !noundef +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workitem.id.y(), !range [[$WI_RANGE]], !noundef +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workitem.id.z(), !range [[$WI_RANGE]], !noundef void test_get_local_id(int d, global int *out) { switch (d) { @@ -601,7 +619,7 @@ void test_get_local_id(int d, global int *out) } // CHECK-LABEL: @test_get_workgroup_size( -// CHECK: call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() +// CHECK: {{.*}}call align 8 dereferenceable(256){{.*}} ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() // CHECK: getelementptr inbounds i8, ptr addrspace(4) %{{.*}}, i64 12 // CHECK: load i16, ptr addrspace(4) %{{.*}}, align 4, !range [[$WS_RANGE:![0-9]*]], !invariant.load{{.*}}, !noundef // CHECK: getelementptr inbounds i8, ptr addrspace(4) %{{.*}}, i64 14 @@ -619,7 +637,7 @@ void test_get_workgroup_size(int d, global int *out) } // CHECK-LABEL: @test_get_grid_size( -// CHECK: call align 4 dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr() +// CHECK: {{.*}}call align 4 dereferenceable(64){{.*}} ptr addrspace(4) @llvm.amdgcn.dispatch.ptr() // CHECK: getelementptr inbounds i8, ptr addrspace(4) %{{.*}}, i64 12 // CHECK: load i32, ptr addrspace(4) %{{.*}}, align 4, !invariant.load // CHECK: getelementptr inbounds i8, ptr addrspace(4) %{{.*}}, i64 16 @@ -637,177 +655,185 @@ void test_get_grid_size(int d, global int *out) } // CHECK-LABEL: @test_fmed3_f32 -// CHECK: call float @llvm.amdgcn.fmed3.f32( +// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.fmed3.f32( void test_fmed3_f32(global float* out, float a, float b, float c) { *out = __builtin_amdgcn_fmed3f(a, b, c); } // CHECK-LABEL: @test_s_getpc -// CHECK: call i64 @llvm.amdgcn.s.getpc() +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.s.getpc() void test_s_getpc(global ulong* out) { *out = __builtin_amdgcn_s_getpc(); } // CHECK-LABEL: @test_ds_append_lds( -// CHECK: call i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) %ptr, i1 false) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) %ptr, i1 false) kernel void test_ds_append_lds(global int* out, local int* ptr) { +#if !defined(__SPIRV__) *out = __builtin_amdgcn_ds_append(ptr); +#else + *out = __builtin_amdgcn_ds_append((__attribute__((address_space(3))) int*)(int*)ptr); +#endif } // CHECK-LABEL: @test_ds_consume_lds( -// CHECK: call i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) %ptr, i1 false) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) %ptr, i1 false) kernel void test_ds_consume_lds(global int* out, local int* ptr) { +#if !defined(__SPIRV__) *out = __builtin_amdgcn_ds_consume(ptr); +#else + *out = __builtin_amdgcn_ds_consume((__attribute__((address_space(3))) int*)(int*)ptr); +#endif } // CHECK-LABEL: @test_gws_init( -// CHECK: call void @llvm.amdgcn.ds.gws.init(i32 %value, i32 %id) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.ds.gws.init(i32 %value, i32 %id) kernel void test_gws_init(uint value, uint id) { __builtin_amdgcn_ds_gws_init(value, id); } // CHECK-LABEL: @test_gws_barrier( -// CHECK: call void @llvm.amdgcn.ds.gws.barrier(i32 %value, i32 %id) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.ds.gws.barrier(i32 %value, i32 %id) kernel void test_gws_barrier(uint value, uint id) { __builtin_amdgcn_ds_gws_barrier(value, id); } // CHECK-LABEL: @test_gws_sema_v( -// CHECK: call void @llvm.amdgcn.ds.gws.sema.v(i32 %id) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.ds.gws.sema.v(i32 %id) kernel void test_gws_sema_v(uint id) { __builtin_amdgcn_ds_gws_sema_v(id); } // CHECK-LABEL: @test_gws_sema_br( -// CHECK: call void @llvm.amdgcn.ds.gws.sema.br(i32 %value, i32 %id) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.ds.gws.sema.br(i32 %value, i32 %id) kernel void test_gws_sema_br(uint value, uint id) { __builtin_amdgcn_ds_gws_sema_br(value, id); } // CHECK-LABEL: @test_gws_sema_p( -// CHECK: call void @llvm.amdgcn.ds.gws.sema.p(i32 %id) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.ds.gws.sema.p(i32 %id) kernel void test_gws_sema_p(uint id) { __builtin_amdgcn_ds_gws_sema_p(id); } // CHECK-LABEL: @test_mbcnt_lo( -// CHECK: call i32 @llvm.amdgcn.mbcnt.lo(i32 %src0, i32 %src1) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.mbcnt.lo(i32 %src0, i32 %src1) kernel void test_mbcnt_lo(global uint* out, uint src0, uint src1) { *out = __builtin_amdgcn_mbcnt_lo(src0, src1); } // CHECK-LABEL: @test_mbcnt_hi( -// CHECK: call i32 @llvm.amdgcn.mbcnt.hi(i32 %src0, i32 %src1) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.mbcnt.hi(i32 %src0, i32 %src1) kernel void test_mbcnt_hi(global uint* out, uint src0, uint src1) { *out = __builtin_amdgcn_mbcnt_hi(src0, src1); } // CHECK-LABEL: @test_alignbit( -// CHECK: tail call i32 @llvm.fshr.i32(i32 %src0, i32 %src1, i32 %src2) +// CHECK: tail call{{.*}} i32 @llvm.fshr.i32(i32 %src0, i32 %src1, i32 %src2) kernel void test_alignbit(global uint* out, uint src0, uint src1, uint src2) { *out = __builtin_amdgcn_alignbit(src0, src1, src2); } // CHECK-LABEL: @test_alignbyte( -// CHECK: tail call i32 @llvm.amdgcn.alignbyte(i32 %src0, i32 %src1, i32 %src2) +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.alignbyte(i32 %src0, i32 %src1, i32 %src2) kernel void test_alignbyte(global uint* out, uint src0, uint src1, uint src2) { *out = __builtin_amdgcn_alignbyte(src0, src1, src2); } // CHECK-LABEL: @test_ubfe( -// CHECK: tail call i32 @llvm.amdgcn.ubfe.i32(i32 %src0, i32 %src1, i32 %src2) +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.ubfe.i32(i32 %src0, i32 %src1, i32 %src2) kernel void test_ubfe(global uint* out, uint src0, uint src1, uint src2) { *out = __builtin_amdgcn_ubfe(src0, src1, src2); } // CHECK-LABEL: @test_sbfe( -// CHECK: tail call i32 @llvm.amdgcn.sbfe.i32(i32 %src0, i32 %src1, i32 %src2) +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.sbfe.i32(i32 %src0, i32 %src1, i32 %src2) kernel void test_sbfe(global uint* out, uint src0, uint src1, uint src2) { *out = __builtin_amdgcn_sbfe(src0, src1, src2); } // CHECK-LABEL: @test_cvt_pkrtz( -// CHECK: tail call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %src0, float %src1) +// CHECK: tail call{{.*}} <2 x half> @llvm.amdgcn.cvt.pkrtz(float %src0, float %src1) kernel void test_cvt_pkrtz(global half2* out, float src0, float src1) { *out = __builtin_amdgcn_cvt_pkrtz(src0, src1); } // CHECK-LABEL: @test_cvt_pknorm_i16( -// CHECK: tail call <2 x i16> @llvm.amdgcn.cvt.pknorm.i16(float %src0, float %src1) +// CHECK: tail call{{.*}} <2 x i16> @llvm.amdgcn.cvt.pknorm.i16(float %src0, float %src1) kernel void test_cvt_pknorm_i16(global short2* out, float src0, float src1) { *out = __builtin_amdgcn_cvt_pknorm_i16(src0, src1); } // CHECK-LABEL: @test_cvt_pknorm_u16( -// CHECK: tail call <2 x i16> @llvm.amdgcn.cvt.pknorm.u16(float %src0, float %src1) +// CHECK: tail call{{.*}} <2 x i16> @llvm.amdgcn.cvt.pknorm.u16(float %src0, float %src1) kernel void test_cvt_pknorm_u16(global ushort2* out, float src0, float src1) { *out = __builtin_amdgcn_cvt_pknorm_u16(src0, src1); } // CHECK-LABEL: @test_cvt_pk_i16( -// CHECK: tail call <2 x i16> @llvm.amdgcn.cvt.pk.i16(i32 %src0, i32 %src1) +// CHECK: tail call{{.*}} <2 x i16> @llvm.amdgcn.cvt.pk.i16(i32 %src0, i32 %src1) kernel void test_cvt_pk_i16(global short2* out, int src0, int src1) { *out = __builtin_amdgcn_cvt_pk_i16(src0, src1); } // CHECK-LABEL: @test_cvt_pk_u16( -// CHECK: tail call <2 x i16> @llvm.amdgcn.cvt.pk.u16(i32 %src0, i32 %src1) +// CHECK: tail call{{.*}} <2 x i16> @llvm.amdgcn.cvt.pk.u16(i32 %src0, i32 %src1) kernel void test_cvt_pk_u16(global ushort2* out, uint src0, uint src1) { *out = __builtin_amdgcn_cvt_pk_u16(src0, src1); } // CHECK-LABEL: @test_cvt_pk_u8_f32 -// CHECK: tail call i32 @llvm.amdgcn.cvt.pk.u8.f32(float %src0, i32 %src1, i32 %src2) +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.cvt.pk.u8.f32(float %src0, i32 %src1, i32 %src2) kernel void test_cvt_pk_u8_f32(global uint* out, float src0, uint src1, uint src2) { *out = __builtin_amdgcn_cvt_pk_u8_f32(src0, src1, src2); } // CHECK-LABEL: @test_sad_u8( -// CHECK: tail call i32 @llvm.amdgcn.sad.u8(i32 %src0, i32 %src1, i32 %src2) +// CHECK: tail call{{.*}} i32 @llvm.amdgcn.sad.u8(i32 %src0, i32 %src1, i32 %src2) kernel void test_sad_u8(global uint* out, uint src0, uint src1, uint src2) { *out = __builtin_amdgcn_sad_u8(src0, src1, src2); } // CHECK-LABEL: test_msad_u8( -// CHECK: call i32 @llvm.amdgcn.msad.u8(i32 %src0, i32 %src1, i32 %src2) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.msad.u8(i32 %src0, i32 %src1, i32 %src2) kernel void test_msad_u8(global uint* out, uint src0, uint src1, uint src2) { *out = __builtin_amdgcn_msad_u8(src0, src1, src2); } // CHECK-LABEL: test_sad_hi_u8( -// CHECK: call i32 @llvm.amdgcn.sad.hi.u8(i32 %src0, i32 %src1, i32 %src2) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.sad.hi.u8(i32 %src0, i32 %src1, i32 %src2) kernel void test_sad_hi_u8(global uint* out, uint src0, uint src1, uint src2) { *out = __builtin_amdgcn_sad_hi_u8(src0, src1, src2); } // CHECK-LABEL: @test_sad_u16( -// CHECK: call i32 @llvm.amdgcn.sad.u16(i32 %src0, i32 %src1, i32 %src2) +// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.sad.u16(i32 %src0, i32 %src1, i32 %src2) kernel void test_sad_u16(global uint* out, uint src0, uint src1, uint src2) { *out = __builtin_amdgcn_sad_u16(src0, src1, src2); } // CHECK-LABEL: @test_qsad_pk_u16_u8( -// CHECK: call i64 @llvm.amdgcn.qsad.pk.u16.u8(i64 %src0, i32 %src1, i64 %src2) +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.qsad.pk.u16.u8(i64 %src0, i32 %src1, i64 %src2) kernel void test_qsad_pk_u16_u8(global ulong* out, ulong src0, uint src1, ulong src2) { *out = __builtin_amdgcn_qsad_pk_u16_u8(src0, src1, src2); } // CHECK-LABEL: @test_mqsad_pk_u16_u8( -// CHECK: call i64 @llvm.amdgcn.mqsad.pk.u16.u8(i64 %src0, i32 %src1, i64 %src2) +// CHECK: {{.*}}call{{.*}} i64 @llvm.amdgcn.mqsad.pk.u16.u8(i64 %src0, i32 %src1, i64 %src2) kernel void test_mqsad_pk_u16_u8(global ulong* out, ulong src0, uint src1, ulong src2) { *out = __builtin_amdgcn_mqsad_pk_u16_u8(src0, src1, src2); } // CHECK-LABEL: test_mqsad_u32_u8( -// CHECK: call <4 x i32> @llvm.amdgcn.mqsad.u32.u8(i64 %src0, i32 %src1, <4 x i32> %src2) +// CHECK: {{.*}}call{{.*}} <4 x i32> @llvm.amdgcn.mqsad.u32.u8(i64 %src0, i32 %src1, <4 x i32> %src2) kernel void test_mqsad_u32_u8(global uint4* out, ulong src0, uint src1, uint4 src2) { *out = __builtin_amdgcn_mqsad_u32_u8(src0, src1, src2); } // CHECK-LABEL: test_s_setreg( -// CHECK: call void @llvm.amdgcn.s.setreg(i32 8193, i32 %val) +// CHECK: {{.*}}call{{.*}} void @llvm.amdgcn.s.setreg(i32 8193, i32 %val) kernel void test_s_setreg(uint val) { __builtin_amdgcn_s_setreg(8193, val); } @@ -835,31 +861,33 @@ void test_atomic_inc_dec(local uint *lptr, global uint *gptr, uint val) { // CHECK-LABEL test_wavefrontsize( unsigned test_wavefrontsize() { - // CHECK: call i32 @llvm.amdgcn.wavefrontsize() + // CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.wavefrontsize() return __builtin_amdgcn_wavefrontsize(); } // CHECK-LABEL test_flt_rounds( unsigned test_flt_rounds() { - // CHECK: call i32 @llvm.get.rounding() + // CHECK: {{.*}}call{{.*}} i32 @llvm.get.rounding() unsigned mode = __builtin_flt_rounds(); - // CHECK: call void @llvm.set.rounding(i32 %0) +#if !defined(__SPIRV__) + // CHECK-AMDGCN: call void @llvm.set.rounding(i32 %0) __builtin_set_flt_rounds(mode); +#endif return mode; } // CHECK-LABEL test_get_fpenv( unsigned long test_get_fpenv() { - // CHECK: call i64 @llvm.get.fpenv.i64() + // CHECK: {{.*}}call{{.*}} i64 @llvm.get.fpenv.i64() return __builtin_amdgcn_get_fpenv(); } // CHECK-LABEL test_set_fpenv( void test_set_fpenv(unsigned long env) { - // CHECK: call void @llvm.set.fpenv.i64(i64 %[[ENV:.+]]) + // CHECK: {{.*}}call{{.*}} void @llvm.set.fpenv.i64(i64 %[[ENV:.+]]) __builtin_amdgcn_set_fpenv(env); } diff --git a/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl b/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl index 259c12384f2c..5ebb0ea0c33c 100644 --- a/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl +++ b/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl @@ -1,11 +1,12 @@ // REQUIRES: amdgpu-registered-target // RUN: %clang_cc1 -emit-llvm -O0 -o - -triple amdgcn %s | FileCheck %s +// RUN: %clang_cc1 -emit-llvm -O0 -o - -triple spirv64-amd-amdhsa %s | FileCheck %s typedef float float32 __attribute__((ext_vector_type(32))); kernel void test_long(int arg0) { long v15_16; - // CHECK: call i64 asm sideeffect "v_lshlrev_b64 v[15:16], 0, $0", "={v[15:16]},v" + // CHECK: call{{.*}} i64 asm sideeffect "v_lshlrev_b64 v[15:16], 0, $0", "={v[15:16]},v" __asm volatile("v_lshlrev_b64 v[15:16], 0, %0" : "={v[15:16]}"(v15_16) : "v"(arg0)); } @@ -14,7 +15,7 @@ kernel void test_agpr() { float reg_a; float reg_b; float32 reg_c; - // CHECK: call <32 x float> asm "v_mfma_f32_32x32x1f32 $0, $1, $2, $3", "=a,v,v,a,~{a0},~{a1},~{a2},~{a3},~{a4},~{a5},~{a6},~{a7},~{a8},~{a9},~{a10},~{a11},~{a12},~{a13},~{a14},~{a15},~{a16},~{a17},~{a18},~{a19},~{a20},~{a21},~{a22},~{a23},~{a24},~{a25},~{a26},~{a27},~{a28},~{a29},~{a30},~{a31}" + // CHECK: call{{.*}} <32 x float> asm "v_mfma_f32_32x32x1f32 $0, $1, $2, $3", "=a,v,v,a,~{a0},~{a1},~{a2},~{a3},~{a4},~{a5},~{a6},~{a7},~{a8},~{a9},~{a10},~{a11},~{a12},~{a13},~{a14},~{a15},~{a16},~{a17},~{a18},~{a19},~{a20},~{a21},~{a22},~{a23},~{a24},~{a25},~{a26},~{a27},~{a28},~{a29},~{a30},~{a31}" __asm ("v_mfma_f32_32x32x1f32 %0, %1, %2, %3" : "=a"(acc_c) : "v"(reg_a), "v"(reg_b), "a"(reg_c) @@ -23,12 +24,12 @@ kernel void test_agpr() { "a16", "a17", "a18", "a19", "a20", "a21", "a22", "a23", "a24", "a25", "a26", "a27", "a28", "a29", "a30", "a31"); - // CHECK: call <32 x float> asm sideeffect "v_mfma_f32_32x32x1f32 a[0:31], $0, $1, a[0:31]", "={a[0:31]},v,v,{a[0:31]}" + // CHECK: call{{.*}} <32 x float> asm sideeffect "v_mfma_f32_32x32x1f32 a[0:31], $0, $1, a[0:31]", "={a[0:31]},v,v,{a[0:31]}" __asm volatile("v_mfma_f32_32x32x1f32 a[0:31], %0, %1, a[0:31]" : "={a[0:31]}"(acc_c) : "v"(reg_a),"v"(reg_b), "{a[0:31]}"(reg_c)); - // CHECK: call float asm "v_accvgpr_read_b32 $0, $1", "={a1},{a1}" + // CHECK: call{{.*}} float asm "v_accvgpr_read_b32 $0, $1", "={a1},{a1}" __asm ("v_accvgpr_read_b32 %0, %1" : "={a1}"(reg_a) : "{a1}"(reg_b)); @@ -37,13 +38,13 @@ kernel void test_agpr() { kernel void test_constraint_DA() { const long x = 0x200000001; int res; - // CHECK: call i32 asm sideeffect "v_mov_b32 $0, $1 & 0xFFFFFFFF", "=v,^DA"(i64 8589934593) + // CHECK: call{{.*}} i32 asm sideeffect "v_mov_b32 $0, $1 & 0xFFFFFFFF", "=v,^DA"(i64 8589934593) __asm volatile("v_mov_b32 %0, %1 & 0xFFFFFFFF" : "=v"(res) : "DA"(x)); } kernel void test_constraint_DB() { const long x = 0x200000001; int res; - // CHECK: call i32 asm sideeffect "v_mov_b32 $0, $1 & 0xFFFFFFFF", "=v,^DB"(i64 8589934593) + // CHECK: call{{.*}} i32 asm sideeffect "v_mov_b32 $0, $1 & 0xFFFFFFFF", "=v,^DB"(i64 8589934593) __asm volatile("v_mov_b32 %0, %1 & 0xFFFFFFFF" : "=v"(res) : "DB"(x)); } diff --git a/clang/test/Preprocessor/hash_builtin.cpp b/clang/test/Preprocessor/hash_builtin.cpp index 77d186c7883f..018b71eca418 100644 --- a/clang/test/Preprocessor/hash_builtin.cpp +++ b/clang/test/Preprocessor/hash_builtin.cpp @@ -1,11 +1,14 @@ // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx906 -E %s -o - | FileCheck %s +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -E %s -o - | FileCheck %s --check-prefix=SPIRV-AMDGCN // CHECK: has_s_memtime_inst +// SPIRV-AMDGCN: has_s_memtime_inst #if __has_builtin(__builtin_amdgcn_s_memtime) int has_s_memtime_inst; #endif // CHECK-NOT: has_gfx10_inst +// SPIRV-AMDGCN: has_gfx10_inst #if __has_builtin(__builtin_amdgcn_mov_dpp8) int has_gfx10_inst; #endif diff --git a/clang/test/Preprocessor/predefined-macros-no-warnings.c b/clang/test/Preprocessor/predefined-macros-no-warnings.c index e0617f8de4da..722e3e77214b 100644 --- a/clang/test/Preprocessor/predefined-macros-no-warnings.c +++ b/clang/test/Preprocessor/predefined-macros-no-warnings.c @@ -173,6 +173,7 @@ // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple spir64 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple spirv32 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple spirv64 +// RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple spirv64-amd-amdhsa // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple wasm32 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple wasm32-wasi // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple wasm32-emscripten diff --git a/clang/test/Preprocessor/predefined-macros.c b/clang/test/Preprocessor/predefined-macros.c index c4a9672f0814..7f036bff401c 100644 --- a/clang/test/Preprocessor/predefined-macros.c +++ b/clang/test/Preprocessor/predefined-macros.c @@ -236,6 +236,16 @@ // CHECK-SPIRV64-DAG: #define __SPIRV64__ 1 // CHECK-SPIRV64-NOT: #define __SPIRV32__ 1 +// RUN: %clang_cc1 %s -E -dM -o - -x cl -triple spirv64-amd-amdhsa \ +// RUN: | FileCheck -match-full-lines %s --check-prefix=CHECK-SPIRV64-AMDGCN +// CHECK-SPIRV64-AMDGCN-DAG: #define __IMAGE_SUPPORT__ 1 +// CHECK-SPIRV64-AMDGCN-DAG: #define __SPIRV__ 1 +// CHECK-SPIRV64-AMDGCN-DAG: #define __SPIRV64__ 1 +// CHECK-SPIRV64-AMDGCN-DAG: #define __AMD__ 1 +// CHECK-SPIRV64-AMDGCN-DAG: #define __AMDGCN__ 1 +// CHECK-SPIRV64-AMDGCN-DAG: #define __AMDGPU__ 1 +// CHECK-SPIRV64-AMDGCN-NOT: #define __SPIRV32__ 1 + // RUN: %clang_cc1 %s -E -dM -o - -x hip -triple x86_64-unknown-linux-gnu \ // RUN: | FileCheck -match-full-lines %s --check-prefix=CHECK-HIP // CHECK-HIP: #define __HIPCC__ 1 diff --git a/clang/test/Sema/builtin-spirv-amdgcn-atomic-inc-dec-failure.cpp b/clang/test/Sema/builtin-spirv-amdgcn-atomic-inc-dec-failure.cpp new file mode 100644 index 000000000000..2b8fac72847d --- /dev/null +++ b/clang/test/Sema/builtin-spirv-amdgcn-atomic-inc-dec-failure.cpp @@ -0,0 +1,25 @@ +// RUN: %clang_cc1 %s -x hip -fcuda-is-device -o - \ +// RUN: -triple=spirv64-amd-amdhsa -fsyntax-only \ +// RUN: -verify=dev +// RUN: %clang_cc1 %s -x hip -triple x86_64 -o - \ +// RUN: -aux-triple spirv64-amd-amdhsa -fsyntax-only \ +// RUN: -verify=host + +// dev-no-diagnostics + +void test_host() { + __UINT32_TYPE__ val32; + __UINT64_TYPE__ val64; + + // host-error@+1 {{reference to __device__ function '__builtin_amdgcn_atomic_inc32' in __host__ function}} + val32 = __builtin_amdgcn_atomic_inc32(&val32, val32, __ATOMIC_SEQ_CST, ""); + + // host-error@+1 {{reference to __device__ function '__builtin_amdgcn_atomic_inc64' in __host__ function}} + val64 = __builtin_amdgcn_atomic_inc64(&val64, val64, __ATOMIC_SEQ_CST, ""); + + // host-error@+1 {{reference to __device__ function '__builtin_amdgcn_atomic_dec32' in __host__ function}} + val32 = __builtin_amdgcn_atomic_dec32(&val32, val32, __ATOMIC_SEQ_CST, ""); + + // host-error@+1 {{reference to __device__ function '__builtin_amdgcn_atomic_dec64' in __host__ function}} + val64 = __builtin_amdgcn_atomic_dec64(&val64, val64, __ATOMIC_SEQ_CST, ""); +} diff --git a/clang/test/Sema/inline-asm-validate-spirv-amdgcn.cl b/clang/test/Sema/inline-asm-validate-spirv-amdgcn.cl new file mode 100644 index 000000000000..0fb1b5f36722 --- /dev/null +++ b/clang/test/Sema/inline-asm-validate-spirv-amdgcn.cl @@ -0,0 +1,111 @@ +// REQUIRES: amdgpu-registered-target +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -fsyntax-only -verify %s + +#pragma OPENCL EXTENSION cl_khr_fp64 : enable + +kernel void test () { + + int sgpr = 0, vgpr = 0, imm = 0; + + // sgpr constraints + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "s" (imm) : ); + + __asm__ ("s_mov_b32 %0, %1" : "={s1}" (sgpr) : "{exec}" (imm) : ); + __asm__ ("s_mov_b32 %0, %1" : "={s1}" (sgpr) : "{exe" (imm) : ); // expected-error {{invalid input constraint '{exe' in asm}} + __asm__ ("s_mov_b32 %0, %1" : "={s1}" (sgpr) : "{exec" (imm) : ); // expected-error {{invalid input constraint '{exec' in asm}} + __asm__ ("s_mov_b32 %0, %1" : "={s1}" (sgpr) : "{exec}a" (imm) : ); // expected-error {{invalid input constraint '{exec}a' in asm}} + + // vgpr constraints + __asm__ ("v_mov_b32 %0, %1" : "=v" (vgpr) : "v" (imm) : ); + + // 'I' constraint (an immediate integer in the range -16 to 64) + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (imm) : ); + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (-16) : ); + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (64) : ); + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (-17) : ); // expected-error {{value '-17' out of range for constraint 'I'}} + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "I" (65) : ); // expected-error {{value '65' out of range for constraint 'I'}} + + // 'J' constraint (an immediate 16-bit signed integer) + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (imm) : ); + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (-32768) : ); + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (32767) : ); + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (-32769) : ); // expected-error {{value '-32769' out of range for constraint 'J'}} + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "J" (32768) : ); // expected-error {{value '32768' out of range for constraint 'J'}} + + // 'A' constraint (an immediate constant that can be inlined) + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "A" (imm) : ); + + // 'B' constraint (an immediate 32-bit signed integer) + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "B" (imm) : ); + + // 'C' constraint (an immediate 32-bit unsigned integer or 'A' constraint) + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "C" (imm) : ); + + // 'DA' constraint (an immediate 64-bit constant that can be split into two 'A' constants) + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "DA" (imm) : ); + + // 'DB' constraint (an immediate 64-bit constant that can be split into two 'B' constants) + __asm__ ("s_mov_b32 %0, %1" : "=s" (sgpr) : "DB" (imm) : ); + +} + +__kernel void +test_float(const __global float *a, const __global float *b, __global float *c, unsigned i) +{ + float ai = a[i]; + float bi = b[i]; + float ci; + + __asm("v_add_f32_e32 v1, v2, v3" : "={v1}"(ci) : "{v2}"(ai), "{v3}"(bi) : ); + __asm("v_add_f32_e32 v1, v2, v3" : ""(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '' in asm}} + __asm("v_add_f32_e32 v1, v2, v3" : "="(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '=' in asm}} + __asm("v_add_f32_e32 v1, v2, v3" : "={a}"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={a}' in asm}} + __asm("v_add_f32_e32 v1, v2, v3" : "={"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={' in asm}} + __asm("v_add_f32_e32 v1, v2, v3" : "={}"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={}' in asm}} + __asm("v_add_f32_e32 v1, v2, v3" : "={v"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={v' in asm}} + __asm("v_add_f32_e32 v1, v2, v3" : "={v1a}"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={v1a}' in asm}} + __asm("v_add_f32_e32 v1, v2, v3" : "={va}"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={va}' in asm}} + __asm("v_add_f32_e32 v1, v2, v3" : "={v1}a"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={v1}a' in asm}} + __asm("v_add_f32_e32 v1, v2, v3" : "={v1"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '={v1' in asm}} + __asm("v_add_f32_e32 v1, v2, v3" : "=v1}"(ci) : "{v2}"(ai), "{v3}"(bi) : ); // expected-error {{invalid output constraint '=v1}' in asm}} + + __asm("v_add_f32_e32 v1, v2, v3" : "={v[1]}"(ci) : "{v[2]}"(ai), "{v[3]}"(bi) : ); + __asm("v_add_f32_e32 v1, v2, v3" : "={v[1}"(ci) : "{v[2]}"(ai), "{v[3]}"(bi) : ); // expected-error {{invalid output constraint '={v[1}' in asm}} + __asm("v_add_f32_e32 v1, v2, v3" : "={v[1]"(ci) : "{v[2]}"(ai), "{v[3]}"(bi) : ); // expected-error {{invalid output constraint '={v[1]' in asm}} + __asm("v_add_f32_e32 v1, v2, v3" : "={v[a]}"(ci) : "{v[2]}"(ai), "{v[3]}"(bi) : ); // expected-error {{invalid output constraint '={v[a]}' in asm}} + + __asm("v_add_f32_e32 v1, v2, v3" : "=v"(ci) : "v"(ai), "v"(bi) : ); + __asm("v_add_f32_e32 v1, v2, v3" : "=v1"(ci) : "v2"(ai), "v3"(bi) : ); /// expected-error {{invalid output constraint '=v1' in asm}} + + __asm("v_add_f32_e32 v1, v2, v3" : "={v1}"(ci) : "{a}"(ai), "{v3}"(bi) : ); // expected-error {{invalid input constraint '{a}' in asm}} + __asm("v_add_f32_e32 v1, v2, v3" : "={v1}"(ci) : "{v2}"(ai), "{a}"(bi) : ); // expected-error {{invalid input constraint '{a}' in asm}} + c[i] = ci; +} + +__kernel void +test_double(const __global double *a, const __global double *b, __global double *c, unsigned i) +{ + double ai = a[i]; + double bi = b[i]; + double ci; + + __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:2]}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); + __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "=v{[1:2]}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '=v{[1:2]}' in asm}} + __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:2]a}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[1:2]a}' in asm}} + __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:2]}a"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[1:2]}a' in asm}} + __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[1:' in asm}} + __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:]}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[1:]}' in asm}} + __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[:2]}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[:2]}' in asm}} + __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:2]"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[1:2]' in asm}} + __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[1:2}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[1:2}' in asm}} + __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "={v[2:1]}"(ci) : "{v[3:4]}"(ai), "{v[5:6]}"(bi) : ); //expected-error {{invalid output constraint '={v[2:1]}' in asm}} + + __asm("v_add_f64_e64 v[1:2], v[3:4], v[5:6]" : "=v[1:2]"(ci) : "v[3:4]"(ai), "v[5:6]"(bi) : ); //expected-error {{invalid output constraint '=v[1:2]' in asm}} + + c[i] = ci; +} + +void test_long(int arg0) { + long v15_16; + __asm volatile("v_lshlrev_b64 v[15:16], 0, %0" : "={v[15:16]}"(v15_16) : "v"(arg0)); +} diff --git a/clang/test/SemaCUDA/allow-int128.cu b/clang/test/SemaCUDA/allow-int128.cu index eb7b7e7f5286..af3e8c2453ad 100644 --- a/clang/test/SemaCUDA/allow-int128.cu +++ b/clang/test/SemaCUDA/allow-int128.cu @@ -1,6 +1,9 @@ // RUN: %clang_cc1 -triple amdgcn-amd-amdhsa \ // RUN: -aux-triple x86_64-unknown-linux-gnu \ // RUN: -fcuda-is-device -verify -fsyntax-only %s +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa \ +// RUN: -aux-triple x86_64-unknown-linux-gnu \ +// RUN: -fcuda-is-device -verify -fsyntax-only %s // RUN: %clang_cc1 -triple nvptx \ // RUN: -aux-triple x86_64-unknown-linux-gnu \ // RUN: -fcuda-is-device -verify -fsyntax-only %s diff --git a/clang/test/SemaCUDA/amdgpu-f128.cu b/clang/test/SemaCUDA/amdgpu-f128.cu index 9a0212cdb93c..1f5a6553dcc4 100644 --- a/clang/test/SemaCUDA/amdgpu-f128.cu +++ b/clang/test/SemaCUDA/amdgpu-f128.cu @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -aux-triple x86_64-unknown-linux-gnu -fcuda-is-device -fsyntax-only -verify %s +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -aux-triple x86_64-unknown-linux-gnu -fcuda-is-device -fsyntax-only -verify %s // expected-no-diagnostics typedef __float128 f128_t; diff --git a/clang/test/SemaCUDA/float16.cu b/clang/test/SemaCUDA/float16.cu index bb5ed6064384..9c7faef284fe 100644 --- a/clang/test/SemaCUDA/float16.cu +++ b/clang/test/SemaCUDA/float16.cu @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -fsyntax-only -triple x86_64 -aux-triple amdgcn -verify %s +// RUN: %clang_cc1 -fsyntax-only -triple x86_64 -aux-triple spirv64-amd-amdhsa -verify %s // RUN: %clang_cc1 -fsyntax-only -triple x86_64 -aux-triple nvptx64 -verify %s // expected-no-diagnostics #include "Inputs/cuda.h" diff --git a/clang/test/SemaCUDA/fp16-arg-return.cu b/clang/test/SemaCUDA/fp16-arg-return.cu index 46d543f44445..9347491caa97 100644 --- a/clang/test/SemaCUDA/fp16-arg-return.cu +++ b/clang/test/SemaCUDA/fp16-arg-return.cu @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -o - -triple amdgcn-amd-amdhsa -fcuda-is-device -fsyntax-only -verify %s +// RUN: %clang_cc1 -o - -triple spirv64-amd-amdhsa -fcuda-is-device -fsyntax-only -verify %s // expected-no-diagnostics diff --git a/clang/test/SemaCUDA/spirv-amdgcn-atomic-ops.cu b/clang/test/SemaCUDA/spirv-amdgcn-atomic-ops.cu new file mode 100644 index 000000000000..ea1f24670ff9 --- /dev/null +++ b/clang/test/SemaCUDA/spirv-amdgcn-atomic-ops.cu @@ -0,0 +1,86 @@ +// RUN: %clang_cc1 -x hip -std=c++11 -triple spirv64-amd-amdhsa -fcuda-is-device -verify -fsyntax-only %s + +#include "Inputs/cuda.h" + +__device__ int test_hip_atomic_load(int *pi32, unsigned int *pu32, long long *pll, unsigned long long *pull, float *fp, double *dbl) { + int val = __hip_atomic_load(0); // expected-error {{too few arguments to function call, expected 3, have 1}} + val = __hip_atomic_load(0, 0, 0, 0); // expected-error {{too many arguments to function call, expected 3, have 4}} + val = __hip_atomic_load(0, 0, 0); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + val = __hip_atomic_load(pi32, 0, 0); // expected-error {{synchronization scope argument to atomic operation is invalid}} + val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_WAVEFRONT); + val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_WORKGROUP); + val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); + val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, 6); // expected-error {{synchronization scope argument to atomic operation is invalid}} + val = __hip_atomic_load(pi32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + val = __hip_atomic_load(pi32, __ATOMIC_SEQ_CST, __HIP_MEMORY_SCOPE_SINGLETHREAD); + val = __hip_atomic_load(pi32, __ATOMIC_CONSUME, __HIP_MEMORY_SCOPE_SINGLETHREAD); + val = __hip_atomic_load(pi32, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_SINGLETHREAD); + val = __hip_atomic_load(pi32, __ATOMIC_ACQ_REL, __HIP_MEMORY_SCOPE_SINGLETHREAD); // expected-warning{{memory order argument to atomic operation is invalid}} + val = __hip_atomic_load(pu32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + val = __hip_atomic_load(pll, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + val = __hip_atomic_load(pull, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + val = __hip_atomic_load(fp, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + val = __hip_atomic_load(dbl, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + return val; +} + +__device__ int test_hip_atomic_store(int *pi32, unsigned int *pu32, long long *pll, unsigned long long *pull, float *fp, double *dbl, + int i32, unsigned int u32, long long i64, unsigned long long u64, float f32, double f64) { + __hip_atomic_store(0); // expected-error {{too few arguments to function call, expected 4, have 1}} + __hip_atomic_store(0, 0, 0, 0, 0); // expected-error {{too many arguments to function call, expected 4, have 5}} + __hip_atomic_store(0, 0, 0, 0); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + __hip_atomic_store(pi32, 0, 0, 0); // expected-error {{synchronization scope argument to atomic operation is invalid}} + __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_WAVEFRONT); + __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_WORKGROUP); + __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); + __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, 6); // expected-error {{synchronization scope argument to atomic operation is invalid}} + __hip_atomic_store(pi32, 0, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(pi32, 0, __ATOMIC_SEQ_CST, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(pi32, 0, __ATOMIC_CONSUME, __HIP_MEMORY_SCOPE_SINGLETHREAD); // expected-warning{{memory order argument to atomic operation is invalid}} + __hip_atomic_store(pi32, 0, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_SINGLETHREAD); // expected-warning{{memory order argument to atomic operation is invalid}} + __hip_atomic_store(pi32, 0, __ATOMIC_ACQ_REL, __HIP_MEMORY_SCOPE_SINGLETHREAD); // expected-warning{{memory order argument to atomic operation is invalid}} + __hip_atomic_store(pi32, i32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(pi32, i32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(pu32, u32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(pll, i64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(pull, u64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(fp, f32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(dbl, f64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(pi32, u32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(pi32, i64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(pi32, u64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(pll, i32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(fp, i32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(fp, i64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(dbl, i64, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + __hip_atomic_store(dbl, i32, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + return 0; +} + +__device__ bool test_hip_atomic_cmpxchg_weak(int *ptr, int val, int desired) { + bool flag = __hip_atomic_compare_exchange_weak(0); // expected-error {{too few arguments to function call, expected 6, have 1}} + flag = __hip_atomic_compare_exchange_weak(0, 0, 0, 0, 0, 0, 0); // expected-error {{too many arguments to function call, expected 6, have 7}} + flag = __hip_atomic_compare_exchange_weak(0, 0, 0, 0, 0, 0); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + flag = __hip_atomic_compare_exchange_weak(ptr, 0, 0, 0, 0, 0); // expected-error {{synchronization scope argument to atomic operation is invalid}}, expected-warning {{null passed to a callee that requires a non-null argument}} + flag = __hip_atomic_compare_exchange_weak(ptr, 0, 0, 0, 0, __HIP_MEMORY_SCOPE_SYSTEM); // expected-warning {{null passed to a callee that requires a non-null argument}} + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_CONSUME, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_WAVEFRONT); + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_WORKGROUP); + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_SEQ_CST, __HIP_MEMORY_SCOPE_SINGLETHREAD); + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_CONSUME, __HIP_MEMORY_SCOPE_SINGLETHREAD); + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_SINGLETHREAD); + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_ACQ_REL, __HIP_MEMORY_SCOPE_SINGLETHREAD); // expected-warning {{failure memory order argument to atomic operation is invalid}} + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_CONSUME, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + flag = __hip_atomic_compare_exchange_weak(ptr, &val, desired, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SINGLETHREAD); + return flag; +} diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx908-param.cl b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx908-param.cl index bb949c0ddd10..969ff4ba9c92 100644 --- a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx908-param.cl +++ b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx908-param.cl @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -triple amdgcn-- -target-cpu gfx908 -verify -S -o - %s +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -verify -S -o - %s #pragma OPENCL EXTENSION cl_khr_fp64:enable diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx90a-param.cl b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx90a-param.cl index 701016148a89..235fa8263140 100644 --- a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx90a-param.cl +++ b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx90a-param.cl @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -triple amdgcn-- -target-cpu gfx90a -verify -S -o - %s +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -verify -S -o - %s #pragma OPENCL EXTENSION cl_khr_fp64:enable diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx940-param.cl b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx940-param.cl index b177b93938e4..0fc2304d51ce 100644 --- a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx940-param.cl +++ b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx940-param.cl @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -triple amdgcn-- -target-cpu gfx940 -verify -S -o - %s +// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -verify -S -o - %s typedef float v2f __attribute__((ext_vector_type(2))); typedef float v4f __attribute__((ext_vector_type(4))); diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst index de27f6b2372d..70865b95cb39 100644 --- a/llvm/docs/SPIRVUsage.rst +++ b/llvm/docs/SPIRVUsage.rst @@ -17,9 +17,9 @@ in `the official SPIR-V specification */``unknown`` Generic SPIR-V target without any vendor-specific settings. + ``amd`` AMDGCN SPIR-V target, with support for target specific + builtins and ASM, meant to be consumed by AMDGCN toolchains. ===================== ============================================================== .. table:: Operating Systems - ===================== ============================================================ + ===================== ============================================================== OS Description - ===================== ============================================================ + ===================== ============================================================== **/``unknown`` Defaults to the OpenCL runtime. ``vulkan`` Vulkan shader runtime. ``vulkan1.2`` Vulkan 1.2 runtime, corresponding to SPIR-V 1.5. ``vulkan1.3`` Vulkan 1.3 runtime, corresponding to SPIR-V 1.6. - ===================== ============================================================ + ``amdhsa`` AMDHSA runtime, meant to be used on HSA compatible runtimes, + corresponding to SPIR-V 1.6. + ===================== ============================================================== .. table:: SPIR-V Environments @@ -114,15 +118,17 @@ Example: ``-target spirv64v1.0`` can be used to compile for SPIR-V version 1.0 with 64-bit pointer width. +``-target spirv64-amd-amdhsa`` can be used to compile for AMDGCN flavoured SPIR-V with 64-bit pointer width. + .. _spirv-extensions: Extensions ---------- -The SPIR-V backend supports a variety of `extensions `_ -that enable or enhance features beyond the core SPIR-V specification. -These extensions can be enabled using the ``-spirv-extensions`` option -followed by the name of the extension(s) you wish to enable. Below is a +The SPIR-V backend supports a variety of `extensions `_ +that enable or enhance features beyond the core SPIR-V specification. +These extensions can be enabled using the ``-spirv-extensions`` option +followed by the name of the extension(s) you wish to enable. Below is a list of supported SPIR-V extensions, sorted alphabetically by their extension names: .. list-table:: Supported SPIR-V Extensions @@ -189,14 +195,14 @@ To enable all extensions except specified, specify ``all`` followed by a list of SPIR-V representation in LLVM IR ================================ -SPIR-V is intentionally designed for seamless integration with various Intermediate -Representations (IRs), including LLVM IR, facilitating straightforward mappings for -most of its entities. The development of the SPIR-V backend has been guided by a +SPIR-V is intentionally designed for seamless integration with various Intermediate +Representations (IRs), including LLVM IR, facilitating straightforward mappings for +most of its entities. The development of the SPIR-V backend has been guided by a principle of compatibility with the `Khronos Group SPIR-V LLVM Translator `_. -Consequently, the input representation accepted by the SPIR-V backend aligns closely -with that detailed in `the SPIR-V Representation in LLVM document `_. -This document, along with the sections that follow, delineate the main points and focus -on any differences between the LLVM IR that this backend processes and the conventions +Consequently, the input representation accepted by the SPIR-V backend aligns closely +with that detailed in `the SPIR-V Representation in LLVM document `_. +This document, along with the sections that follow, delineate the main points and focus +on any differences between the LLVM IR that this backend processes and the conventions used by other tools. .. _spirv-special-types: @@ -237,10 +243,10 @@ previous type has the representation Target Intrinsics ----------------- -The SPIR-V backend employs several LLVM IR intrinsics that facilitate various low-level -operations essential for generating correct and efficient SPIR-V code. These intrinsics -cover a range of functionalities from type assignment and memory management to control -flow and atomic operations. Below is a detailed table of selected intrinsics used in the +The SPIR-V backend employs several LLVM IR intrinsics that facilitate various low-level +operations essential for generating correct and efficient SPIR-V code. These intrinsics +cover a range of functionalities from type assignment and memory management to control +flow and atomic operations. Below is a detailed table of selected intrinsics used in the SPIR-V backend, along with their descriptions and argument details. .. list-table:: LLVM IR Intrinsics for SPIR-V @@ -369,80 +375,80 @@ SPIR-V backend, along with their descriptions and argument details. Builtin Functions ----------------- -The following section highlights the representation of SPIR-V builtins in LLVM IR, +The following section highlights the representation of SPIR-V builtins in LLVM IR, emphasizing builtins that do not have direct counterparts in LLVM. Instructions as Function Calls ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -SPIR-V builtins without direct LLVM counterparts are represented as LLVM function calls. -These functions, termed SPIR-V builtin functions, follow an IA64 mangling scheme with -SPIR-V-specific extensions. Parsing non-mangled calls to builtins is supported in some cases, +SPIR-V builtins without direct LLVM counterparts are represented as LLVM function calls. +These functions, termed SPIR-V builtin functions, follow an IA64 mangling scheme with +SPIR-V-specific extensions. Parsing non-mangled calls to builtins is supported in some cases, but not tested extensively. The general format is: .. code-block:: c __spirv_{OpCodeName}{_OptionalPostfixes} -Where `{OpCodeName}` is the SPIR-V opcode name sans the "Op" prefix, and -`{OptionalPostfixes}` are decoration-specific postfixes, if any. The mangling and -postfixes allow for the representation of SPIR-V's rich instruction set within LLVM's +Where `{OpCodeName}` is the SPIR-V opcode name sans the "Op" prefix, and +`{OptionalPostfixes}` are decoration-specific postfixes, if any. The mangling and +postfixes allow for the representation of SPIR-V's rich instruction set within LLVM's framework. Extended Instruction Sets ~~~~~~~~~~~~~~~~~~~~~~~~~ -SPIR-V defines several extended instruction sets for additional functionalities, such as -OpenCL-specific operations. In LLVM IR, these are represented by function calls to +SPIR-V defines several extended instruction sets for additional functionalities, such as +OpenCL-specific operations. In LLVM IR, these are represented by function calls to mangled builtins and selected based on the environment. For example: .. code-block:: c acos_f32 -represents the `acos` function from the OpenCL extended instruction set for a float32 +represents the `acos` function from the OpenCL extended instruction set for a float32 input. Builtin Variables ~~~~~~~~~~~~~~~~~ -SPIR-V builtin variables, which provide access to special hardware or execution model -properties, are mapped to either LLVM function calls or LLVM global variables. The +SPIR-V builtin variables, which provide access to special hardware or execution model +properties, are mapped to either LLVM function calls or LLVM global variables. The representation follows the naming convention: .. code-block:: c __spirv_BuiltIn{VariableName} -For instance, the SPIR-V builtin `GlobalInvocationId` is accessible in LLVM IR as +For instance, the SPIR-V builtin `GlobalInvocationId` is accessible in LLVM IR as `__spirv_BuiltInGlobalInvocationId`. Vector Load and Store Builtins ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -SPIR-V's capabilities for loading and storing vectors are represented in LLVM IR using -functions that mimic the SPIR-V instructions. These builtins handle cases that LLVM's -native instructions do not directly support, enabling fine-grained control over memory +SPIR-V's capabilities for loading and storing vectors are represented in LLVM IR using +functions that mimic the SPIR-V instructions. These builtins handle cases that LLVM's +native instructions do not directly support, enabling fine-grained control over memory operations. Atomic Operations ~~~~~~~~~~~~~~~~~ -SPIR-V's atomic operations, especially those operating on floating-point data, are -represented in LLVM IR with corresponding function calls. These builtins ensure -atomicity in operations where LLVM might not have direct support, essential for parallel +SPIR-V's atomic operations, especially those operating on floating-point data, are +represented in LLVM IR with corresponding function calls. These builtins ensure +atomicity in operations where LLVM might not have direct support, essential for parallel execution and synchronization. Image Operations ~~~~~~~~~~~~~~~~ -SPIR-V provides extensive support for image and sampler operations, which LLVM -represents through function calls to builtins. These include image reads, writes, and +SPIR-V provides extensive support for image and sampler operations, which LLVM +represents through function calls to builtins. These include image reads, writes, and queries, allowing detailed manipulation of image data and parameters. Group and Subgroup Operations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -For workgroup and subgroup operations, LLVM uses function calls to represent SPIR-V's -group-based instructions. These builtins facilitate group synchronization, data sharing, +For workgroup and subgroup operations, LLVM uses function calls to represent SPIR-V's +group-based instructions. These builtins facilitate group synchronization, data sharing, and collective operations essential for efficient parallel computation. diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp index a6823a8ba323..52fc6f33b4ef 100644 --- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp @@ -56,6 +56,10 @@ static std::string computeDataLayout(const Triple &TT) { if (Arch == Triple::spirv32) return "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-" "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"; + if (TT.getVendor() == Triple::VendorType::AMD && + TT.getOS() == Triple::OSType::AMDHSA) + return "e-i64:64-v16:16-v24:32-v32:32-v48:64-" + "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1-P4-A0"; return "e-i64:64-v16:16-v24:32-v32:32-v48:64-" "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"; } diff --git a/llvm/lib/TargetParser/TargetParser.cpp b/llvm/lib/TargetParser/TargetParser.cpp index 60a784ef002f..00df92e0aade 100644 --- a/llvm/lib/TargetParser/TargetParser.cpp +++ b/llvm/lib/TargetParser/TargetParser.cpp @@ -315,7 +315,47 @@ StringRef AMDGPU::getCanonicalArchName(const Triple &T, StringRef Arch) { void AMDGPU::fillAMDGPUFeatureMap(StringRef GPU, const Triple &T, StringMap &Features) { // XXX - What does the member GPU mean if device name string passed here? - if (T.isAMDGCN()) { + if (T.isSPIRV() && T.getOS() == Triple::OSType::AMDHSA) { + // AMDGCN SPIRV must support the union of all AMDGCN features. + Features["atomic-ds-pk-add-16-insts"] = true; + Features["atomic-flat-pk-add-16-insts"] = true; + Features["atomic-buffer-global-pk-add-f16-insts"] = true; + Features["atomic-global-pk-add-bf16-inst"] = true; + Features["atomic-fadd-rtn-insts"] = true; + Features["ci-insts"] = true; + Features["dot1-insts"] = true; + Features["dot2-insts"] = true; + Features["dot3-insts"] = true; + Features["dot4-insts"] = true; + Features["dot5-insts"] = true; + Features["dot7-insts"] = true; + Features["dot8-insts"] = true; + Features["dot9-insts"] = true; + Features["dot10-insts"] = true; + Features["dot11-insts"] = true; + Features["dl-insts"] = true; + Features["16-bit-insts"] = true; + Features["dpp"] = true; + Features["gfx8-insts"] = true; + Features["gfx9-insts"] = true; + Features["gfx90a-insts"] = true; + Features["gfx940-insts"] = true; + Features["gfx10-insts"] = true; + Features["gfx10-3-insts"] = true; + Features["gfx11-insts"] = true; + Features["gfx12-insts"] = true; + Features["image-insts"] = true; + Features["fp8-conversion-insts"] = true; + Features["s-memrealtime"] = true; + Features["s-memtime-inst"] = true; + Features["gws"] = true; + Features["fp8-insts"] = true; + Features["fp8-conversion-insts"] = true; + Features["atomic-ds-pk-add-16-insts"] = true; + Features["mai-insts"] = true; + Features["wavefrontsize32"] = true; + Features["wavefrontsize64"] = true; + } else if (T.isAMDGCN()) { switch (parseArchAMDGCN(GPU)) { case GK_GFX1201: case GK_GFX1200: diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_subgroup_rotate/subgroup-rotate.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_subgroup_rotate/subgroup-rotate.ll index 63aade4f7f8d..a38c9072ed1b 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_subgroup_rotate/subgroup-rotate.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_subgroup_rotate/subgroup-rotate.ll @@ -17,7 +17,7 @@ ; CHECK-DAG: %[[ScopeSubgroup:.*]] = OpConstant %[[TyInt32]] 3 ; CHECK-DAG: %[[ConstInt2:.*]] = OpConstant %[[TyInt32]] 2 ; CHECK-DAG: %[[ConstInt4:.*]] = OpConstant %[[TyInt32]] 4 - + target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir" diff --git a/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-divergent-spv_assign_ptr_type.ll b/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-divergent-spv_assign_ptr_type.ll index f728eda07986..8d34a40326d7 100644 --- a/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-divergent-spv_assign_ptr_type.ll +++ b/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-divergent-spv_assign_ptr_type.ll @@ -1,10 +1,11 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -print-after-all -o - 2>&1 | FileCheck %s +; RUN: llc -O0 -mtriple=spirv64-amd-amdhsa %s -print-after-all -o - 2>&1 | FileCheck %s ; CHECK: *** IR Dump After SPIRV emit intrinsics (emit-intrinsics) *** define spir_kernel void @test_pointer_cast(ptr addrspace(1) %src) { -; CHECK-NOT: call void @llvm.spv.assign.ptr.type.p1(ptr addrspace(1) %src, metadata i8 undef, i32 1) -; CHECK: call void @llvm.spv.assign.ptr.type.p1(ptr addrspace(1) %src, metadata i32 0, i32 1) +; CHECK-NOT: call{{.*}} void @llvm.spv.assign.ptr.type.p1(ptr addrspace(1) %src, metadata i8 undef, i32 1) +; CHECK: call{{.*}} void @llvm.spv.assign.ptr.type.p1(ptr addrspace(1) %src, metadata i32 0, i32 1) %b = bitcast ptr addrspace(1) %src to ptr addrspace(1) %g = getelementptr inbounds i32, ptr addrspace(1) %b, i64 52 ret void diff --git a/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-duplicate-spv_assign_type.ll b/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-duplicate-spv_assign_type.ll index 9db4f26a27d4..a9f169a5977a 100644 --- a/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-duplicate-spv_assign_type.ll +++ b/llvm/test/CodeGen/SPIRV/passes/SPIRVEmitIntrinsics-no-duplicate-spv_assign_type.ll @@ -1,11 +1,12 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -print-after-all -o - 2>&1 | FileCheck %s +; RUN: llc -O0 -mtriple=spirv64-amd-amdhsa %s -print-after-all -o - 2>&1 | FileCheck %s ; CHECK: *** IR Dump After SPIRV emit intrinsics (emit-intrinsics) *** define spir_kernel void @test(ptr addrspace(1) %srcimg) { -; CHECK: call void @llvm.spv.assign.type.p1(ptr addrspace(1) %srcimg, metadata target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) poison) +; CHECK: call{{.*}} void @llvm.spv.assign.type.p1(ptr addrspace(1) %srcimg, metadata target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) poison) %call1 = call spir_func <2 x i32> @_Z13get_image_dim14ocl_image2d_ro(ptr addrspace(1) %srcimg) -; CHECK-NOT: call void @llvm.spv.assign.type.p1(ptr addrspace(1) %srcimg, metadata target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) poison) +; CHECK-NOT: call{{.*}} void @llvm.spv.assign.type.p1(ptr addrspace(1) %srcimg, metadata target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) poison) %call2 = call spir_func <2 x i32> @_Z13get_image_dim14ocl_image2d_ro(ptr addrspace(1) %srcimg) ret void ; CHECK: } diff --git a/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_arithmetics.ll b/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_arithmetics.ll index d0c4dff43121..53883fd1691f 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_arithmetics.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_arithmetics.ll @@ -32,9 +32,9 @@ ; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalInvocationId]] Constant ; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalSize]] Constant ; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalOffset]] Constant -; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalOffset]] LinkageAttributes "__spirv_BuiltInGlobalOffset" Import -; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalSize]] LinkageAttributes "__spirv_BuiltInGlobalSize" Import -; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalInvocationId]] LinkageAttributes "__spirv_BuiltInGlobalInvocationId" Import +; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalOffset]] LinkageAttributes "__spirv_BuiltInGlobalOffset" Import +; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalSize]] LinkageAttributes "__spirv_BuiltInGlobalSize" Import +; CHECK-SPIRV-DAG: OpDecorate %[[#GlobalInvocationId]] LinkageAttributes "__spirv_BuiltInGlobalInvocationId" Import %"class._ZTSN2cl4sycl5rangeILi2EEE.cl::sycl::range" = type { %"class._ZTSN2cl4sycl6detail5arrayILi2EEE.cl::sycl::detail::array" } %"class._ZTSN2cl4sycl6detail5arrayILi2EEE.cl::sycl::detail::array" = type { [2 x i64] } diff --git a/llvm/test/CodeGen/SPIRV/transcoding/sub_group_non_uniform_vote.ll b/llvm/test/CodeGen/SPIRV/transcoding/sub_group_non_uniform_vote.ll index 9654de6b8413..1073473a224d 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/sub_group_non_uniform_vote.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/sub_group_non_uniform_vote.ll @@ -5,7 +5,7 @@ ;; } ;; ;; kernel void testSubGroupNonUniformAll(global int* dst) { -;; dst[0] = sub_group_non_uniform_all(0); +;; dst[0] = sub_group_non_uniform_all(0); ;; } ;; ;; kernel void testSubGroupNonUniformAny(global int* dst) { -- GitLab From 3fefb3c598db995433093ed158c08368809b3f78 Mon Sep 17 00:00:00 2001 From: Nathan Sidwell Date: Fri, 7 Jun 2024 06:59:52 -0400 Subject: [PATCH 234/462] [BOLT][NFC] Infailable fns return void (#92018) Both `reverseBranchCondition` and `replaceBranchTarget` return a success boolean. But all-but-one caller ignores the return value, and the exception emits a fatal error on failure. Thus, just return nothing. --- bolt/include/bolt/Core/MCPlusBuilder.h | 10 ++-------- bolt/lib/Passes/VeneerElimination.cpp | 7 ++----- bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp | 7 +++---- bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp | 7 +++---- bolt/lib/Target/X86/X86MCPlusBuilder.cpp | 6 ++---- 5 files changed, 12 insertions(+), 25 deletions(-) diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h index f7cf538bd0e8..a5fb3901428d 100644 --- a/bolt/include/bolt/Core/MCPlusBuilder.h +++ b/bolt/include/bolt/Core/MCPlusBuilder.h @@ -1706,12 +1706,9 @@ public: } /// Reverses the branch condition in Inst and update its taken target to TBB. - /// - /// Returns true on success. - virtual bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, + virtual void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx) const { llvm_unreachable("not implemented"); - return false; } virtual bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, @@ -1751,12 +1748,9 @@ public: } /// Sets the taken target of the branch instruction to Target. - /// - /// Returns true on success. - virtual bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, + virtual void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx) const { llvm_unreachable("not implemented"); - return false; } /// Extract a symbol and an addend out of the fixup value expression. diff --git a/bolt/lib/Passes/VeneerElimination.cpp b/bolt/lib/Passes/VeneerElimination.cpp index 0bec11128c7c..87fe625e8c3b 100644 --- a/bolt/lib/Passes/VeneerElimination.cpp +++ b/bolt/lib/Passes/VeneerElimination.cpp @@ -77,11 +77,8 @@ Error VeneerElimination::runOnFunctions(BinaryContext &BC) { continue; VeneerCallers++; - if (!BC.MIB->replaceBranchTarget( - Instr, VeneerDestinations[TargetSymbol], BC.Ctx.get())) { - return createFatalBOLTError( - "BOLT-ERROR: updating veneer call destination failed\n"); - } + BC.MIB->replaceBranchTarget(Instr, VeneerDestinations[TargetSymbol], + BC.Ctx.get()); } } } diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp index 0ae9d3668b93..a74eda8e4a56 100644 --- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp +++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp @@ -616,7 +616,7 @@ public: return getTargetAddend(Op.getExpr()); } - bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, + void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx) const override { assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) && "Invalid instruction"); @@ -638,7 +638,6 @@ public: *OI = MCOperand::createExpr( MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); - return true; } /// Matches indirect branch patterns in AArch64 related to a jump table (JT), @@ -969,7 +968,7 @@ public: } } - bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, + void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx) const override { if (isTB(Inst) || isCB(Inst)) { Inst.setOpcode(getInvertedBranchOpcode(Inst.getOpcode())); @@ -984,7 +983,7 @@ public: LLVM_DEBUG(Inst.dump()); llvm_unreachable("Unrecognized branch instruction"); } - return replaceBranchTarget(Inst, TBB, Ctx); + replaceBranchTarget(Inst, TBB, Ctx); } int getPCRelEncodingSize(const MCInst &Inst) const override { diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp index 74f2f0aae91e..eb3f38a0b8f4 100644 --- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp +++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp @@ -151,14 +151,14 @@ public: } } - bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, + void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx) const override { auto Opcode = getInvertedBranchOpcode(Inst.getOpcode()); Inst.setOpcode(Opcode); - return replaceBranchTarget(Inst, TBB, Ctx); + replaceBranchTarget(Inst, TBB, Ctx); } - bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, + void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx) const override { assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) && "Invalid instruction"); @@ -170,7 +170,6 @@ public: Inst.getOperand(SymOpIndex) = MCOperand::createExpr( MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); - return true; } IndirectBranchType analyzeIndirectBranch( diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp index a33a9dc8c013..e350e701c7b7 100644 --- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp +++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp @@ -2794,14 +2794,13 @@ public: Inst.addOperand(MCOperand::createImm(CC)); } - bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, + void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx) const override { unsigned InvCC = getInvertedCondCode(getCondCode(Inst)); assert(InvCC != X86::COND_INVALID && "invalid branch instruction"); Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC); Inst.getOperand(0) = MCOperand::createExpr( MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); - return true; } bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx, @@ -2844,13 +2843,12 @@ public: } } - bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, + void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx) const override { assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) && "Invalid instruction"); Inst.getOperand(0) = MCOperand::createExpr( MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); - return true; } MCPhysReg getX86R11() const override { return X86::R11; } -- GitLab From 74d62c2f73799ea6ffbf165ff752ca0b0f826eca Mon Sep 17 00:00:00 2001 From: aengelke Date: Fri, 7 Jun 2024 13:17:27 +0200 Subject: [PATCH 235/462] [CodeGen][SDAG] Remove CombinedNodes SmallPtrSet (#94609) This "small" set grows quite large and it's more performant to store whether a node has been combined before in the node itself. As this information is only relevant for nodes that are currently not in the worklist, add a second state to the CombinerWorklistIndex (-2) to indicate that a node is currently not in a worklist, but was combined before. This brings a substantial performance improvement. --- llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 4 ++- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 33 ++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 9a54dbd434d9..2edb039cbbfd 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -649,7 +649,9 @@ private: /// Return a pointer to the specified value type. static const EVT *getValueTypeList(EVT VT); - /// Index in worklist of DAGCombiner, or -1. + /// Index in worklist of DAGCombiner, or negative if the node is not in the + /// worklist. -1 = not in worklist; -2 = not in worklist, but has already been + /// combined at least once. int CombinerWorklistIndex = -1; uint32_t CFIType = 0; diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index a913472525cf..2d5968bf5c2e 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -180,12 +180,6 @@ namespace { /// in the worklist, this is different from the tail of the worklist. SmallSetVector PruningList; - /// Set of nodes which have been combined (at least once). - /// - /// This is used to allow us to reliably add any operands of a DAG node - /// which have not yet been combined to the worklist. - SmallPtrSet CombinedNodes; - /// Map from candidate StoreNode to the pair of RootNode and count. /// The count is used to track how many times we have seen the StoreNode /// with the same RootNode bail out in dependence check. If we have seen @@ -235,7 +229,8 @@ namespace { if (N) { assert(N->getCombinerWorklistIndex() >= 0 && "Found a worklist entry without a corresponding map entry!"); - N->setCombinerWorklistIndex(-1); + // Set to -2 to indicate that we combined the node. + N->setCombinerWorklistIndex(-2); } return N; } @@ -267,7 +262,8 @@ namespace { /// Add to the worklist making sure its instance is at the back (next to be /// processed.) - void AddToWorklist(SDNode *N, bool IsCandidateForPruning = true) { + void AddToWorklist(SDNode *N, bool IsCandidateForPruning = true, + bool SkipIfCombinedBefore = false) { assert(N->getOpcode() != ISD::DELETED_NODE && "Deleted Node added to Worklist"); @@ -276,10 +272,13 @@ namespace { if (N->getOpcode() == ISD::HANDLENODE) return; + if (SkipIfCombinedBefore && N->getCombinerWorklistIndex() == -2) + return; + if (IsCandidateForPruning) ConsiderForPruning(N); - if (N->getCombinerWorklistIndex() == -1) { + if (N->getCombinerWorklistIndex() < 0) { N->setCombinerWorklistIndex(Worklist.size()); Worklist.push_back(N); } @@ -287,12 +286,14 @@ namespace { /// Remove all instances of N from the worklist. void removeFromWorklist(SDNode *N) { - CombinedNodes.erase(N); PruningList.remove(N); StoreRootCountMap.erase(N); int WorklistIndex = N->getCombinerWorklistIndex(); - if (WorklistIndex == -1) + // If not in the worklist, the index might be -1 or -2 (was combined + // before). As the node gets deleted anyway, there's no need to update + // the index. + if (WorklistIndex < 0) return; // Not in the worklist. // Null out the entry rather than erasing it to avoid a linear operation. @@ -1768,13 +1769,13 @@ void DAGCombiner::Run(CombineLevel AtLevel) { LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG)); // Add any operands of the new node which have not yet been combined to the - // worklist as well. Because the worklist uniques things already, this - // won't repeatedly process the same operand. + // worklist as well. getNextWorklistEntry flags nodes that have been + // combined before. Because the worklist uniques things already, this won't + // repeatedly process the same operand. for (const SDValue &ChildN : N->op_values()) - if (!CombinedNodes.count(ChildN.getNode())) - AddToWorklist(ChildN.getNode()); + AddToWorklist(ChildN.getNode(), /*IsCandidateForPruning=*/true, + /*SkipIfCombinedBefore=*/true); - CombinedNodes.insert(N); SDValue RV = combine(N); if (!RV.getNode()) -- GitLab From 9ece3eb1459309f9fbd18ce8ec8f771c238e8815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Fri, 7 Jun 2024 12:59:28 +0200 Subject: [PATCH 236/462] [clang][Interp] Check ConstantExpr results for initialization They need to be fully initialized, similar to global variables. --- clang/lib/AST/Interp/EvalEmitter.cpp | 5 +++++ clang/lib/AST/Interp/EvaluationResult.cpp | 14 ++++++-------- clang/test/AST/Interp/cxx20.cpp | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/clang/lib/AST/Interp/EvalEmitter.cpp b/clang/lib/AST/Interp/EvalEmitter.cpp index 6d8aa3f20f01..f6191d8ed634 100644 --- a/clang/lib/AST/Interp/EvalEmitter.cpp +++ b/clang/lib/AST/Interp/EvalEmitter.cpp @@ -41,6 +41,7 @@ EvaluationResult EvalEmitter::interpretExpr(const Expr *E, bool ConvertResultToRValue) { S.setEvalLocation(E->getExprLoc()); this->ConvertResultToRValue = ConvertResultToRValue; + this->CheckFullyInitialized = isa(E); EvalResult.setSource(E); if (!this->visitExpr(E)) { @@ -175,6 +176,10 @@ bool EvalEmitter::emitRetVoid(const SourceInfo &Info) { bool EvalEmitter::emitRetValue(const SourceInfo &Info) { const auto &Ptr = S.Stk.pop(); + + if (CheckFullyInitialized && !EvalResult.checkFullyInitialized(S, Ptr)) + return false; + if (std::optional APV = Ptr.toRValue(S.getCtx())) { EvalResult.setValue(*APV); return true; diff --git a/clang/lib/AST/Interp/EvaluationResult.cpp b/clang/lib/AST/Interp/EvaluationResult.cpp index c04b0fc0a112..29977232975f 100644 --- a/clang/lib/AST/Interp/EvaluationResult.cpp +++ b/clang/lib/AST/Interp/EvaluationResult.cpp @@ -141,16 +141,14 @@ bool EvaluationResult::checkFullyInitialized(InterpState &S, const Pointer &Ptr) const { assert(Source); assert(empty()); - - // Our Source must be a VarDecl. - const Decl *SourceDecl = Source.dyn_cast(); - assert(SourceDecl); - const auto *VD = cast(SourceDecl); - assert(VD->getType()->isRecordType() || VD->getType()->isArrayType()); - SourceLocation InitLoc = VD->getAnyInitializer()->getExprLoc(); - assert(!Ptr.isZero()); + SourceLocation InitLoc; + if (const auto *D = Source.dyn_cast()) + InitLoc = cast(D)->getAnyInitializer()->getExprLoc(); + else if (const auto *E = Source.dyn_cast()) + InitLoc = E->getExprLoc(); + if (const Record *R = Ptr.getRecord()) return CheckFieldsInitialized(S, InitLoc, Ptr, R); const auto *CAT = diff --git a/clang/test/AST/Interp/cxx20.cpp b/clang/test/AST/Interp/cxx20.cpp index c750be866ca7..434823644a7a 100644 --- a/clang/test/AST/Interp/cxx20.cpp +++ b/clang/test/AST/Interp/cxx20.cpp @@ -797,3 +797,20 @@ namespace self_referencing { S s(1); } } + +namespace GH64949 { + struct f { + int g; // both-note {{subobject declared here}} + constexpr ~f() {} + }; + + class h { + public: + consteval h(char *) {} + f i; + }; + + void test() { h{nullptr}; } // both-error {{call to consteval function 'GH64949::h::h' is not a constant expression}} \ + // both-note {{subobject 'g' is not initialized}} \ + // both-warning {{expression result unused}} +} -- GitLab From 9eb8a130c5d708dbabe824113add072436ae9997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Fri, 7 Jun 2024 13:15:05 +0200 Subject: [PATCH 237/462] [clang][Interp][NFC] Fix a const-correctness warning --- clang/lib/AST/Interp/MemberPointer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/AST/Interp/MemberPointer.h b/clang/lib/AST/Interp/MemberPointer.h index 5c61f6a43957..f56dc530431e 100644 --- a/clang/lib/AST/Interp/MemberPointer.h +++ b/clang/lib/AST/Interp/MemberPointer.h @@ -86,7 +86,7 @@ public: bool hasBase() const { return !Base.isZero(); } void print(llvm::raw_ostream &OS) const { - OS << "MemberPtr(" << Base << " " << (void *)Dcl << " + " << PtrOffset + OS << "MemberPtr(" << Base << " " << (const void *)Dcl << " + " << PtrOffset << ")"; } -- GitLab From b8cc85b318c0dd89e4dd69e3691ffcad5e401885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Fri, 7 Jun 2024 13:27:26 +0200 Subject: [PATCH 238/462] [clang][Interp] Limit lambda capture lazy visting to actual captures Check this by looking at the VarDecl. --- clang/lib/AST/Interp/ByteCodeExprGen.cpp | 13 +++++++------ clang/test/AST/Interp/lambda.cpp | 13 +++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index 8dc71b2527f3..ff2b51e3fb6f 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -3895,12 +3895,13 @@ bool ByteCodeExprGen::visitDeclRef(const ValueDecl *D, const Expr *E) { return this->emitGetThisFieldPtr(Offset, E); return this->emitGetPtrThisField(Offset, E); } else if (const auto *DRE = dyn_cast(E); - DRE && DRE->refersToEnclosingVariableOrCapture() && - isa(D)) { - if (!this->visitVarDecl(cast(D))) - return false; - // Retry. - return this->visitDeclRef(D, E); + DRE && DRE->refersToEnclosingVariableOrCapture()) { + if (const auto *VD = dyn_cast(D); VD && VD->isInitCapture()) { + if (!this->visitVarDecl(cast(D))) + return false; + // Retry. + return this->visitDeclRef(D, E); + } } // Try to lazily visit (or emit dummy pointers for) declarations diff --git a/clang/test/AST/Interp/lambda.cpp b/clang/test/AST/Interp/lambda.cpp index 71e7077550b2..0eb12643b1b7 100644 --- a/clang/test/AST/Interp/lambda.cpp +++ b/clang/test/AST/Interp/lambda.cpp @@ -267,3 +267,16 @@ namespace CaptureDefaults { constexpr auto t4 = ([x=42]() consteval { return x; }()); static_assert(t4 == 42, ""); + +namespace InvalidCapture { + + int &f(int *p); + char &f(...); + void g() { + int n = -1; // both-note {{declared here}} + [=] { + int arr[n]; // both-warning {{variable length arrays in C++ are a Clang extension}} \ + both-note {{read of non-const variable 'n' is not allowed in a constant expression}} + } (); + } +} -- GitLab From 5a0181f568e56e37df80d0f74eca4775776fa8cd Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Thu, 6 Jun 2024 11:51:05 +0800 Subject: [PATCH 239/462] [serialization] no transitive decl change (#92083) Following of https://github.com/llvm/llvm-project/pull/86912 The motivation of the patch series is that, for a module interface unit `X`, when the dependent modules of `X` changes, if the changes is not relevant with `X`, we hope the BMI of `X` won't change. For the specific patch, we hope if the changes was about irrelevant declaration changes, we hope the BMI of `X` won't change. **However**, I found the patch itself is not very useful in practice, since the adding or removing declarations, will change the state of identifiers and types in most cases. That said, for the most simple example, ``` // partA.cppm export module m:partA; // partA.v1.cppm export module m:partA; export void a() {} // partB.cppm export module m:partB; export void b() {} // m.cppm export module m; export import :partA; export import :partB; // onlyUseB; export module onlyUseB; import m; export inline void onluUseB() { b(); } ``` the BMI of `onlyUseB` will change after we change the implementation of `partA.cppm` to `partA.v1.cppm`. Since `partA.v1.cppm` introduces new identifiers and types (the function prototype). So in this patch, we have to write the tests as: ``` // partA.cppm export module m:partA; export int getA() { ... } export int getA2(int) { ... } // partA.v1.cppm export module m:partA; export int getA() { ... } export int getA(int) { ... } export int getA2(int) { ... } // partB.cppm export module m:partB; export void b() {} // m.cppm export module m; export import :partA; export import :partB; // onlyUseB; export module onlyUseB; import m; export inline void onluUseB() { b(); } ``` so that the new introduced declaration `int getA(int)` doesn't introduce new identifiers and types, then the BMI of `onlyUseB` can keep unchanged. While it looks not so great, the patch should be the base of the patch to erase the transitive change for identifiers and types since I don't know how can we introduce new types and identifiers without introducing new declarations. Given how tightly the relationship between declarations, types and identifiers, I think we can only reach the ideal state after we made the series for all of the three entties. The design of the patch is similar to https://github.com/llvm/llvm-project/pull/86912, which extends the 32-bit DeclID to 64-bit and use the higher bits to store the module file index and the lower bits to store the Local Decl ID. A slight difference is that we only use 48 bits to store the new DeclID since we try to use the higher 16 bits to store the module ID in the prefix of Decl class. Previously, we use 32 bits to store the module ID and 32 bits to store the DeclID. I don't want to allocate additional space so I tried to make the additional space the same as 64 bits. An potential interesting thing here is about the relationship between the module ID and the module file index. I feel we can get the module file index by the module ID. But I didn't prove it or implement it. Since I want to make the patch itself as small as possible. We can make it in the future if we want. Another change in the patch is the new concept Decl Index, which means the index of the very big array `DeclsLoaded` in ASTReader. Previously, the index of a loaded declaration is simply the Decl ID minus PREDEFINED_DECL_NUMs. So there are some places they got used ambiguously. But this patch tried to split these two concepts. As https://github.com/llvm/llvm-project/pull/86912 did, the change will increase the on-disk PCM file sizes. As the declaration ID may be the most IDs in the PCM file, this can have the biggest impact on the size. In my experiments, this change will bring 6.6% increase of the on-disk PCM size. No compile-time performance regression observed. Given the benefits in the motivation example, I think the cost is worthwhile. --- clang/include/clang/AST/ASTUnresolvedSet.h | 9 +- clang/include/clang/AST/DeclAccessPair.h | 37 +++- clang/include/clang/AST/DeclBase.h | 17 +- clang/include/clang/AST/DeclID.h | 18 +- clang/include/clang/AST/UnresolvedSet.h | 1 + .../include/clang/Serialization/ASTBitCodes.h | 29 +++- clang/include/clang/Serialization/ASTReader.h | 36 ++-- .../include/clang/Serialization/ModuleFile.h | 18 +- .../clang/Serialization/ModuleManager.h | 2 +- clang/lib/AST/DeclBase.cpp | 40 ++++- clang/lib/AST/DeclCXX.cpp | 4 +- clang/lib/Serialization/ASTReader.cpp | 163 ++++++++++-------- clang/lib/Serialization/ASTReaderDecl.cpp | 16 +- clang/lib/Serialization/ASTWriter.cpp | 7 +- clang/lib/Serialization/ModuleFile.cpp | 3 +- .../Modules/no-transitive-decls-change.cppm | 112 ++++++++++++ 16 files changed, 345 insertions(+), 167 deletions(-) create mode 100644 clang/test/Modules/no-transitive-decls-change.cppm diff --git a/clang/include/clang/AST/ASTUnresolvedSet.h b/clang/include/clang/AST/ASTUnresolvedSet.h index 398ffb188c95..dcce3bc63df2 100644 --- a/clang/include/clang/AST/ASTUnresolvedSet.h +++ b/clang/include/clang/AST/ASTUnresolvedSet.h @@ -16,6 +16,7 @@ #include "clang/AST/ASTVector.h" #include "clang/AST/DeclAccessPair.h" +#include "clang/AST/DeclID.h" #include "clang/AST/UnresolvedSet.h" #include "clang/Basic/Specifiers.h" #include @@ -56,6 +57,10 @@ public: Decls.push_back(DeclAccessPair::make(D, AS), C); } + void addLazyDecl(ASTContext &C, GlobalDeclID ID, AccessSpecifier AS) { + Decls.push_back(DeclAccessPair::makeLazy(ID.get(), AS), C); + } + /// Replaces the given declaration with the new one, once. /// /// \return true if the set changed @@ -109,10 +114,10 @@ public: void reserve(ASTContext &C, unsigned N) { Impl.reserve(C, N); } - void addLazyDecl(ASTContext &C, uintptr_t ID, AccessSpecifier AS) { + void addLazyDecl(ASTContext &C, GlobalDeclID ID, AccessSpecifier AS) { assert(Impl.empty() || Impl.Decls.isLazy()); Impl.Decls.setLazy(true); - Impl.addDecl(C, reinterpret_cast(ID << 2), AS); + Impl.addLazyDecl(C, ID, AS); } }; diff --git a/clang/include/clang/AST/DeclAccessPair.h b/clang/include/clang/AST/DeclAccessPair.h index 805342c2910a..4becfde96305 100644 --- a/clang/include/clang/AST/DeclAccessPair.h +++ b/clang/include/clang/AST/DeclAccessPair.h @@ -19,6 +19,7 @@ #include "clang/Basic/Specifiers.h" #include "llvm/Support/DataTypes.h" +#include "llvm/Support/Endian.h" namespace clang { @@ -27,9 +28,17 @@ class NamedDecl; /// A POD class for pairing a NamedDecl* with an access specifier. /// Can be put into unions. class DeclAccessPair { - uintptr_t Ptr; // we'd use llvm::PointerUnion, but it isn't trivial + /// Use the lower 2 bit to store AccessSpecifier. Use the higher + /// 61 bit to store the pointer to a NamedDecl or the DeclID to + /// a NamedDecl. If the 3rd bit is set, storing the DeclID, otherwise + /// storing the pointer. + llvm::support::detail::packed_endian_specific_integral< + uint64_t, llvm::endianness::native, alignof(void *)> + Ptr; - enum { Mask = 0x3 }; + enum { ASMask = 0x3, Mask = 0x7 }; + + bool isDeclID() const { return (Ptr >> 2) & 0x1; } public: static DeclAccessPair make(NamedDecl *D, AccessSpecifier AS) { @@ -38,12 +47,22 @@ public: return p; } + static DeclAccessPair makeLazy(uint64_t ID, AccessSpecifier AS) { + DeclAccessPair p; + p.Ptr = (ID << 3) | (0x1 << 2) | uint64_t(AS); + return p; + } + + uint64_t getDeclID() const { + assert(isDeclID()); + return (~Mask & Ptr) >> 3; + } + NamedDecl *getDecl() const { + assert(!isDeclID()); return reinterpret_cast(~Mask & Ptr); } - AccessSpecifier getAccess() const { - return AccessSpecifier(Mask & Ptr); - } + AccessSpecifier getAccess() const { return AccessSpecifier(ASMask & Ptr); } void setDecl(NamedDecl *D) { set(D, getAccess()); @@ -52,12 +71,18 @@ public: set(getDecl(), AS); } void set(NamedDecl *D, AccessSpecifier AS) { - Ptr = uintptr_t(AS) | reinterpret_cast(D); + Ptr = uint64_t(AS) | reinterpret_cast(D); } operator NamedDecl*() const { return getDecl(); } NamedDecl *operator->() const { return getDecl(); } }; + +// Make sure DeclAccessPair is pointer-aligned types. +static_assert(alignof(DeclAccessPair) == alignof(void *)); +// Make sure DeclAccessPair is still POD. +static_assert(std::is_standard_layout_v && + std::is_trivial_v); } #endif diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h index 600ce73c7f01..5f19af1891b7 100644 --- a/clang/include/clang/AST/DeclBase.h +++ b/clang/include/clang/AST/DeclBase.h @@ -708,10 +708,7 @@ public: /// Set the owning module ID. This may only be called for /// deserialized Decls. - void setOwningModuleID(unsigned ID) { - assert(isFromASTFile() && "Only works on a deserialized declaration"); - *((unsigned*)this - 2) = ID; - } + void setOwningModuleID(unsigned ID); public: /// Determine the availability of the given declaration. @@ -784,19 +781,11 @@ public: /// Retrieve the global declaration ID associated with this /// declaration, which specifies where this Decl was loaded from. - GlobalDeclID getGlobalID() const { - if (isFromASTFile()) - return (*((const GlobalDeclID *)this - 1)); - return GlobalDeclID(); - } + GlobalDeclID getGlobalID() const; /// Retrieve the global ID of the module that owns this particular /// declaration. - unsigned getOwningModuleID() const { - if (isFromASTFile()) - return *((const unsigned*)this - 2); - return 0; - } + unsigned getOwningModuleID() const; private: Module *getOwningModuleSlow() const; diff --git a/clang/include/clang/AST/DeclID.h b/clang/include/clang/AST/DeclID.h index 614ba06b6386..32d2ed41a374 100644 --- a/clang/include/clang/AST/DeclID.h +++ b/clang/include/clang/AST/DeclID.h @@ -19,6 +19,8 @@ #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/iterator.h" +#include + namespace clang { /// Predefined declaration IDs. @@ -107,12 +109,16 @@ public: /// /// DeclID should only be used directly in serialization. All other users /// should use LocalDeclID or GlobalDeclID. - using DeclID = uint32_t; + using DeclID = uint64_t; protected: DeclIDBase() : ID(PREDEF_DECL_NULL_ID) {} explicit DeclIDBase(DeclID ID) : ID(ID) {} + explicit DeclIDBase(unsigned LocalID, unsigned ModuleFileIndex) { + ID = (DeclID)LocalID | ((DeclID)ModuleFileIndex << 32); + } + public: DeclID get() const { return ID; } @@ -124,6 +130,10 @@ public: bool isInvalid() const { return ID == PREDEF_DECL_NULL_ID; } + unsigned getModuleFileIndex() const { return ID >> 32; } + + unsigned getLocalDeclIndex() const; + friend bool operator==(const DeclIDBase &LHS, const DeclIDBase &RHS) { return LHS.ID == RHS.ID; } @@ -156,6 +166,9 @@ public: LocalDeclID(PredefinedDeclIDs ID) : Base(ID) {} explicit LocalDeclID(DeclID ID) : Base(ID) {} + explicit LocalDeclID(unsigned LocalID, unsigned ModuleFileIndex) + : Base(LocalID, ModuleFileIndex) {} + LocalDeclID &operator++() { ++ID; return *this; @@ -175,6 +188,9 @@ public: GlobalDeclID() : Base() {} explicit GlobalDeclID(DeclID ID) : Base(ID) {} + explicit GlobalDeclID(unsigned LocalID, unsigned ModuleFileIndex) + : Base(LocalID, ModuleFileIndex) {} + // For DeclIDIterator to be able to convert a GlobalDeclID // to a LocalDeclID. explicit operator LocalDeclID() const { return LocalDeclID(this->ID); } diff --git a/clang/include/clang/AST/UnresolvedSet.h b/clang/include/clang/AST/UnresolvedSet.h index ee31be969b6e..1369725ab4e9 100644 --- a/clang/include/clang/AST/UnresolvedSet.h +++ b/clang/include/clang/AST/UnresolvedSet.h @@ -47,6 +47,7 @@ public: // temporaries with defaulted ctors are not zero initialized. UnresolvedSetIterator() : iterator_adaptor_base(nullptr) {} + uint64_t getDeclID() const { return I->getDeclID(); } NamedDecl *getDecl() const { return I->getDecl(); } void setDecl(NamedDecl *ND) const { return I->setDecl(ND); } AccessSpecifier getAccess() const { return I->getAccess(); } diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h index f59ff6af4c76..52a6c5e10f80 100644 --- a/clang/include/clang/Serialization/ASTBitCodes.h +++ b/clang/include/clang/Serialization/ASTBitCodes.h @@ -255,6 +255,12 @@ public: } }; +// The unaligned decl ID used in the Blobs of bistreams. +using unaligned_decl_id_t = + llvm::support::detail::packed_endian_specific_integral< + serialization::DeclID, llvm::endianness::native, + llvm::support::unaligned>; + /// The number of predefined preprocessed entity IDs. const unsigned int NUM_PREDEF_PP_ENTITY_IDS = 1; @@ -1980,33 +1986,44 @@ enum CleanupObjectKind { COK_Block, COK_CompoundLiteral }; /// Describes the categories of an Objective-C class. struct ObjCCategoriesInfo { - // The ID of the definition - LocalDeclID DefinitionID; + // The ID of the definition. Use unaligned_decl_id_t to keep + // ObjCCategoriesInfo 32-bit aligned. + unaligned_decl_id_t DefinitionID; // Offset into the array of category lists. unsigned Offset; + ObjCCategoriesInfo() = default; + ObjCCategoriesInfo(LocalDeclID ID, unsigned Offset) + : DefinitionID(ID.get()), Offset(Offset) {} + + LocalDeclID getDefinitionID() const { return LocalDeclID(DefinitionID); } + friend bool operator<(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.DefinitionID < Y.DefinitionID; + return X.getDefinitionID() < Y.getDefinitionID(); } friend bool operator>(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.DefinitionID > Y.DefinitionID; + return X.getDefinitionID() > Y.getDefinitionID(); } friend bool operator<=(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.DefinitionID <= Y.DefinitionID; + return X.getDefinitionID() <= Y.getDefinitionID(); } friend bool operator>=(const ObjCCategoriesInfo &X, const ObjCCategoriesInfo &Y) { - return X.DefinitionID >= Y.DefinitionID; + return X.getDefinitionID() >= Y.getDefinitionID(); } }; +static_assert(alignof(ObjCCategoriesInfo) <= 4); +static_assert(std::is_standard_layout_v && + std::is_trivial_v); + /// A key used when looking up entities by \ref DeclarationName. /// /// Different \ref DeclarationNames are mapped to different keys, but the diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index bea58d60d36c..0a9006223dcb 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -504,12 +504,6 @@ private: /// = I + 1 has already been loaded. llvm::PagedVector DeclsLoaded; - using GlobalDeclMapType = ContinuousRangeMap; - - /// Mapping from global declaration IDs to the module in which the - /// declaration resides. - GlobalDeclMapType GlobalDeclMap; - using FileOffset = std::pair; using FileOffsetsTy = SmallVector; using DeclUpdateOffsetsMap = llvm::DenseMap; @@ -592,10 +586,11 @@ private: struct FileDeclsInfo { ModuleFile *Mod = nullptr; - ArrayRef Decls; + ArrayRef Decls; FileDeclsInfo() = default; - FileDeclsInfo(ModuleFile *Mod, ArrayRef Decls) + FileDeclsInfo(ModuleFile *Mod, + ArrayRef Decls) : Mod(Mod), Decls(Decls) {} }; @@ -604,11 +599,7 @@ private: /// An array of lexical contents of a declaration context, as a sequence of /// Decl::Kind, DeclID pairs. - using unaligned_decl_id_t = - llvm::support::detail::packed_endian_specific_integral< - serialization::DeclID, llvm::endianness::native, - llvm::support::unaligned>; - using LexicalContents = ArrayRef; + using LexicalContents = ArrayRef; /// Map from a DeclContext to its lexical contents. llvm::DenseMap> @@ -1489,10 +1480,11 @@ private: unsigned ClientLoadCapabilities); public: - class ModuleDeclIterator : public llvm::iterator_adaptor_base< - ModuleDeclIterator, const LocalDeclID *, - std::random_access_iterator_tag, const Decl *, - ptrdiff_t, const Decl *, const Decl *> { + class ModuleDeclIterator + : public llvm::iterator_adaptor_base< + ModuleDeclIterator, const serialization::unaligned_decl_id_t *, + std::random_access_iterator_tag, const Decl *, ptrdiff_t, + const Decl *, const Decl *> { ASTReader *Reader = nullptr; ModuleFile *Mod = nullptr; @@ -1500,11 +1492,11 @@ public: ModuleDeclIterator() : iterator_adaptor_base(nullptr) {} ModuleDeclIterator(ASTReader *Reader, ModuleFile *Mod, - const LocalDeclID *Pos) + const serialization::unaligned_decl_id_t *Pos) : iterator_adaptor_base(Pos), Reader(Reader), Mod(Mod) {} value_type operator*() const { - return Reader->GetDecl(Reader->getGlobalDeclID(*Mod, *I)); + return Reader->GetDecl(Reader->getGlobalDeclID(*Mod, (LocalDeclID)*I)); } value_type operator->() const { return **this; } @@ -1544,6 +1536,9 @@ private: StringRef Arg2 = StringRef(), StringRef Arg3 = StringRef()) const; void Error(llvm::Error &&Err) const; + /// Translate a \param GlobalDeclID to the index of DeclsLoaded array. + unsigned translateGlobalDeclIDToIndex(GlobalDeclID ID) const; + public: /// Load the AST file and validate its contents against the given /// Preprocessor. @@ -1915,7 +1910,8 @@ public: /// Retrieve the module file that owns the given declaration, or NULL /// if the declaration is not from a module file. - ModuleFile *getOwningModuleFile(const Decl *D); + ModuleFile *getOwningModuleFile(const Decl *D) const; + ModuleFile *getOwningModuleFile(GlobalDeclID ID) const; /// Returns the source location for the decl \p ID. SourceLocation getSourceLocationForDeclID(GlobalDeclID ID); diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 992d26a8b88c..56193d44dd6f 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -454,23 +454,11 @@ public: /// by the declaration ID (-1). const DeclOffset *DeclOffsets = nullptr; - /// Base declaration ID for declarations local to this module. - serialization::DeclID BaseDeclID = 0; - - /// Remapping table for declaration IDs in this module. - ContinuousRangeMap DeclRemap; - - /// Mapping from the module files that this module file depends on - /// to the base declaration ID for that module as it is understood within this - /// module. - /// - /// This is effectively a reverse global-to-local mapping for declaration - /// IDs, so that we can interpret a true global ID (for this translation unit) - /// as a local ID (for this module file). - llvm::DenseMap GlobalToLocalDeclIDs; + /// Base declaration index in ASTReader for declarations local to this module. + unsigned BaseDeclIndex = 0; /// Array of file-level DeclIDs sorted by file. - const LocalDeclID *FileSortedDecls = nullptr; + const serialization::unaligned_decl_id_t *FileSortedDecls = nullptr; unsigned NumFileSortedDecls = 0; /// Array of category list location information within this diff --git a/clang/include/clang/Serialization/ModuleManager.h b/clang/include/clang/Serialization/ModuleManager.h index d770bc52eaf4..f898dab39f06 100644 --- a/clang/include/clang/Serialization/ModuleManager.h +++ b/clang/include/clang/Serialization/ModuleManager.h @@ -45,7 +45,7 @@ namespace serialization { /// Manages the set of modules loaded by an AST reader. class ModuleManager { /// The chain of AST files, in the order in which we started to load - /// them (this order isn't really useful for anything). + /// them. SmallVector, 2> Chain; /// The chain of non-module PCH files. The first entry is the one named diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp index 1e9c879e371b..7f1ed9c691e9 100644 --- a/clang/lib/AST/DeclBase.cpp +++ b/clang/lib/AST/DeclBase.cpp @@ -74,18 +74,17 @@ void *Decl::operator new(std::size_t Size, const ASTContext &Context, GlobalDeclID ID, std::size_t Extra) { // Allocate an extra 8 bytes worth of storage, which ensures that the // resulting pointer will still be 8-byte aligned. - static_assert(sizeof(unsigned) * 2 >= alignof(Decl), - "Decl won't be misaligned"); + static_assert(sizeof(uint64_t) >= alignof(Decl), "Decl won't be misaligned"); void *Start = Context.Allocate(Size + Extra + 8); void *Result = (char*)Start + 8; - unsigned *PrefixPtr = (unsigned *)Result - 2; + uint64_t *PrefixPtr = (uint64_t *)Result - 1; - // Zero out the first 4 bytes; this is used to store the owning module ID. - PrefixPtr[0] = 0; + *PrefixPtr = ID.get(); - // Store the global declaration ID in the second 4 bytes. - PrefixPtr[1] = ID.get(); + // We leave the upper 16 bits to store the module IDs. 48 bits should be + // sufficient to store a declaration ID. + assert(*PrefixPtr < llvm::maskTrailingOnes(48)); return Result; } @@ -111,6 +110,29 @@ void *Decl::operator new(std::size_t Size, const ASTContext &Ctx, return ::operator new(Size + Extra, Ctx); } +GlobalDeclID Decl::getGlobalID() const { + if (!isFromASTFile()) + return GlobalDeclID(); + // See the comments in `Decl::operator new` for details. + uint64_t ID = *((const uint64_t *)this - 1); + return GlobalDeclID(ID & llvm::maskTrailingOnes(48)); +} + +unsigned Decl::getOwningModuleID() const { + if (!isFromASTFile()) + return 0; + + uint64_t ID = *((const uint64_t *)this - 1); + return ID >> 48; +} + +void Decl::setOwningModuleID(unsigned ID) { + assert(isFromASTFile() && "Only works on a deserialized declaration"); + uint64_t *IDAddress = (uint64_t *)this - 1; + *IDAddress &= llvm::maskTrailingOnes(48); + *IDAddress |= (uint64_t)ID << 48; +} + Module *Decl::getOwningModuleSlow() const { assert(isFromASTFile() && "Not from AST file?"); return getASTContext().getExternalSource()->getModule(getOwningModuleID()); @@ -2163,3 +2185,7 @@ DependentDiagnostic *DependentDiagnostic::Create(ASTContext &C, return DD; } + +unsigned DeclIDBase::getLocalDeclIndex() const { + return ID & llvm::maskTrailingOnes(32); +} diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp index 75c441293d62..7f2c786547b9 100644 --- a/clang/lib/AST/DeclCXX.cpp +++ b/clang/lib/AST/DeclCXX.cpp @@ -68,8 +68,8 @@ void LazyASTUnresolvedSet::getFromExternalSource(ASTContext &C) const { assert(Source && "getFromExternalSource with no external source"); for (ASTUnresolvedSet::iterator I = Impl.begin(); I != Impl.end(); ++I) - I.setDecl(cast(Source->GetExternalDecl( - GlobalDeclID(reinterpret_cast(I.getDecl()) >> 2)))); + I.setDecl( + cast(Source->GetExternalDecl(GlobalDeclID(I.getDeclID())))); Impl.Decls.setLazy(false); } diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 33cea32c3be6..59338b44db32 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -1658,7 +1658,7 @@ bool ASTReader::ReadSLocEntry(int ID) { unsigned NumFileDecls = Record[7]; if (NumFileDecls && ContextObj) { - const LocalDeclID *FirstDecl = F->FileSortedDecls + Record[6]; + const unaligned_decl_id_t *FirstDecl = F->FileSortedDecls + Record[6]; assert(F->FileSortedDecls && "FILE_SORTED_DECLS not encountered yet ?"); FileDeclIDs[FID] = FileDeclsInfo(F, llvm::ArrayRef(FirstDecl, NumFileDecls)); @@ -3377,26 +3377,11 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F, "duplicate DECL_OFFSET record in AST file"); F.DeclOffsets = (const DeclOffset *)Blob.data(); F.LocalNumDecls = Record[0]; - unsigned LocalBaseDeclID = Record[1]; - F.BaseDeclID = getTotalNumDecls(); - - if (F.LocalNumDecls > 0) { - // Introduce the global -> local mapping for declarations within this - // module. - GlobalDeclMap.insert(std::make_pair( - GlobalDeclID(getTotalNumDecls() + NUM_PREDEF_DECL_IDS), &F)); - - // Introduce the local -> global mapping for declarations within this - // module. - F.DeclRemap.insertOrReplace( - std::make_pair(LocalBaseDeclID, F.BaseDeclID - LocalBaseDeclID)); - - // Introduce the global -> local mapping for declarations within this - // module. - F.GlobalToLocalDeclIDs[&F] = LocalBaseDeclID; + F.BaseDeclIndex = getTotalNumDecls(); + if (F.LocalNumDecls > 0) DeclsLoaded.resize(DeclsLoaded.size() + F.LocalNumDecls); - } + break; } @@ -3631,7 +3616,7 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F, break; case FILE_SORTED_DECLS: - F.FileSortedDecls = (const LocalDeclID *)Blob.data(); + F.FileSortedDecls = (const unaligned_decl_id_t *)Blob.data(); F.NumFileSortedDecls = Record[0]; break; @@ -4058,7 +4043,6 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { RemapBuilder PreprocessedEntityRemap(F.PreprocessedEntityRemap); RemapBuilder SubmoduleRemap(F.SubmoduleRemap); RemapBuilder SelectorRemap(F.SelectorRemap); - RemapBuilder DeclRemap(F.DeclRemap); RemapBuilder TypeRemap(F.TypeRemap); auto &ImportedModuleVector = F.TransitiveImports; @@ -4097,8 +4081,6 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { endian::readNext(Data); uint32_t SelectorIDOffset = endian::readNext(Data); - uint32_t DeclIDOffset = - endian::readNext(Data); uint32_t TypeIndexOffset = endian::readNext(Data); @@ -4116,11 +4098,7 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const { PreprocessedEntityRemap); mapOffset(SubmoduleIDOffset, OM->BaseSubmoduleID, SubmoduleRemap); mapOffset(SelectorIDOffset, OM->BaseSelectorID, SelectorRemap); - mapOffset(DeclIDOffset, OM->BaseDeclID, DeclRemap); mapOffset(TypeIndexOffset, OM->BaseTypeIndex, TypeRemap); - - // Global -> local mappings. - F.GlobalToLocalDeclIDs[OM] = DeclIDOffset; } } @@ -4645,7 +4623,7 @@ ASTReader::ASTReadResult ASTReader::ReadAST(StringRef FileName, ModuleKind Type, // that we load any additional categories. if (ContextObj) { for (unsigned I = 0, N = ObjCClassesLoaded.size(); I != N; ++I) { - loadObjCCategories(GlobalDeclID(ObjCClassesLoaded[I]->getGlobalID()), + loadObjCCategories(ObjCClassesLoaded[I]->getGlobalID(), ObjCClassesLoaded[I], PreviousGeneration); } } @@ -7644,18 +7622,25 @@ CXXBaseSpecifier *ASTReader::GetExternalCXXBaseSpecifiers(uint64_t Offset) { GlobalDeclID ASTReader::getGlobalDeclID(ModuleFile &F, LocalDeclID LocalID) const { - DeclID ID = LocalID.get(); - if (ID < NUM_PREDEF_DECL_IDS) - return GlobalDeclID(ID); + if (LocalID.get() < NUM_PREDEF_DECL_IDS) + return GlobalDeclID(LocalID.get()); + + unsigned OwningModuleFileIndex = LocalID.getModuleFileIndex(); + DeclID ID = LocalID.getLocalDeclIndex(); if (!F.ModuleOffsetMap.empty()) ReadModuleOffsetMap(F); - ContinuousRangeMap::iterator I = - F.DeclRemap.find(ID - NUM_PREDEF_DECL_IDS); - assert(I != F.DeclRemap.end() && "Invalid index into decl index remap"); + ModuleFile *OwningModuleFile = + OwningModuleFileIndex == 0 + ? &F + : F.TransitiveImports[OwningModuleFileIndex - 1]; + + if (OwningModuleFileIndex == 0) + ID -= NUM_PREDEF_DECL_IDS; - return GlobalDeclID(ID + I->second); + uint64_t NewModuleFileIndex = OwningModuleFile->Index + 1; + return GlobalDeclID(ID, NewModuleFileIndex); } bool ASTReader::isDeclIDFromModule(GlobalDeclID ID, ModuleFile &M) const { @@ -7663,31 +7648,33 @@ bool ASTReader::isDeclIDFromModule(GlobalDeclID ID, ModuleFile &M) const { if (ID.get() < NUM_PREDEF_DECL_IDS) return false; - return ID.get() - NUM_PREDEF_DECL_IDS >= M.BaseDeclID && - ID.get() - NUM_PREDEF_DECL_IDS < M.BaseDeclID + M.LocalNumDecls; + unsigned ModuleFileIndex = ID.getModuleFileIndex(); + return M.Index == ModuleFileIndex - 1; +} + +ModuleFile *ASTReader::getOwningModuleFile(GlobalDeclID ID) const { + // Predefined decls aren't from any module. + if (ID.get() < NUM_PREDEF_DECL_IDS) + return nullptr; + + uint64_t ModuleFileIndex = ID.getModuleFileIndex(); + assert(ModuleFileIndex && "Untranslated Local Decl?"); + + return &getModuleManager()[ModuleFileIndex - 1]; } -ModuleFile *ASTReader::getOwningModuleFile(const Decl *D) { +ModuleFile *ASTReader::getOwningModuleFile(const Decl *D) const { if (!D->isFromASTFile()) return nullptr; - GlobalDeclMapType::const_iterator I = - GlobalDeclMap.find(GlobalDeclID(D->getGlobalID())); - assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); - return I->second; + + return getOwningModuleFile(D->getGlobalID()); } SourceLocation ASTReader::getSourceLocationForDeclID(GlobalDeclID ID) { if (ID.get() < NUM_PREDEF_DECL_IDS) return SourceLocation(); - unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; - - if (Index > DeclsLoaded.size()) { - Error("declaration ID out-of-range for AST file"); - return SourceLocation(); - } - - if (Decl *D = DeclsLoaded[Index]) + if (Decl *D = GetExistingDecl(ID)) return D->getLocation(); SourceLocation Loc; @@ -7754,8 +7741,19 @@ static Decl *getPredefinedDecl(ASTContext &Context, PredefinedDeclIDs ID) { llvm_unreachable("PredefinedDeclIDs unknown enum value"); } +unsigned ASTReader::translateGlobalDeclIDToIndex(GlobalDeclID GlobalID) const { + ModuleFile *OwningModuleFile = getOwningModuleFile(GlobalID); + if (!OwningModuleFile) { + assert(GlobalID.get() < NUM_PREDEF_DECL_IDS && "Untransalted Global ID?"); + return GlobalID.get(); + } + + return OwningModuleFile->BaseDeclIndex + GlobalID.getLocalDeclIndex(); +} + Decl *ASTReader::GetExistingDecl(GlobalDeclID ID) { assert(ContextObj && "reading decl with no AST context"); + if (ID.get() < NUM_PREDEF_DECL_IDS) { Decl *D = getPredefinedDecl(*ContextObj, (PredefinedDeclIDs)ID); if (D) { @@ -7768,7 +7766,7 @@ Decl *ASTReader::GetExistingDecl(GlobalDeclID ID) { return D; } - unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; + unsigned Index = translateGlobalDeclIDToIndex(ID); if (Index >= DeclsLoaded.size()) { assert(0 && "declaration ID out-of-range for AST file"); @@ -7783,7 +7781,7 @@ Decl *ASTReader::GetDecl(GlobalDeclID ID) { if (ID.get() < NUM_PREDEF_DECL_IDS) return GetExistingDecl(ID); - unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; + unsigned Index = translateGlobalDeclIDToIndex(ID); if (Index >= DeclsLoaded.size()) { assert(0 && "declaration ID out-of-range for AST file"); @@ -7802,20 +7800,31 @@ Decl *ASTReader::GetDecl(GlobalDeclID ID) { LocalDeclID ASTReader::mapGlobalIDToModuleFileGlobalID(ModuleFile &M, GlobalDeclID GlobalID) { - DeclID ID = GlobalID.get(); - if (ID < NUM_PREDEF_DECL_IDS) + if (GlobalID.get() < NUM_PREDEF_DECL_IDS) + return LocalDeclID(GlobalID.get()); + + if (!M.ModuleOffsetMap.empty()) + ReadModuleOffsetMap(M); + + ModuleFile *Owner = getOwningModuleFile(GlobalID); + DeclID ID = GlobalID.getLocalDeclIndex(); + + if (Owner == &M) { + ID += NUM_PREDEF_DECL_IDS; return LocalDeclID(ID); + } - GlobalDeclMapType::const_iterator I = GlobalDeclMap.find(GlobalID); - assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); - ModuleFile *Owner = I->second; + uint64_t OrignalModuleFileIndex = 0; + for (unsigned I = 0; I < M.TransitiveImports.size(); I++) + if (M.TransitiveImports[I] == Owner) { + OrignalModuleFileIndex = I + 1; + break; + } - llvm::DenseMap::iterator Pos = - M.GlobalToLocalDeclIDs.find(Owner); - if (Pos == M.GlobalToLocalDeclIDs.end()) + if (!OrignalModuleFileIndex) return LocalDeclID(); - return LocalDeclID(ID - Owner->BaseDeclID + Pos->second); + return LocalDeclID(ID, OrignalModuleFileIndex); } GlobalDeclID ASTReader::ReadDeclID(ModuleFile &F, const RecordData &Record, @@ -7894,32 +7903,34 @@ void ASTReader::FindExternalLexicalDecls( namespace { -class DeclIDComp { +class UnalignedDeclIDComp { ASTReader &Reader; ModuleFile &Mod; public: - DeclIDComp(ASTReader &Reader, ModuleFile &M) : Reader(Reader), Mod(M) {} + UnalignedDeclIDComp(ASTReader &Reader, ModuleFile &M) + : Reader(Reader), Mod(M) {} - bool operator()(LocalDeclID L, LocalDeclID R) const { + bool operator()(unaligned_decl_id_t L, unaligned_decl_id_t R) const { SourceLocation LHS = getLocation(L); SourceLocation RHS = getLocation(R); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - bool operator()(SourceLocation LHS, LocalDeclID R) const { + bool operator()(SourceLocation LHS, unaligned_decl_id_t R) const { SourceLocation RHS = getLocation(R); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - bool operator()(LocalDeclID L, SourceLocation RHS) const { + bool operator()(unaligned_decl_id_t L, SourceLocation RHS) const { SourceLocation LHS = getLocation(L); return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS); } - SourceLocation getLocation(LocalDeclID ID) const { + SourceLocation getLocation(unaligned_decl_id_t ID) const { return Reader.getSourceManager().getFileLoc( - Reader.getSourceLocationForDeclID(Reader.getGlobalDeclID(Mod, ID))); + Reader.getSourceLocationForDeclID( + Reader.getGlobalDeclID(Mod, (LocalDeclID)ID))); } }; @@ -7942,8 +7953,8 @@ void ASTReader::FindFileRegionDecls(FileID File, BeginLoc = SM.getLocForStartOfFile(File).getLocWithOffset(Offset); SourceLocation EndLoc = BeginLoc.getLocWithOffset(Length); - DeclIDComp DIDComp(*this, *DInfo.Mod); - ArrayRef::iterator BeginIt = + UnalignedDeclIDComp DIDComp(*this, *DInfo.Mod); + ArrayRef::iterator BeginIt = llvm::lower_bound(DInfo.Decls, BeginLoc, DIDComp); if (BeginIt != DInfo.Decls.begin()) --BeginIt; @@ -7952,17 +7963,18 @@ void ASTReader::FindFileRegionDecls(FileID File, // to backtrack until we find it otherwise we will fail to report that the // region overlaps with an objc container. while (BeginIt != DInfo.Decls.begin() && - GetDecl(getGlobalDeclID(*DInfo.Mod, *BeginIt)) + GetDecl(getGlobalDeclID(*DInfo.Mod, (LocalDeclID)(*BeginIt))) ->isTopLevelDeclInObjCContainer()) --BeginIt; - ArrayRef::iterator EndIt = + ArrayRef::iterator EndIt = llvm::upper_bound(DInfo.Decls, EndLoc, DIDComp); if (EndIt != DInfo.Decls.end()) ++EndIt; - for (ArrayRef::iterator DIt = BeginIt; DIt != EndIt; ++DIt) - Decls.push_back(GetDecl(getGlobalDeclID(*DInfo.Mod, *DIt))); + for (ArrayRef::iterator DIt = BeginIt; DIt != EndIt; + ++DIt) + Decls.push_back(GetDecl(getGlobalDeclID(*DInfo.Mod, (LocalDeclID)(*DIt)))); } bool @@ -8169,7 +8181,6 @@ LLVM_DUMP_METHOD void ASTReader::dump() { dumpModuleIDMap("Global bit offset map", GlobalBitOffsetsMap); dumpModuleIDMap("Global source location entry map", GlobalSLocEntryMap); dumpModuleIDMap("Global type map", GlobalTypeMap); - dumpModuleIDMap("Global declaration map", GlobalDeclMap); dumpModuleIDMap("Global identifier map", GlobalIdentifierMap); dumpModuleIDMap("Global macro map", GlobalMacroMap); dumpModuleIDMap("Global submodule map", GlobalSubmoduleMap); @@ -9185,7 +9196,7 @@ void ASTRecordReader::readUnresolvedSet(LazyASTUnresolvedSet &Set) { while (NumDecls--) { GlobalDeclID ID = readDeclID(); AccessSpecifier AS = (AccessSpecifier) readInt(); - Set.addLazyDecl(getContext(), ID.get(), AS); + Set.addLazyDecl(getContext(), ID, AS); } } diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp index 765c083ce8df..cf2dc32e30b9 100644 --- a/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/clang/lib/Serialization/ASTReaderDecl.cpp @@ -2924,7 +2924,7 @@ void ASTDeclReader::mergeTemplatePattern(RedeclarableTemplateDecl *D, auto *ExistingPattern = Existing->getTemplatedDecl(); RedeclarableResult Result( /*MergeWith*/ ExistingPattern, - GlobalDeclID(DPattern->getCanonicalDecl()->getGlobalID()), IsKeyDecl); + DPattern->getCanonicalDecl()->getGlobalID(), IsKeyDecl); if (auto *DClass = dyn_cast(DPattern)) { // Merge with any existing definition. @@ -3245,11 +3245,10 @@ bool ASTReader::isConsumerInterestedIn(Decl *D) { /// Get the correct cursor and offset for loading a declaration. ASTReader::RecordLocation ASTReader::DeclCursorForID(GlobalDeclID ID, SourceLocation &Loc) { - GlobalDeclMapType::iterator I = GlobalDeclMap.find(ID); - assert(I != GlobalDeclMap.end() && "Corrupted global declaration map"); - ModuleFile *M = I->second; - const DeclOffset &DOffs = - M->DeclOffsets[ID.get() - M->BaseDeclID - NUM_PREDEF_DECL_IDS]; + ModuleFile *M = getOwningModuleFile(ID); + assert(M); + unsigned LocalDeclIndex = ID.getLocalDeclIndex(); + const DeclOffset &DOffs = M->DeclOffsets[LocalDeclIndex]; Loc = ReadSourceLocation(*M, DOffs.getRawLoc()); return RecordLocation(M, DOffs.getBitOffset(M->DeclsBlockStartOffset)); } @@ -3792,7 +3791,6 @@ void ASTReader::markIncompleteDeclChain(Decl *D) { /// Read the declaration at the given offset from the AST file. Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) { - unsigned Index = ID.get() - NUM_PREDEF_DECL_IDS; SourceLocation DeclLoc; RecordLocation Loc = DeclCursorForID(ID, DeclLoc); llvm::BitstreamCursor &DeclsCursor = Loc.F->DeclsCursor; @@ -4123,7 +4121,7 @@ Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) { } assert(D && "Unknown declaration reading AST file"); - LoadedDecl(Index, D); + LoadedDecl(translateGlobalDeclIDToIndex(ID), D); // Set the DeclContext before doing any deserialization, to make sure internal // calls to Decl::getASTContext() by Decl's methods will find the // TranslationUnitDecl without crashing. @@ -4449,7 +4447,7 @@ namespace { M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap, Compare); if (Result == M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap || - Result->DefinitionID != LocalID) { + Result->getDefinitionID() != LocalID) { // We didn't find anything. If the class definition is in this module // file, then the module files it depends on cannot have any categories, // so suppress further lookup. diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 953990a4aca6..ee3e687636e6 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -3357,12 +3357,10 @@ void ASTWriter::WriteTypeDeclOffsets() { Abbrev = std::make_shared(); Abbrev->Add(BitCodeAbbrevOp(DECL_OFFSET)); Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of declarations - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // base decl ID Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // declarations block unsigned DeclOffsetAbbrev = Stream.EmitAbbrev(std::move(Abbrev)); { - RecordData::value_type Record[] = {DECL_OFFSET, DeclOffsets.size(), - FirstDeclID.get() - NUM_PREDEF_DECL_IDS}; + RecordData::value_type Record[] = {DECL_OFFSET, DeclOffsets.size()}; Stream.EmitRecordWithBlob(DeclOffsetAbbrev, Record, bytes(DeclOffsets)); } } @@ -5423,7 +5421,6 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, M.NumPreprocessedEntities); writeBaseIDOrNone(M.BaseSubmoduleID, M.LocalNumSubmodules); writeBaseIDOrNone(M.BaseSelectorID, M.LocalNumSelectors); - writeBaseIDOrNone(M.BaseDeclID, M.LocalNumDecls); writeBaseIDOrNone(M.BaseTypeIndex, M.LocalNumTypes); } } @@ -6618,13 +6615,11 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) { // Note, this will get called multiple times, once one the reader starts up // and again each time it's done reading a PCH or module. - FirstDeclID = LocalDeclID(NUM_PREDEF_DECL_IDS + Chain->getTotalNumDecls()); FirstTypeID = NUM_PREDEF_TYPE_IDS + Chain->getTotalNumTypes(); FirstIdentID = NUM_PREDEF_IDENT_IDS + Chain->getTotalNumIdentifiers(); FirstMacroID = NUM_PREDEF_MACRO_IDS + Chain->getTotalNumMacros(); FirstSubmoduleID = NUM_PREDEF_SUBMODULE_IDS + Chain->getTotalNumSubmodules(); FirstSelectorID = NUM_PREDEF_SELECTOR_IDS + Chain->getTotalNumSelectors(); - NextDeclID = FirstDeclID; NextTypeID = FirstTypeID; NextIdentID = FirstIdentID; NextMacroID = FirstMacroID; diff --git a/clang/lib/Serialization/ModuleFile.cpp b/clang/lib/Serialization/ModuleFile.cpp index 2c42d33a8f5d..f64a59bd9489 100644 --- a/clang/lib/Serialization/ModuleFile.cpp +++ b/clang/lib/Serialization/ModuleFile.cpp @@ -87,7 +87,6 @@ LLVM_DUMP_METHOD void ModuleFile::dump() { << " Number of types: " << LocalNumTypes << '\n'; dumpLocalRemap("Type index local -> global map", TypeRemap); - llvm::errs() << " Base decl ID: " << BaseDeclID << '\n' + llvm::errs() << " Base decl index: " << BaseDeclIndex << '\n' << " Number of decls: " << LocalNumDecls << '\n'; - dumpLocalRemap("Decl ID local -> global map", DeclRemap); } diff --git a/clang/test/Modules/no-transitive-decls-change.cppm b/clang/test/Modules/no-transitive-decls-change.cppm new file mode 100644 index 000000000000..42ac061bc90b --- /dev/null +++ b/clang/test/Modules/no-transitive-decls-change.cppm @@ -0,0 +1,112 @@ +// Testing that changing a declaration in an unused module file won't change +// the BMI of the current module file. +// +// RUN: rm -rf %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/m-partA.cppm -emit-reduced-module-interface -o %t/m-partA.pcm +// RUN: %clang_cc1 -std=c++20 %t/m-partA.v1.cppm -emit-reduced-module-interface -o \ +// RUN: %t/m-partA.v1.pcm +// RUN: %clang_cc1 -std=c++20 %t/m-partB.cppm -emit-reduced-module-interface -o %t/m-partB.pcm +// RUN: %clang_cc1 -std=c++20 %t/m.cppm -emit-reduced-module-interface -o %t/m.pcm \ +// RUN: -fmodule-file=m:partA=%t/m-partA.pcm -fmodule-file=m:partB=%t/m-partB.pcm +// RUN: %clang_cc1 -std=c++20 %t/m.cppm -emit-reduced-module-interface -o %t/m.v1.pcm \ +// RUN: -fmodule-file=m:partA=%t/m-partA.v1.pcm -fmodule-file=m:partB=%t/m-partB.pcm +// +// RUN: %clang_cc1 -std=c++20 %t/useBOnly.cppm -emit-reduced-module-interface -o %t/useBOnly.pcm \ +// RUN: -fmodule-file=m=%t/m.pcm -fmodule-file=m:partA=%t/m-partA.pcm \ +// RUN: -fmodule-file=m:partB=%t/m-partB.pcm +// RUN: %clang_cc1 -std=c++20 %t/useBOnly.cppm -emit-reduced-module-interface -o %t/useBOnly.v1.pcm \ +// RUN: -fmodule-file=m=%t/m.v1.pcm -fmodule-file=m:partA=%t/m-partA.v1.pcm \ +// RUN: -fmodule-file=m:partB=%t/m-partB.pcm +// Since useBOnly only uses partB from module M, the change in partA shouldn't affect +// useBOnly. +// RUN: diff %t/useBOnly.pcm %t/useBOnly.v1.pcm &> /dev/null + +//--- m-partA.cppm +export module m:partA; + +namespace A_Impl { + inline int getAImpl() { + return 43; + } + + inline int getA2Impl() { + return 43; + } +} + +namespace A { + using A_Impl::getAImpl; +} + +export inline int getA() { + return 43; +} + +export inline int getA2(int) { + return 88; +} + +//--- m-partA.v1.cppm +export module m:partA; + +namespace A_Impl { + inline int getAImpl() { + return 43; + } + + inline int getA2Impl() { + return 43; + } +} + +namespace A { + using A_Impl::getAImpl; + // Adding a new declaration without introducing a new declaration name. + using A_Impl::getA2Impl; +} + +inline int getA() { + return 43; +} + +inline int getA2(int) { + return 88; +} + +// Now we add a new declaration without introducing new identifier and new types. +// The consuming module which didn't use m:partA completely is expected to be +// not changed. +inline int getA(int) { + return 88; +} + +//--- m-partB.cppm +export module m:partB; + +export inline int getB() { + return 430; +} + +//--- m.cppm +export module m; +export import :partA; +export import :partB; + +//--- useBOnly.cppm +export module useBOnly; +import m; + +export inline int get() { + return getB(); +} + +//--- useAOnly.cppm +export module useAOnly; +import m; + +export inline int get() { + A a; + return a.getValue(); +} -- GitLab From df6750eaa80099a1e96439bc060a18a26d7212e6 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Fri, 7 Jun 2024 13:23:15 +0100 Subject: [PATCH 240/462] [AMDGPU] Fix interaction between WQM and llvm.amdgcn.init.exec (#93680) Whole quad mode requires inserting a copy of the initial EXEC mask. In a function that also uses llvm.amdgcn.init.exec, insert the COPY after initializing EXEC. --- llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp | 29 +++++++++----- llvm/test/CodeGen/AMDGPU/wqm.ll | 46 ++++++++++++++++++++++ 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp index 5b4c44302fa6..913942dda19d 100644 --- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp +++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp @@ -225,7 +225,7 @@ private: void lowerCopyInstrs(); void lowerKillInstrs(bool IsWQM); void lowerInitExec(MachineInstr &MI); - void lowerInitExecInstrs(); + MachineBasicBlock::iterator lowerInitExecInstrs(MachineBasicBlock &Entry); public: static char ID; @@ -1648,9 +1648,23 @@ void SIWholeQuadMode::lowerInitExec(MachineInstr &MI) { LIS->createAndComputeVirtRegInterval(CountReg); } -void SIWholeQuadMode::lowerInitExecInstrs() { - for (MachineInstr *MI : InitExecInstrs) +/// Lower INIT_EXEC instructions. Return a suitable insert point in \p Entry +/// for instructions that depend on EXEC. +MachineBasicBlock::iterator +SIWholeQuadMode::lowerInitExecInstrs(MachineBasicBlock &Entry) { + MachineBasicBlock::iterator InsertPt = Entry.getFirstNonPHI(); + + for (MachineInstr *MI : InitExecInstrs) { + // Try to handle undefined cases gracefully: + // - multiple INIT_EXEC instructions + // - INIT_EXEC instructions not in the entry block + if (MI->getParent() == &Entry) + InsertPt = std::next(MI->getIterator()); + lowerInitExec(*MI); + } + + return InsertPt; } bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { @@ -1701,19 +1715,16 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { LiveMaskReg = Exec; + MachineBasicBlock &Entry = MF.front(); + MachineBasicBlock::iterator EntryMI = lowerInitExecInstrs(Entry); + // Shader is simple does not need any state changes or any complex lowering if (!(GlobalFlags & (StateWQM | StateStrict)) && LowerToCopyInstrs.empty() && LowerToMovInstrs.empty() && KillInstrs.empty()) { - lowerInitExecInstrs(); lowerLiveMaskQueries(); return !InitExecInstrs.empty() || !LiveMaskQueries.empty(); } - lowerInitExecInstrs(); - - MachineBasicBlock &Entry = MF.front(); - MachineBasicBlock::iterator EntryMI = Entry.getFirstNonPHI(); - // Store a copy of the original live mask when required if (NeedsLiveMask || (GlobalFlags & StateWQM)) { LiveMaskReg = MRI->createVirtualRegister(TRI->getBoolRC()); diff --git a/llvm/test/CodeGen/AMDGPU/wqm.ll b/llvm/test/CodeGen/AMDGPU/wqm.ll index 6fcf5067b022..3bf6c104a025 100644 --- a/llvm/test/CodeGen/AMDGPU/wqm.ll +++ b/llvm/test/CodeGen/AMDGPU/wqm.ll @@ -3395,6 +3395,52 @@ main_body: ret void } +; Test the interaction between wqm and llvm.amdgcn.init.exec. +define amdgpu_gs void @wqm_init_exec() { +; GFX9-W64-LABEL: wqm_init_exec: +; GFX9-W64: ; %bb.0: ; %bb +; GFX9-W64-NEXT: s_mov_b64 exec, -1 +; GFX9-W64-NEXT: s_mov_b32 s0, 0 +; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-W64-NEXT: s_mov_b32 s1, s0 +; GFX9-W64-NEXT: s_mov_b32 s2, s0 +; GFX9-W64-NEXT: s_mov_b32 s3, s0 +; GFX9-W64-NEXT: v_mov_b32_e32 v1, v0 +; GFX9-W64-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-W64-NEXT: v_mov_b32_e32 v3, v0 +; GFX9-W64-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 +; GFX9-W64-NEXT: s_wqm_b64 exec, exec +; GFX9-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $exec +; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-W64-NEXT: ds_write_b32 v0, v1 +; GFX9-W64-NEXT: s_endpgm +; +; GFX10-W32-LABEL: wqm_init_exec: +; GFX10-W32: ; %bb.0: ; %bb +; GFX10-W32-NEXT: s_mov_b32 exec_lo, -1 +; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo +; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 +; GFX10-W32-NEXT: s_mov_b32 s0, 0 +; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo +; GFX10-W32-NEXT: s_mov_b32 s2, s0 +; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s1 +; GFX10-W32-NEXT: v_mov_b32_e32 v1, v0 +; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0 +; GFX10-W32-NEXT: v_mov_b32_e32 v3, v0 +; GFX10-W32-NEXT: v_mov_b32_e32 v4, s0 +; GFX10-W32-NEXT: s_mov_b32 s1, s0 +; GFX10-W32-NEXT: s_mov_b32 s3, s0 +; GFX10-W32-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 +; GFX10-W32-NEXT: ds_write_b32 v0, v4 +; GFX10-W32-NEXT: s_endpgm +bb: + call void @llvm.amdgcn.init.exec(i64 -1) + call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> zeroinitializer, <4 x i32> zeroinitializer, i32 0, i32 0, i32 0) + %i = call i32 @llvm.amdgcn.wqm.i32(i32 0) + store i32 %i, i32 addrspace(3)* null, align 4 + ret void +} + declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #1 declare void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float>, i32, i32, <8 x i32>, i32, i32) #1 -- GitLab From acc927ac2369605c4ec9268d542bb51072d73794 Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Fri, 7 Jun 2024 07:31:37 -0500 Subject: [PATCH 241/462] [Frontend][OpenMP] Sort all the things in OMP.td, NFC (#94653) The file OMP.td is becoming tedious to update by hand due to the seemingly random ordering of various items in it. This patch brings order to it by sorting most of the contents. The clause definitions are sorted alphabetically with respect to the spelling of the clause.[1] The directive definitions are split into two leaf directives and compound directives.[2] Within each, definitions are sorted alphabetically with respect to the spelling, with the exception that "end xyz" directives are placed immediately following the definition of "xyz".[3] Within each directive definition, the lists of clauses are also sorted alphabetically. [1] All spellings are made of lowercase letters, _, or space. Ordering that includes non-letters follows the order assumed by the `sort` utility. [2] Compound directives refer to the consituent leaf directives, hence the leaf definitions must come first. [3] Some of the "end xyz" directives have properties derived from the corresponding "xyz" directive. This exception guarantees that "xyz" precedes the "end xyz". --- llvm/include/llvm/Frontend/OpenMP/OMP.td | 3066 +++++++++++----------- 1 file changed, 1548 insertions(+), 1518 deletions(-) diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td index 609df6a4c54a..afb8d5f72575 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMP.td +++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td @@ -28,151 +28,48 @@ def OpenMP : DirectiveLanguage { } //===----------------------------------------------------------------------===// -// Definition of OpenMP clauses +// Definitions of OpenMP clauses +// Sorted alphabetically wrt clause spelling. //===----------------------------------------------------------------------===// -def OMPC_Allocator : Clause<"allocator"> { - let clangClass = "OMPAllocatorClause"; - let flangClass = "ScalarIntExpr"; -} -def OMPC_If : Clause<"if"> { - let clangClass = "OMPIfClause"; - let flangClass = "OmpIfClause"; -} -def OMPC_Final : Clause<"final"> { - let clangClass = "OMPFinalClause"; - let flangClass = "ScalarLogicalExpr"; -} -def OMPC_NumThreads : Clause<"num_threads"> { - let clangClass = "OMPNumThreadsClause"; - let flangClass = "ScalarIntExpr"; -} -def OMPC_SafeLen : Clause<"safelen"> { - let clangClass = "OMPSafelenClause"; - let flangClass = "ScalarIntConstantExpr"; -} -def OMPC_SimdLen : Clause<"simdlen"> { - let clangClass = "OMPSimdlenClause"; - let flangClass = "ScalarIntConstantExpr"; -} -def OMPC_Collapse : Clause<"collapse"> { - let clangClass = "OMPCollapseClause"; - let flangClass = "ScalarIntConstantExpr"; -} -def OMPC_Default : Clause<"default"> { - let clangClass = "OMPDefaultClause"; - let flangClass = "OmpDefaultClause"; -} -def OMPC_Private : Clause<"private"> { - let clangClass = "OMPPrivateClause"; - let flangClass = "OmpObjectList"; -} -def OMPC_Sizes: Clause<"sizes"> { - let clangClass = "OMPSizesClause"; - let flangClass = "ScalarIntExpr"; - let isValueList = true; - } -def OMPC_Full: Clause<"full"> { - let clangClass = "OMPFullClause"; -} -def OMPC_Partial: Clause<"partial"> { - let clangClass = "OMPPartialClause"; - let flangClass = "ScalarIntConstantExpr"; - let isValueOptional = true; - } -def OMPC_FirstPrivate : Clause<"firstprivate"> { - let clangClass = "OMPFirstprivateClause"; - let flangClass = "OmpObjectList"; +def OMPC_Acquire : Clause<"acquire"> { + let clangClass = "OMPAcquireClause"; } -def OMPC_LastPrivate : Clause<"lastprivate"> { - let clangClass = "OMPLastprivateClause"; - let flangClass = "OmpObjectList"; +def OMPC_AcqRel : Clause<"acq_rel"> { + let clangClass = "OMPAcqRelClause"; } -def OMPC_Shared : Clause<"shared"> { - let clangClass = "OMPSharedClause"; - let flangClass = "OmpObjectList"; +def OMPC_AdjustArgs : Clause<"adjust_args"> { } -def OMPC_Reduction : Clause<"reduction"> { - let clangClass = "OMPReductionClause"; - let flangClass = "OmpReductionClause"; +def OMPC_Affinity : Clause<"affinity"> { + let clangClass = "OMPAffinityClause"; } -def OMPC_Linear : Clause<"linear"> { - let clangClass = "OMPLinearClause"; - let flangClass = "OmpLinearClause"; +def OMPC_Align : Clause<"align"> { + let clangClass = "OMPAlignClause"; } def OMPC_Aligned : Clause<"aligned"> { let clangClass = "OMPAlignedClause"; let flangClass = "OmpAlignedClause"; } -def OMPC_Copyin : Clause<"copyin"> { - let clangClass = "OMPCopyinClause"; - let flangClass = "OmpObjectList"; +def OMPC_Allocate : Clause<"allocate"> { + let clangClass = "OMPAllocateClause"; + let flangClass = "OmpAllocateClause"; } -def OMPC_CopyPrivate : Clause<"copyprivate"> { - let clangClass = "OMPCopyprivateClause"; - let flangClass = "OmpObjectList"; +def OMPC_Allocator : Clause<"allocator"> { + let clangClass = "OMPAllocatorClause"; + let flangClass = "ScalarIntExpr"; } -def OMP_PROC_BIND_master : ClauseVal<"master",2,1> {} -def OMP_PROC_BIND_close : ClauseVal<"close",3,1> {} -def OMP_PROC_BIND_spread : ClauseVal<"spread",4,1> {} -def OMP_PROC_BIND_primary : ClauseVal<"primary",5,1> {} -def OMP_PROC_BIND_default : ClauseVal<"default",6,0> {} -def OMP_PROC_BIND_unknown : ClauseVal<"unknown",7,0> { let isDefault = true; } -def OMPC_ProcBind : Clause<"proc_bind"> { - let clangClass = "OMPProcBindClause"; - let flangClass = "OmpProcBindClause"; - let enumClauseValue = "ProcBindKind"; - let allowedClauseValues = [ - OMP_PROC_BIND_primary, - OMP_PROC_BIND_master, - OMP_PROC_BIND_close, - OMP_PROC_BIND_spread, - OMP_PROC_BIND_default, - OMP_PROC_BIND_unknown - ]; +def OMPC_AppendArgs : Clause<"append_args"> { } - -def OMP_SCHEDULE_Static : ClauseVal<"static", 2, 1> {} -def OMP_SCHEDULE_Dynamic : ClauseVal<"dynamic", 3, 1> {} -def OMP_SCHEDULE_Guided : ClauseVal<"guided", 4, 1> {} -def OMP_SCHEDULE_Auto : ClauseVal<"auto", 5, 1> {} -def OMP_SCHEDULE_Runtime : ClauseVal<"runtime", 6, 1> {} -def OMP_SCHEDULE_Default : ClauseVal<"default", 7, 0> { let isDefault = 1; } - -def OMPC_Schedule : Clause<"schedule"> { - let clangClass = "OMPScheduleClause"; - let flangClass = "OmpScheduleClause"; - let enumClauseValue = "ScheduleKind"; - let allowedClauseValues = [ - OMP_SCHEDULE_Static, - OMP_SCHEDULE_Dynamic, - OMP_SCHEDULE_Guided, - OMP_SCHEDULE_Auto, - OMP_SCHEDULE_Runtime, - OMP_SCHEDULE_Default - ]; +def OMPC_At : Clause<"at"> { + let clangClass = "OMPAtClause"; } - -def OMP_MEMORY_ORDER_SeqCst : ClauseVal<"seq_cst", 1, 1> {} -def OMP_MEMORY_ORDER_AcqRel : ClauseVal<"acq_rel", 2, 1> {} -def OMP_MEMORY_ORDER_Acquire : ClauseVal<"acquire", 3, 1> {} -def OMP_MEMORY_ORDER_Release : ClauseVal<"release", 4, 1> {} -def OMP_MEMORY_ORDER_Relaxed : ClauseVal<"relaxed", 5, 1> {} -def OMP_MEMORY_ORDER_Default : ClauseVal<"default", 6, 0> { - let isDefault = 1; +def OMPC_AtomicDefaultMemOrder : Clause<"atomic_default_mem_order"> { + let clangClass = "OMPAtomicDefaultMemOrderClause"; + let flangClass = "OmpAtomicDefaultMemOrderClause"; } -def OMPC_MemoryOrder : Clause<"memory_order"> { - let enumClauseValue = "MemoryOrderKind"; - let allowedClauseValues = [ - OMP_MEMORY_ORDER_SeqCst, - OMP_MEMORY_ORDER_AcqRel, - OMP_MEMORY_ORDER_Acquire, - OMP_MEMORY_ORDER_Release, - OMP_MEMORY_ORDER_Relaxed, - OMP_MEMORY_ORDER_Default - ]; +def OMPC_Bind : Clause<"bind"> { + let clangClass = "OMPBindClause"; } - def OMP_CANCELLATION_CONSTRUCT_Parallel : ClauseVal<"parallel", 1, 1> {} def OMP_CANCELLATION_CONSTRUCT_Loop : ClauseVal<"loop", 2, 1> {} def OMP_CANCELLATION_CONSTRUCT_Sections : ClauseVal<"sections", 3, 1> {} @@ -180,7 +77,6 @@ def OMP_CANCELLATION_CONSTRUCT_Taskgroup : ClauseVal<"taskgroup", 4, 1> {} def OMP_CANCELLATION_CONSTRUCT_None : ClauseVal<"none", 5, 0> { let isDefault = 1; } - def OMPC_CancellationConstructType : Clause<"cancellation_construct_type"> { let enumClauseValue = "CancellationConstructType"; let allowedClauseValues = [ @@ -191,35 +87,46 @@ def OMPC_CancellationConstructType : Clause<"cancellation_construct_type"> { OMP_CANCELLATION_CONSTRUCT_None ]; } - -def OMPC_Ordered : Clause<"ordered"> { - let clangClass = "OMPOrderedClause"; +def OMPC_Capture : Clause<"capture"> { + let clangClass = "OMPCaptureClause"; +} +def OMPC_Collapse : Clause<"collapse"> { + let clangClass = "OMPCollapseClause"; let flangClass = "ScalarIntConstantExpr"; - let isValueOptional = true; } -def OMPC_NoWait : Clause<"nowait"> { - let clangClass = "OMPNowaitClause"; +def OMPC_Compare : Clause<"compare"> { + let clangClass = "OMPCompareClause"; } -def OMPC_Untied : Clause<"untied"> { let clangClass = "OMPUntiedClause"; } -def OMPC_Mergeable : Clause<"mergeable"> { - let clangClass = "OMPMergeableClause"; +def OMPC_Copyin : Clause<"copyin"> { + let clangClass = "OMPCopyinClause"; + let flangClass = "OmpObjectList"; +} +def OMPC_CopyPrivate : Clause<"copyprivate"> { + let clangClass = "OMPCopyprivateClause"; + let flangClass = "OmpObjectList"; +} +def OMPC_Default : Clause<"default"> { + let clangClass = "OMPDefaultClause"; + let flangClass = "OmpDefaultClause"; +} +def OMPC_DefaultMap : Clause<"defaultmap"> { + let clangClass = "OMPDefaultmapClause"; + let flangClass = "OmpDefaultmapClause"; } -def OMPC_Read : Clause<"read"> { let clangClass = "OMPReadClause"; } -def OMPC_Write : Clause<"write"> { let clangClass = "OMPWriteClause"; } -def OMPC_Update : Clause<"update"> { let clangClass = "OMPUpdateClause"; } -def OMPC_Capture : Clause<"capture"> { let clangClass = "OMPCaptureClause"; } -def OMPC_Compare : Clause<"compare"> { let clangClass = "OMPCompareClause"; } -def OMPC_Fail : Clause<"fail"> { let clangClass = "OMPFailClause"; } -def OMPC_SeqCst : Clause<"seq_cst"> { let clangClass = "OMPSeqCstClause"; } -def OMPC_AcqRel : Clause<"acq_rel"> { let clangClass = "OMPAcqRelClause"; } -def OMPC_Acquire : Clause<"acquire"> { let clangClass = "OMPAcquireClause"; } -def OMPC_Release : Clause<"release"> { let clangClass = "OMPReleaseClause"; } -def OMPC_Relaxed : Clause<"relaxed"> { let clangClass = "OMPRelaxedClause"; } -def OMPC_Weak : Clause<"weak"> { let clangClass = "OMPWeakClause"; } def OMPC_Depend : Clause<"depend"> { let clangClass = "OMPDependClause"; let flangClass = "OmpDependClause"; } +def OMPC_Depobj : Clause<"depobj"> { + let clangClass = "OMPDepobjClause"; + let isImplicit = true; +} +def OMPC_Destroy : Clause<"destroy"> { + let clangClass = "OMPDestroyClause"; +} +def OMPC_Detach : Clause<"detach"> { + let clangClass = "OMPDetachClause"; +} def OMPC_Device : Clause<"device"> { let clangClass = "OMPDeviceClause"; let flangClass = "OmpDeviceClause"; @@ -227,28 +134,51 @@ def OMPC_Device : Clause<"device"> { def OMPC_DeviceType : Clause<"device_type"> { let flangClass = "OmpDeviceTypeClause"; } -def OMPC_Threads : Clause<"threads"> { let clangClass = "OMPThreadsClause"; } -def OMPC_Simd : Clause<"simd"> { let clangClass = "OMPSIMDClause"; } -def OMPC_Map : Clause<"map"> { - let clangClass = "OMPMapClause"; - let flangClass = "OmpMapClause"; -} -def OMPC_NumTeams : Clause<"num_teams"> { - let clangClass = "OMPNumTeamsClause"; +def OMPC_DistSchedule : Clause<"dist_schedule"> { + let clangClass = "OMPDistScheduleClause"; let flangClass = "ScalarIntExpr"; + let isValueOptional = true; } -def OMPC_ThreadLimit : Clause<"thread_limit"> { - let clangClass = "OMPThreadLimitClause"; - let flangClass = "ScalarIntExpr"; +def OMPC_Doacross : Clause<"doacross"> { + let clangClass = "OMPDoacrossClause"; } -def OMPC_Priority : Clause<"priority"> { - let clangClass = "OMPPriorityClause"; +def OMPC_DynamicAllocators : Clause<"dynamic_allocators"> { + let clangClass = "OMPDynamicAllocatorsClause"; +} +def OMPC_Enter : Clause<"enter"> { + let flangClass = "OmpObjectList"; +} +def OMPC_Exclusive : Clause<"exclusive"> { + let clangClass = "OMPExclusiveClause"; +} +def OMPC_Fail : Clause<"fail"> { + let clangClass = "OMPFailClause"; +} +def OMPC_Filter : Clause<"filter"> { + let clangClass = "OMPFilterClause"; let flangClass = "ScalarIntExpr"; } - +def OMPC_Final : Clause<"final"> { + let clangClass = "OMPFinalClause"; + let flangClass = "ScalarLogicalExpr"; +} +def OMPC_FirstPrivate : Clause<"firstprivate"> { + let clangClass = "OMPFirstprivateClause"; + let flangClass = "OmpObjectList"; +} +def OMPC_Flush : Clause<"flush"> { + let clangClass = "OMPFlushClause"; + let isImplicit = true; +} +def OMPC_From : Clause<"from"> { + let clangClass = "OMPFromClause"; + let flangClass = "OmpObjectList"; +} +def OMPC_Full: Clause<"full"> { + let clangClass = "OMPFullClause"; +} def OMP_GRAINSIZE_Strict : ClauseVal<"strict", 1, 1> {} def OMP_GRAINSIZE_Unknown : ClauseVal<"unkonwn", 2, 0> { let isDefault = 1; } - def OMPC_GrainSize : Clause<"grainsize"> { let clangClass = "OMPGrainsizeClause"; let flangClass = "ScalarIntExpr"; @@ -258,101 +188,128 @@ def OMPC_GrainSize : Clause<"grainsize"> { OMP_GRAINSIZE_Unknown ]; } -def OMPC_NoGroup : Clause<"nogroup"> { - let clangClass = "OMPNogroupClause"; -} - -def OMP_NUMTASKS_Strict : ClauseVal<"strict", 1, 1> {} -def OMP_NUMTASKS_Unknown : ClauseVal<"unkonwn", 2, 0> { let isDefault = 1; } - -def OMPC_NumTasks : Clause<"num_tasks"> { - let clangClass = "OMPNumTasksClause"; - let flangClass = "ScalarIntExpr"; - let enumClauseValue = "NumTasksType"; - let allowedClauseValues = [ - OMP_NUMTASKS_Strict, - OMP_NUMTASKS_Unknown - ]; +def OMPC_HasDeviceAddr : Clause<"has_device_addr"> { + let clangClass = "OMPHasDeviceAddrClause"; + let flangClass = "OmpObjectList"; } def OMPC_Hint : Clause<"hint"> { let clangClass = "OMPHintClause"; let flangClass = "ConstantExpr"; } -def OMPC_DistSchedule : Clause<"dist_schedule"> { - let clangClass = "OMPDistScheduleClause"; - let flangClass = "ScalarIntExpr"; - let isValueOptional = true; +def OMPC_If : Clause<"if"> { + let clangClass = "OMPIfClause"; + let flangClass = "OmpIfClause"; } -def OMPC_DefaultMap : Clause<"defaultmap"> { - let clangClass = "OMPDefaultmapClause"; - let flangClass = "OmpDefaultmapClause"; +def OMPC_Inbranch : Clause<"inbranch"> { } -def OMPC_To : Clause<"to"> { - let clangClass = "OMPToClause"; - let flangClass = "OmpObjectList"; +def OMPC_Inclusive : Clause<"inclusive"> { + let clangClass = "OMPInclusiveClause"; } -def OMPC_Enter : Clause<"enter"> { - let flangClass = "OmpObjectList"; +def OMPC_Indirect : Clause<"indirect"> { } -def OMPC_From : Clause<"from"> { - let clangClass = "OMPFromClause"; - let flangClass = "OmpObjectList"; +def OMPC_Init : Clause<"init"> { + let clangClass = "OMPInitClause"; } -def OMPC_UseDevicePtr : Clause<"use_device_ptr"> { - let clangClass = "OMPUseDevicePtrClause"; - let flangClass = "OmpObjectList"; +def OMPC_InReduction : Clause<"in_reduction"> { + let clangClass = "OMPInReductionClause"; + let flangClass = "OmpInReductionClause"; } def OMPC_IsDevicePtr : Clause<"is_device_ptr"> { let clangClass = "OMPIsDevicePtrClause"; let flangClass = "OmpObjectList"; } -def OMPC_HasDeviceAddr : Clause<"has_device_addr"> { - let clangClass = "OMPHasDeviceAddrClause"; +def OMPC_LastPrivate : Clause<"lastprivate"> { + let clangClass = "OMPLastprivateClause"; let flangClass = "OmpObjectList"; } -def OMPC_TaskReduction : Clause<"task_reduction"> { - let clangClass = "OMPTaskReductionClause"; - let flangClass = "OmpReductionClause"; -} -def OMPC_InReduction : Clause<"in_reduction"> { - let clangClass = "OMPInReductionClause"; - let flangClass = "OmpInReductionClause"; -} -def OMPC_UnifiedAddress : Clause<"unified_address"> { - let clangClass = "OMPUnifiedAddressClause"; +def OMPC_Linear : Clause<"linear"> { + let clangClass = "OMPLinearClause"; + let flangClass = "OmpLinearClause"; } -def OMPC_UnifiedSharedMemory : Clause<"unified_shared_memory"> { - let clangClass = "OMPUnifiedSharedMemoryClause"; +def OMPC_Link : Clause<"link"> { + let flangClass = "OmpObjectList"; } -def OMPC_ReverseOffload : Clause<"reverse_offload"> { - let clangClass = "OMPReverseOffloadClause"; +def OMPC_Map : Clause<"map"> { + let clangClass = "OMPMapClause"; + let flangClass = "OmpMapClause"; } -def OMPC_DynamicAllocators : Clause<"dynamic_allocators"> { - let clangClass = "OMPDynamicAllocatorsClause"; +def OMPC_Match : Clause<"match"> { } -def OMPC_AtomicDefaultMemOrder : Clause<"atomic_default_mem_order"> { - let clangClass = "OMPAtomicDefaultMemOrderClause"; - let flangClass = "OmpAtomicDefaultMemOrderClause"; +def OMP_MEMORY_ORDER_SeqCst : ClauseVal<"seq_cst", 1, 1> {} +def OMP_MEMORY_ORDER_AcqRel : ClauseVal<"acq_rel", 2, 1> {} +def OMP_MEMORY_ORDER_Acquire : ClauseVal<"acquire", 3, 1> {} +def OMP_MEMORY_ORDER_Release : ClauseVal<"release", 4, 1> {} +def OMP_MEMORY_ORDER_Relaxed : ClauseVal<"relaxed", 5, 1> {} +def OMP_MEMORY_ORDER_Default : ClauseVal<"default", 6, 0> { + let isDefault = 1; } -def OMPC_At : Clause<"at"> { - let clangClass = "OMPAtClause"; +def OMPC_MemoryOrder : Clause<"memory_order"> { + let enumClauseValue = "MemoryOrderKind"; + let allowedClauseValues = [ + OMP_MEMORY_ORDER_SeqCst, + OMP_MEMORY_ORDER_AcqRel, + OMP_MEMORY_ORDER_Acquire, + OMP_MEMORY_ORDER_Release, + OMP_MEMORY_ORDER_Relaxed, + OMP_MEMORY_ORDER_Default + ]; } -def OMPC_Severity : Clause<"severity"> { - let clangClass = "OMPSeverityClause"; +def OMPC_Mergeable : Clause<"mergeable"> { + let clangClass = "OMPMergeableClause"; } def OMPC_Message : Clause<"message"> { let clangClass = "OMPMessageClause"; } -def OMPC_Allocate : Clause<"allocate"> { - let clangClass = "OMPAllocateClause"; - let flangClass = "OmpAllocateClause"; +def OMPC_Nocontext : Clause<"nocontext"> { + let clangClass = "OMPNocontextClause"; + let flangClass = "ScalarLogicalExpr"; +} +def OMPC_NoGroup : Clause<"nogroup"> { + let clangClass = "OMPNogroupClause"; } def OMPC_NonTemporal : Clause<"nontemporal"> { let clangClass = "OMPNontemporalClause"; let flangClass = "Name"; let isValueList = true; } - +def OMPC_Notinbranch : Clause<"notinbranch"> { +} +def OMPC_Novariants : Clause<"novariants"> { + let clangClass = "OMPNovariantsClause"; + let flangClass = "ScalarLogicalExpr"; +} +def OMPC_NoWait : Clause<"nowait"> { + let clangClass = "OMPNowaitClause"; +} +def OMP_NUMTASKS_Strict : ClauseVal<"strict", 1, 1> {} +def OMP_NUMTASKS_Unknown : ClauseVal<"unkonwn", 2, 0> { let isDefault = 1; } +def OMPC_NumTasks : Clause<"num_tasks"> { + let clangClass = "OMPNumTasksClause"; + let flangClass = "ScalarIntExpr"; + let enumClauseValue = "NumTasksType"; + let allowedClauseValues = [ + OMP_NUMTASKS_Strict, + OMP_NUMTASKS_Unknown + ]; +} +def OMPC_NumTeams : Clause<"num_teams"> { + let clangClass = "OMPNumTeamsClause"; + let flangClass = "ScalarIntExpr"; +} +def OMPC_NumThreads : Clause<"num_threads"> { + let clangClass = "OMPNumThreadsClause"; + let flangClass = "ScalarIntExpr"; +} +def OMPC_OMPX_Attribute : Clause<"ompx_attribute"> { + let clangClass = "OMPXAttributeClause"; +} +def OMPC_OMX_Bare : Clause<"ompx_bare"> { + let clangClass = "OMPXBareClause"; +} +def OMPC_OMPX_DynCGroupMem : Clause<"ompx_dyn_cgroup_mem"> { + let clangClass = "OMPXDynCGroupMemClause"; + let flangClass = "ScalarIntExpr"; +} def OMP_ORDER_concurrent : ClauseVal<"concurrent",1,1> {} def OMP_ORDER_unknown : ClauseVal<"unknown",2,0> { let isDefault = 1; } def OMPC_Order : Clause<"order"> { @@ -364,362 +321,467 @@ def OMPC_Order : Clause<"order"> { OMP_ORDER_concurrent ]; } -def OMPC_Init : Clause<"init"> { - let clangClass = "OMPInitClause"; +def OMPC_Ordered : Clause<"ordered"> { + let clangClass = "OMPOrderedClause"; + let flangClass = "ScalarIntConstantExpr"; + let isValueOptional = true; } -def OMPC_Use : Clause<"use"> { - let clangClass = "OMPUseClause"; +def OMPC_Partial: Clause<"partial"> { + let clangClass = "OMPPartialClause"; + let flangClass = "ScalarIntConstantExpr"; + let isValueOptional = true; } -def OMPC_Destroy : Clause<"destroy"> { - let clangClass = "OMPDestroyClause"; +def OMPC_Priority : Clause<"priority"> { + let clangClass = "OMPPriorityClause"; + let flangClass = "ScalarIntExpr"; } -def OMPC_Novariants : Clause<"novariants"> { - let clangClass = "OMPNovariantsClause"; - let flangClass = "ScalarLogicalExpr"; +def OMPC_Private : Clause<"private"> { + let clangClass = "OMPPrivateClause"; + let flangClass = "OmpObjectList"; } -def OMPC_Nocontext : Clause<"nocontext"> { - let clangClass = "OMPNocontextClause"; - let flangClass = "ScalarLogicalExpr"; +def OMP_PROC_BIND_master : ClauseVal<"master",2,1> {} +def OMP_PROC_BIND_close : ClauseVal<"close",3,1> {} +def OMP_PROC_BIND_spread : ClauseVal<"spread",4,1> {} +def OMP_PROC_BIND_primary : ClauseVal<"primary",5,1> {} +def OMP_PROC_BIND_default : ClauseVal<"default",6,0> {} +def OMP_PROC_BIND_unknown : ClauseVal<"unknown",7,0> { let isDefault = true; } +def OMPC_ProcBind : Clause<"proc_bind"> { + let clangClass = "OMPProcBindClause"; + let flangClass = "OmpProcBindClause"; + let enumClauseValue = "ProcBindKind"; + let allowedClauseValues = [ + OMP_PROC_BIND_primary, + OMP_PROC_BIND_master, + OMP_PROC_BIND_close, + OMP_PROC_BIND_spread, + OMP_PROC_BIND_default, + OMP_PROC_BIND_unknown + ]; } -def OMPC_Detach : Clause<"detach"> { - let clangClass = "OMPDetachClause"; +def OMPC_Read : Clause<"read"> { + let clangClass = "OMPReadClause"; } -def OMPC_Inclusive : Clause<"inclusive"> { - let clangClass = "OMPInclusiveClause"; +def OMPC_Reduction : Clause<"reduction"> { + let clangClass = "OMPReductionClause"; + let flangClass = "OmpReductionClause"; } -def OMPC_Exclusive : Clause<"exclusive"> { - let clangClass = "OMPExclusiveClause"; +def OMPC_Relaxed : Clause<"relaxed"> { + let clangClass = "OMPRelaxedClause"; } -def OMPC_UsesAllocators : Clause<"uses_allocators"> { - let clangClass = "OMPUsesAllocatorsClause"; +def OMPC_Release : Clause<"release"> { + let clangClass = "OMPReleaseClause"; } -def OMPC_Affinity : Clause<"affinity"> { - let clangClass = "OMPAffinityClause"; +def OMPC_ReverseOffload : Clause<"reverse_offload"> { + let clangClass = "OMPReverseOffloadClause"; } -def OMPC_UseDeviceAddr : Clause<"use_device_addr"> { - let clangClass = "OMPUseDeviceAddrClause"; +def OMPC_SafeLen : Clause<"safelen"> { + let clangClass = "OMPSafelenClause"; + let flangClass = "ScalarIntConstantExpr"; +} +def OMP_SCHEDULE_Static : ClauseVal<"static", 2, 1> {} +def OMP_SCHEDULE_Dynamic : ClauseVal<"dynamic", 3, 1> {} +def OMP_SCHEDULE_Guided : ClauseVal<"guided", 4, 1> {} +def OMP_SCHEDULE_Auto : ClauseVal<"auto", 5, 1> {} +def OMP_SCHEDULE_Runtime : ClauseVal<"runtime", 6, 1> {} +def OMP_SCHEDULE_Default : ClauseVal<"default", 7, 0> { let isDefault = 1; } +def OMPC_Schedule : Clause<"schedule"> { + let clangClass = "OMPScheduleClause"; + let flangClass = "OmpScheduleClause"; + let enumClauseValue = "ScheduleKind"; + let allowedClauseValues = [ + OMP_SCHEDULE_Static, + OMP_SCHEDULE_Dynamic, + OMP_SCHEDULE_Guided, + OMP_SCHEDULE_Auto, + OMP_SCHEDULE_Runtime, + OMP_SCHEDULE_Default + ]; +} +def OMPC_SeqCst : Clause<"seq_cst"> { + let clangClass = "OMPSeqCstClause"; +} +def OMPC_Severity : Clause<"severity"> { + let clangClass = "OMPSeverityClause"; +} +def OMPC_Shared : Clause<"shared"> { + let clangClass = "OMPSharedClause"; let flangClass = "OmpObjectList"; } -def OMPC_Uniform : Clause<"uniform"> { - let flangClass = "Name"; +def OMPC_Simd : Clause<"simd"> { + let clangClass = "OMPSIMDClause"; +} +def OMPC_SimdLen : Clause<"simdlen"> { + let clangClass = "OMPSimdlenClause"; + let flangClass = "ScalarIntConstantExpr"; +} +def OMPC_Sizes: Clause<"sizes"> { + let clangClass = "OMPSizesClause"; + let flangClass = "ScalarIntExpr"; let isValueList = true; } -def OMPC_Match : Clause<"match"> {} -def OMPC_AdjustArgs : Clause<"adjust_args"> { } -def OMPC_AppendArgs : Clause<"append_args"> { } -def OMPC_Depobj : Clause<"depobj"> { - let clangClass = "OMPDepobjClause"; - let isImplicit = true; +def OMPC_TaskReduction : Clause<"task_reduction"> { + let clangClass = "OMPTaskReductionClause"; + let flangClass = "OmpReductionClause"; } -def OMPC_Flush : Clause<"flush"> { - let clangClass = "OMPFlushClause"; - let isImplicit = true; +def OMPC_ThreadLimit : Clause<"thread_limit"> { + let clangClass = "OMPThreadLimitClause"; + let flangClass = "ScalarIntExpr"; } def OMPC_ThreadPrivate : Clause<"threadprivate"> { let alternativeName = "threadprivate or thread local"; let isImplicit = true; } +def OMPC_Threads : Clause<"threads"> { + let clangClass = "OMPThreadsClause"; +} +def OMPC_To : Clause<"to"> { + let clangClass = "OMPToClause"; + let flangClass = "OmpObjectList"; +} +def OMPC_UnifiedAddress : Clause<"unified_address"> { + let clangClass = "OMPUnifiedAddressClause"; +} +def OMPC_UnifiedSharedMemory : Clause<"unified_shared_memory"> { + let clangClass = "OMPUnifiedSharedMemoryClause"; +} +def OMPC_Uniform : Clause<"uniform"> { + let flangClass = "Name"; + let isValueList = true; +} def OMPC_Unknown : Clause<"unknown"> { let isImplicit = true; let isDefault = true; } -def OMPC_Link : Clause<"link"> { - let flangClass = "OmpObjectList"; +def OMPC_Untied : Clause<"untied"> { + let clangClass = "OMPUntiedClause"; } -def OMPC_Indirect : Clause<"indirect"> {} -def OMPC_Inbranch : Clause<"inbranch"> {} -def OMPC_Notinbranch : Clause<"notinbranch"> {} -def OMPC_Filter : Clause<"filter"> { - let clangClass = "OMPFilterClause"; - let flangClass = "ScalarIntExpr"; +def OMPC_Update : Clause<"update"> { + let clangClass = "OMPUpdateClause"; } -def OMPC_Align : Clause<"align"> { - let clangClass = "OMPAlignClause"; +def OMPC_Use : Clause<"use"> { + let clangClass = "OMPUseClause"; } -def OMPC_When: Clause<"when"> {} - -def OMPC_Bind : Clause<"bind"> { - let clangClass = "OMPBindClause"; +def OMPC_UsesAllocators : Clause<"uses_allocators"> { + let clangClass = "OMPUsesAllocatorsClause"; } - -def OMPC_OMPX_DynCGroupMem : Clause<"ompx_dyn_cgroup_mem"> { - let clangClass = "OMPXDynCGroupMemClause"; - let flangClass = "ScalarIntExpr"; +def OMPC_UseDeviceAddr : Clause<"use_device_addr"> { + let clangClass = "OMPUseDeviceAddrClause"; + let flangClass = "OmpObjectList"; } - -def OMPC_Doacross : Clause<"doacross"> { - let clangClass = "OMPDoacrossClause"; +def OMPC_UseDevicePtr : Clause<"use_device_ptr"> { + let clangClass = "OMPUseDevicePtrClause"; + let flangClass = "OmpObjectList"; } - -def OMPC_OMPX_Attribute : Clause<"ompx_attribute"> { - let clangClass = "OMPXAttributeClause"; +def OMPC_Weak : Clause<"weak"> { + let clangClass = "OMPWeakClause"; } - -def OMPC_OMX_Bare : Clause<"ompx_bare"> { - let clangClass = "OMPXBareClause"; +def OMPC_When: Clause<"when"> { +} +def OMPC_Write : Clause<"write"> { + let clangClass = "OMPWriteClause"; } //===----------------------------------------------------------------------===// -// Definition of OpenMP directives +// Definitions of OpenMP leaf directives +// Sorted alphabetically wrt directive spelling, except "end xyz" immediately +// follows "xyz". //===----------------------------------------------------------------------===// -def OMP_ThreadPrivate : Directive<"threadprivate"> { +def OMP_Allocate : Directive<"allocate"> { + let allowedOnceClauses = [ + VersionedClause, + VersionedClause, + ]; let association = AS_None; } -def OMP_Parallel : Directive<"parallel"> { +def OMP_Allocators : Directive<"allocators"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, + ]; + let association = AS_Block; +} +def OMP_Assumes : Directive<"assumes"> { + let association = AS_None; +} +def OMP_EndAssumes : Directive<"end assumes"> { + let association = AS_Delimited; +} +def OMP_Atomic : Directive<"atomic"> { + let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; let association = AS_Block; } -def OMP_Task : Directive<"task"> { +def OMP_Barrier : Directive<"barrier"> { + let association = AS_None; +} +def OMP_BeginAssumes : Directive<"begin assumes"> { + let association = AS_Delimited; +} +def OMP_BeginDeclareTarget : Directive<"begin declare target"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; + let association = AS_Delimited; +} +def OMP_BeginDeclareVariant : Directive<"begin declare variant"> { + let association = AS_Delimited; +} +def OMP_Cancel : Directive<"cancel"> { let allowedOnceClauses = [ - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause + ]; + let association = AS_None; +} +def OMP_CancellationPoint : Directive<"cancellation point"> { + let association = AS_None; +} +def OMP_Critical : Directive<"critical"> { + let allowedClauses = [ + VersionedClause, ]; let association = AS_Block; } -def OMP_Simd : Directive<"simd"> { +def OMP_DeclareMapper : Directive<"declare mapper"> { + let allowedClauses = [ + VersionedClause, + ]; + let association = AS_None; +} +def OMP_DeclareReduction : Directive<"declare reduction"> { + let association = AS_None; +} +def OMP_DeclareSimd : Directive<"declare simd"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause ]; - let association = AS_Loop; + let allowedExclusiveClauses = [ + VersionedClause, + VersionedClause, + ]; + let association = AS_Declaration; } -def OMP_Tile : Directive<"tile"> { +def OMP_DeclareTarget : Directive<"declare target"> { + let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + ]; let allowedOnceClauses = [ - VersionedClause, + VersionedClause, ]; - let association = AS_Loop; + let association = AS_None; } -def OMP_Unroll : Directive<"unroll"> { - let allowedOnceClauses = [ - VersionedClause, - VersionedClause, +def OMP_EndDeclareTarget : Directive<"end declare target"> { + let association = AS_Delimited; +} +def OMP_DeclareVariant : Directive<"declare variant"> { + let allowedClauses = [ + VersionedClause, ]; - let association = AS_Loop; + let allowedExclusiveClauses = [ + VersionedClause, + VersionedClause, + ]; + let association = AS_Declaration; } -def OMP_For : Directive<"for"> { +def OMP_EndDeclareVariant : Directive<"end declare variant"> { + let association = AS_Delimited; +} +def OMP_Depobj : Directive<"depobj"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + // TODO This should ne `none` instead. Comment carried over from + // OMPKinds.def. + VersionedClause, + VersionedClause, + VersionedClause, + ]; + let association = AS_None; +} +def OMP_dispatch : Directive<"dispatch"> { + let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause ]; - let association = AS_Loop; + let association = AS_Block; } -def OMP_Do : Directive<"do"> { +def OMP_Distribute : Directive<"distribute"> { let allowedClauses = [ - VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, ]; let association = AS_Loop; } -def OMP_Sections : Directive<"sections"> { +def OMP_Do : Directive<"do"> { let allowedClauses = [ - VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause ]; - let association = AS_Block; -} -def OMP_Section : Directive<"section"> { - let association = AS_Separating; -} -def OMP_Single : Directive<"single"> { - let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, + let allowedOnceClauses = [ + VersionedClause, VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, ]; - let association = AS_Block; -} -def OMP_Master : Directive<"master"> { - let association = AS_Block; + let association = AS_Loop; } -def OMP_Critical : Directive<"critical"> { - let allowedClauses = [ - VersionedClause +def OMP_EndDo : Directive<"end do"> { + let allowedOnceClauses = [ + VersionedClause, ]; - let association = AS_Block; -} -def OMP_TaskYield : Directive<"taskyield"> { - let association = AS_None; -} -def OMP_Barrier : Directive<"barrier"> { - let association = AS_None; + // Needed for association computation, since OMP_Do has it "from leafConstructs". + let leafConstructs = OMP_Do.leafConstructs; + let association = OMP_Do.association; } def OMP_Error : Directive<"error"> { let allowedClauses = [ VersionedClause, + VersionedClause, VersionedClause, - VersionedClause - ]; - let association = AS_None; -} -def OMP_TaskWait : Directive<"taskwait"> { - let allowedClauses = [ - VersionedClause, - VersionedClause ]; let association = AS_None; } -def OMP_TaskGroup : Directive<"taskgroup"> { - let allowedClauses = [ - VersionedClause, - VersionedClause - ]; - let association = AS_Block; -} def OMP_Flush : Directive<"flush"> { let allowedOnceClauses = [ VersionedClause, VersionedClause, - VersionedClause, // TODO This should ne `none` instead. Comment carried over from // OMPKinds.def. - VersionedClause + VersionedClause, + VersionedClause, ]; let association = AS_None; } -def OMP_Ordered : Directive<"ordered"> { +def OMP_For : Directive<"for"> { let allowedClauses = [ - VersionedClause, - VersionedClause - ]; - let allowedOnceClauses = [ - VersionedClause, - VersionedClause - ]; - let association = AS_None; - // There is also a block-associated "ordered" directive. + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + ]; + let association = AS_Loop; } -def OMP_Atomic : Directive<"atomic"> { +def OMP_interop : Directive<"interop"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + ]; + let association = AS_None; +} +def OMP_loop : Directive<"loop"> { + let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + ]; + let association = AS_Loop; +} +def OMP_masked : Directive<"masked"> { + let allowedOnceClauses = [ + VersionedClause, ]; let association = AS_Block; } -def OMP_Target : Directive<"target"> { +def OMP_Master : Directive<"master"> { + let association = AS_Block; +} +def OMP_Metadirective : Directive<"metadirective"> { + let allowedClauses = [ + VersionedClause, + ]; + let allowedOnceClauses = [ + VersionedClause, + ]; + let association = AS_None; +} +def OMP_Nothing : Directive<"nothing"> { + let association = AS_None; +} +def OMP_Ordered : Directive<"ordered"> { let allowedClauses = [ - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, ]; - let association = AS_Block; + let association = AS_None; + // There is also a block-associated "ordered" directive. } -def OMP_Teams : Directive<"teams"> { +def OMP_Parallel : Directive<"parallel"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, ]; let association = AS_Block; } -def OMP_Cancel : Directive<"cancel"> { - let allowedOnceClauses = [ - VersionedClause - ]; - let association = AS_None; -} def OMP_Requires : Directive<"requires"> { let allowedOnceClauses = [ VersionedClause, @@ -732,1596 +794,1564 @@ def OMP_Requires : Directive<"requires"> { // // TODO: Correct this supprted version number whenever complete // implementation of reverse_offload is available. - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause + VersionedClause, ]; let association = AS_None; } -def OMP_Nothing : Directive<"nothing"> { - let association = AS_None; +def OMP_Scan : Directive<"scan"> { + let allowedClauses = [ + VersionedClause, + VersionedClause, + ]; + let association = AS_Separating; +} +def OMP_scope : Directive<"scope"> { + let allowedClauses = [ + VersionedClause, + VersionedClause, + ]; + let allowedOnceClauses = [ + VersionedClause, + ]; + let association = AS_Block; +} +def OMP_Section : Directive<"section"> { + let association = AS_Separating; +} +def OMP_Sections : Directive<"sections"> { + let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + ]; + let association = AS_Block; +} +def OMP_EndSections : Directive<"end sections"> { + let allowedOnceClauses = [ + VersionedClause, + ]; + let leafConstructs = OMP_Sections.leafConstructs; + let association = OMP_Sections.association; +} +def OMP_Simd : Directive<"simd"> { + let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + ]; + let allowedOnceClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + ]; + let association = AS_Loop; +} +def OMP_Single : Directive<"single"> { + let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + ]; + let association = AS_Block; +} +def OMP_EndSingle : Directive<"end single"> { + let allowedClauses = [ + VersionedClause, + ]; + let allowedOnceClauses = [ + VersionedClause, + ]; + let leafConstructs = OMP_Single.leafConstructs; + let association = OMP_Single.association; +} +def OMP_Target : Directive<"target"> { + let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + ]; + let allowedOnceClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + ]; + let association = AS_Block; } def OMP_TargetData : Directive<"target data"> { let allowedOnceClauses = [ VersionedClause, - VersionedClause + VersionedClause, ]; let requiredClauses = [ VersionedClause, + VersionedClause, VersionedClause, - VersionedClause ]; let association = AS_Block; } def OMP_TargetEnterData : Directive<"target enter data"> { let allowedClauses = [ - VersionedClause + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, ]; let requiredClauses = [ - VersionedClause + VersionedClause, ]; let association = AS_None; } def OMP_TargetExitData : Directive<"target exit data"> { let allowedClauses = [ - VersionedClause + VersionedClause, ]; let allowedOnceClauses = [ VersionedClause, VersionedClause, - VersionedClause + VersionedClause, ]; let requiredClauses = [ - VersionedClause + VersionedClause, ]; let association = AS_None; } -def OMP_TargetParallel : Directive<"target parallel"> { +def OMP_TargetUpdate : Directive<"target update"> { let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + ]; + let allowedOnceClauses = [ + VersionedClause, VersionedClause, - VersionedClause, VersionedClause, + ]; + let association = AS_None; +} +def OMP_Task : Directive<"task"> { + let allowedClauses = [ + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Target, OMP_Parallel]; + let association = AS_Block; } -def OMP_TargetParallelFor : Directive<"target parallel for"> { +def OMP_TaskGroup : Directive<"taskgroup"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + ]; + let association = AS_Block; +} +def OMP_TaskLoop : Directive<"taskloop"> { + let allowedClauses = [ + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Target, OMP_Parallel, OMP_For]; + let allowedExclusiveClauses = [ + VersionedClause, + VersionedClause, + ]; + let association = AS_Loop; } -def OMP_TargetParallelDo : Directive<"target parallel do"> { +def OMP_TaskWait : Directive<"taskwait"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + ]; + let association = AS_None; +} +def OMP_TaskYield : Directive<"taskyield"> { + let association = AS_None; +} +def OMP_Teams : Directive<"teams"> { + let allowedClauses = [ + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Target, OMP_Parallel, OMP_Do]; + let association = AS_Block; } -def OMP_TargetUpdate : Directive<"target update"> { - let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause - ]; +def OMP_ThreadPrivate : Directive<"threadprivate"> { + let association = AS_None; +} +def OMP_Tile : Directive<"tile"> { let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, ]; + let association = AS_Loop; +} +def OMP_Unknown : Directive<"unknown"> { + let isDefault = true; let association = AS_None; } -def OMP_masked : Directive<"masked"> { +def OMP_Unroll : Directive<"unroll"> { + let allowedOnceClauses = [ + VersionedClause, + VersionedClause, + ]; + let association = AS_Loop; +} +def OMP_Workshare : Directive<"workshare"> { let allowedOnceClauses = [ - VersionedClause + VersionedClause, ]; let association = AS_Block; } -def OMP_ParallelFor : Directive<"parallel for"> { +def OMP_EndWorkshare : Directive<"end workshare"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Parallel, OMP_For]; + let leafConstructs = OMP_Workshare.leafConstructs; + let association = OMP_Workshare.association; } -def OMP_ParallelDo : Directive<"parallel do"> { + +//===----------------------------------------------------------------------===// +// Definitions of OpenMP compound directives +// Sorted alphabetically wrt directive spelling, except "end xyz" immediately +// follows "xyz". +//===----------------------------------------------------------------------===// + +def OMP_DistributeParallelDo : Directive<"distribute parallel do"> { let allowedClauses = [ + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause ]; - let leafConstructs = [OMP_Parallel, OMP_Do]; + let leafConstructs = [OMP_Distribute, OMP_Parallel, OMP_Do]; } -def OMP_ParallelForSimd : Directive<"parallel for simd"> { +def OMP_DistributeParallelDoSimd : Directive<"distribute parallel do simd"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - ]; - let leafConstructs = [OMP_Parallel, OMP_For, OMP_Simd]; -} -def OMP_ParallelDoSimd : Directive<"parallel do simd"> { - let allowedClauses = [ - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - ]; - let allowedOnceClauses = [ VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause ]; - let leafConstructs = [OMP_Parallel, OMP_Do, OMP_Simd]; + let leafConstructs = [OMP_Distribute, OMP_Parallel, OMP_Do, OMP_Simd]; } -def OMP_ParallelMaster : Directive<"parallel master"> { +def OMP_DistributeParallelFor : Directive<"distribute parallel for"> { let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Parallel, OMP_Master]; + let leafConstructs = [OMP_Distribute, OMP_Parallel, OMP_For]; } -def OMP_ParallelMasked : Directive<"parallel masked"> { +def OMP_DistributeParallelForSimd : Directive<"distribute parallel for simd"> { let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Parallel, OMP_masked]; + let leafConstructs = [OMP_Distribute, OMP_Parallel, OMP_For, OMP_Simd]; } -def OMP_ParallelSections : Directive<"parallel sections"> { +def OMP_DistributeSimd : Directive<"distribute simd"> { let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - ]; - let allowedOnceClauses = [ - VersionedClause, - VersionedClause - ]; - let leafConstructs = [OMP_Parallel, OMP_Sections]; -} -def OMP_ForSimd : Directive<"for simd"> { - let allowedClauses = [ + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, + ]; + let allowedOnceClauses = [ VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause ]; - let leafConstructs = [OMP_For, OMP_Simd]; + let leafConstructs = [OMP_Distribute, OMP_Simd]; } def OMP_DoSimd : Directive<"do simd"> { let allowedClauses = [ VersionedClause, - VersionedClause, VersionedClause, VersionedClause, VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause ]; let leafConstructs = [OMP_Do, OMP_Simd]; } -def OMP_CancellationPoint : Directive<"cancellation point"> { - let association = AS_None; -} -def OMP_DeclareReduction : Directive<"declare reduction"> { - let association = AS_None; -} -def OMP_DeclareMapper : Directive<"declare mapper"> { - let allowedClauses = [ - VersionedClause +def OMP_EndDoSimd : Directive<"end do simd"> { + let allowedOnceClauses = [ + VersionedClause, ]; - let association = AS_None; + let leafConstructs = OMP_DoSimd.leafConstructs; + let association = OMP_DoSimd.association; } -def OMP_DeclareSimd : Directive<"declare simd"> { +def OMP_ForSimd : Directive<"for simd"> { let allowedClauses = [ - VersionedClause, VersionedClause, - VersionedClause - ]; - let allowedOnceClauses = [ - VersionedClause - ]; - let allowedExclusiveClauses = [ - VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let association = AS_Declaration; + let leafConstructs = [OMP_For, OMP_Simd]; } -def OMP_TaskLoop : Directive<"taskloop"> { +def OMP_MaskedTaskloop : Directive<"masked taskloop"> { let allowedClauses = [ - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause - ]; - let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let allowedExclusiveClauses = [ - VersionedClause, - VersionedClause - ]; - let association = AS_Loop; + let leafConstructs = [OMP_masked, OMP_TaskLoop]; } -def OMP_TaskLoopSimd : Directive<"taskloop simd"> { +def OMP_MaskedTaskloopSimd : Directive<"masked taskloop simd"> { let allowedClauses = [ - VersionedClause, VersionedClause, VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, VersionedClause, VersionedClause, VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause, - VersionedClause - ]; - let allowedOnceClauses = [ - VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause - ]; - let allowedExclusiveClauses = [ - VersionedClause, - VersionedClause + VersionedClause, ]; - let leafConstructs = [OMP_TaskLoop, OMP_Simd]; + let leafConstructs = [OMP_masked, OMP_TaskLoop, OMP_Simd]; } -def OMP_Distribute : Directive<"distribute"> { +def OMP_MasterTaskloop : Directive<"master taskloop"> { let allowedClauses = [ - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause - ]; - let allowedOnceClauses = [ - VersionedClause, - VersionedClause - ]; - let association = AS_Loop; -} -def OMP_BeginDeclareTarget : Directive<"begin declare target"> { - let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause - ]; - let association = AS_Delimited; -} -def OMP_DeclareTarget : Directive<"declare target"> { - let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause - ]; - let allowedOnceClauses = [ - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let association = AS_None; -} -def OMP_EndDeclareTarget : Directive<"end declare target"> { - let association = AS_Delimited; + let leafConstructs = [OMP_Master, OMP_TaskLoop]; } -def OMP_DistributeParallelFor : Directive<"distribute parallel for"> { +def OMP_MasterTaskloopSimd : Directive<"master taskloop simd"> { let allowedClauses = [ - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Distribute, OMP_Parallel, OMP_For]; + let leafConstructs = [OMP_Master, OMP_TaskLoop, OMP_Simd]; } -def OMP_DistributeParallelDo : Directive<"distribute parallel do"> { +def OMP_ParallelDo : Directive<"parallel do"> { let allowedClauses = [ - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, ]; let allowedOnceClauses = [ VersionedClause, - VersionedClause, VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause, - VersionedClause ]; - let leafConstructs = [OMP_Distribute, OMP_Parallel, OMP_Do]; + let leafConstructs = [OMP_Parallel, OMP_Do]; } -def OMP_DistributeParallelForSimd : Directive<"distribute parallel for simd"> { - let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - ]; - let leafConstructs = [OMP_Distribute, OMP_Parallel, OMP_For, OMP_Simd]; -} -def OMP_DistributeParallelDoSimd : Directive<"distribute parallel do simd"> { - let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause - ]; - let leafConstructs = [OMP_Distribute, OMP_Parallel, OMP_Do, OMP_Simd]; -} -def OMP_DistributeSimd : Directive<"distribute simd"> { +def OMP_ParallelDoSimd : Directive<"parallel do simd"> { let allowedClauses = [ VersionedClause, VersionedClause, VersionedClause, VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause ]; - let leafConstructs = [OMP_Distribute, OMP_Simd]; + let leafConstructs = [OMP_Parallel, OMP_Do, OMP_Simd]; } - -def OMP_TargetParallelForSimd : Directive<"target parallel for simd"> { +def OMP_ParallelFor : Directive<"parallel for"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - ]; - let allowedOnceClauses = [ - VersionedClause, - VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Target, OMP_Parallel, OMP_For, OMP_Simd]; + let leafConstructs = [OMP_Parallel, OMP_For]; } -def OMP_TargetParallelDoSimd : Directive<"target parallel do simd"> { +def OMP_ParallelForSimd : Directive<"parallel for simd"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause ]; - let leafConstructs = [OMP_Target, OMP_Parallel, OMP_Do, OMP_Simd]; + let leafConstructs = [OMP_Parallel, OMP_For, OMP_Simd]; } -def OMP_TargetSimd : Directive<"target simd"> { +def OMP_parallel_loop : Directive<"parallel loop"> { let allowedClauses = [ - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, ]; let allowedOnceClauses = [ + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, ]; - let leafConstructs = [OMP_Target, OMP_Simd]; + let leafConstructs = [OMP_Parallel, OMP_loop]; } -def OMP_TeamsDistribute : Directive<"teams distribute"> { +def OMP_ParallelMasked : Directive<"parallel masked"> { let allowedClauses = [ + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + ]; + let leafConstructs = [OMP_Parallel, OMP_masked]; +} +def OMP_ParallelMaskedTaskloop : + Directive<"parallel masked taskloop"> { + let allowedClauses = [ VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let allowedOnceClauses = [ - VersionedClause - ]; - let leafConstructs = [OMP_Teams, OMP_Distribute]; + let leafConstructs = [OMP_Parallel, OMP_masked, OMP_TaskLoop]; } -def OMP_TeamsDistributeSimd : Directive<"teams distribute simd"> { +def OMP_ParallelMaskedTaskloopSimd : + Directive<"parallel masked taskloop simd"> { let allowedClauses = [ - VersionedClause, VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - ]; - let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, ]; - let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Simd]; + let leafConstructs = [OMP_Parallel, OMP_masked, OMP_TaskLoop, OMP_Simd]; } - -def OMP_TeamsDistributeParallelForSimd : - Directive<"teams distribute parallel for simd"> { +def OMP_ParallelMaster : Directive<"parallel master"> { let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_For, OMP_Simd]; + let leafConstructs = [OMP_Parallel, OMP_Master]; } -def OMP_TeamsDistributeParallelDoSimd : - Directive<"teams distribute parallel do simd"> { +def OMP_ParallelMasterTaskloop : + Directive<"parallel master taskloop"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause - ]; - let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_Do, OMP_Simd]; + let leafConstructs = [OMP_Parallel, OMP_Master, OMP_TaskLoop]; } -def OMP_TeamsDistributeParallelFor : - Directive<"teams distribute parallel for"> { +def OMP_ParallelMasterTaskloopSimd : + Directive<"parallel master taskloop simd"> { let allowedClauses = [ - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_For]; + let leafConstructs = [OMP_Parallel, OMP_Master, OMP_TaskLoop, OMP_Simd]; } -def OMP_TeamsDistributeParallelDo : - Directive<"teams distribute parallel do"> { +def OMP_ParallelSections : Directive<"parallel sections"> { let allowedClauses = [ - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, ]; -let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + let allowedOnceClauses = [ + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause ]; - let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_Do]; + let leafConstructs = [OMP_Parallel, OMP_Sections]; } -def OMP_TargetTeams : Directive<"target teams"> { +def OMP_ParallelWorkshare : Directive<"parallel workshare"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, ]; - let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Target, OMP_Teams]; + let leafConstructs = [OMP_Parallel, OMP_Workshare]; } -def OMP_TargetTeamsDistribute : Directive<"target teams distribute"> { +def OMP_TargetParallel : Directive<"target parallel"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute]; + let leafConstructs = [OMP_Target, OMP_Parallel]; } - -def OMP_TargetTeamsDistributeParallelFor : - Directive<"target teams distribute parallel for"> { +def OMP_TargetParallelDo : Directive<"target parallel do"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - ]; - let allowedOnceClauses = [ - VersionedClause, - ]; - let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_For]; -} -def OMP_TargetTeamsDistributeParallelDo : - Directive<"target teams distribute parallel do"> { - let allowedClauses = [ VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause ]; - let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_Do]; + let leafConstructs = [OMP_Target, OMP_Parallel, OMP_Do]; } -def OMP_TargetTeamsDistributeParallelForSimd : - Directive<"target teams distribute parallel for simd"> { +def OMP_TargetParallelDoSimd : Directive<"target parallel do simd"> { let allowedClauses = [ - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + ]; + let leafConstructs = [OMP_Target, OMP_Parallel, OMP_Do, OMP_Simd]; +} +def OMP_TargetParallelFor : Directive<"target parallel for"> { + let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + ]; + let allowedOnceClauses = [ + VersionedClause, + VersionedClause, + ]; + let leafConstructs = [OMP_Target, OMP_Parallel, OMP_For]; +} +def OMP_TargetParallelForSimd : Directive<"target parallel for simd"> { + let allowedClauses = [ VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, ]; let allowedOnceClauses = [ VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_For, OMP_Simd]; + let leafConstructs = [OMP_Target, OMP_Parallel, OMP_For, OMP_Simd]; } -def OMP_TargetTeamsDistributeParallelDoSimd : - Directive<"target teams distribute parallel do simd"> { +def OMP_target_parallel_loop : Directive<"target parallel loop"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, ]; - let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_Do, OMP_Simd]; + let leafConstructs = [OMP_Target, OMP_Parallel, OMP_loop]; } -def OMP_TargetTeamsDistributeSimd : - Directive<"target teams distribute simd"> { +def OMP_TargetSimd : Directive<"target simd"> { let allowedClauses = [ VersionedClause, VersionedClause, VersionedClause, VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause, VersionedClause, VersionedClause, VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, VersionedClause, VersionedClause, - VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause - ]; - let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Simd]; -} -def OMP_Allocate : Directive<"allocate"> { - let allowedOnceClauses = [ - VersionedClause, - VersionedClause - ]; - let association = AS_None; -} -def OMP_Allocators : Directive<"allocators"> { - let allowedClauses = [ - VersionedClause + VersionedClause, ]; - let association = AS_Block; + let leafConstructs = [OMP_Target, OMP_Simd]; } -def OMP_DeclareVariant : Directive<"declare variant"> { +def OMP_TargetTeams : Directive<"target teams"> { let allowedClauses = [ - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let allowedExclusiveClauses = [ - VersionedClause, - VersionedClause + let allowedOnceClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let association = AS_Declaration; + let leafConstructs = [OMP_Target, OMP_Teams]; } -def OMP_MasterTaskloop : Directive<"master taskloop"> { +def OMP_TargetTeamsDistribute : Directive<"target teams distribute"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Master, OMP_TaskLoop]; + let allowedOnceClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + ]; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute]; } -def OMP_MaskedTaskloop : Directive<"masked taskloop"> { +def OMP_TargetTeamsDistributeParallelDo : + Directive<"target teams distribute parallel do"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause - ]; - let leafConstructs = [OMP_masked, OMP_TaskLoop]; -} -def OMP_ParallelMasterTaskloop : - Directive<"parallel master taskloop"> { - let allowedClauses = [ - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + ]; + let allowedOnceClauses = [ VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Parallel, OMP_Master, OMP_TaskLoop]; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_Do]; } -def OMP_ParallelMaskedTaskloop : - Directive<"parallel masked taskloop"> { +def OMP_TargetTeamsDistributeParallelDoSimd : + Directive<"target teams distribute parallel do simd"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + ]; + let allowedOnceClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Parallel, OMP_masked, OMP_TaskLoop]; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_Do, OMP_Simd]; } -def OMP_MasterTaskloopSimd : Directive<"master taskloop simd"> { +def OMP_TargetTeamsDistributeParallelFor : + Directive<"target teams distribute parallel for"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Master, OMP_TaskLoop, OMP_Simd]; + let allowedOnceClauses = [ + VersionedClause, + ]; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_For]; } -def OMP_MaskedTaskloopSimd : Directive<"masked taskloop simd"> { +def OMP_TargetTeamsDistributeParallelForSimd : + Directive<"target teams distribute parallel for simd"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause - ]; - let leafConstructs = [OMP_masked, OMP_TaskLoop, OMP_Simd]; -} -def OMP_ParallelMasterTaskloopSimd : - Directive<"parallel master taskloop simd"> { - let allowedClauses = [ - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Parallel, OMP_Master, OMP_TaskLoop, OMP_Simd]; + let allowedOnceClauses = [ + VersionedClause, + ]; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_For, OMP_Simd]; } -def OMP_ParallelMaskedTaskloopSimd : - Directive<"parallel masked taskloop simd"> { +def OMP_TargetTeamsDistributeSimd : + Directive<"target teams distribute simd"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Parallel, OMP_masked, OMP_TaskLoop, OMP_Simd]; -} -def OMP_Depobj : Directive<"depobj"> { - let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - // TODO This should ne `none` instead. Comment carried over from - // OMPKinds.def. - VersionedClause - ]; - let association = AS_None; -} -def OMP_Scan : Directive<"scan"> { - let allowedClauses = [ - VersionedClause, - VersionedClause + let allowedOnceClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let association = AS_Separating; -} -def OMP_Assumes : Directive<"assumes"> { - let association = AS_None; -} -def OMP_BeginAssumes : Directive<"begin assumes"> { - let association = AS_Delimited; -} -def OMP_EndAssumes : Directive<"end assumes"> { - let association = AS_Delimited; -} -def OMP_BeginDeclareVariant : Directive<"begin declare variant"> { - let association = AS_Delimited; -} -def OMP_EndDeclareVariant : Directive<"end declare variant"> { - let association = AS_Delimited; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Simd]; } -def OMP_scope : Directive<"scope"> { +def OMP_target_teams_loop : Directive<"target teams loop"> { let allowedClauses = [ - VersionedClause, - VersionedClause, - ]; - let allowedOnceClauses = [ - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let association = AS_Block; -} -def OMP_Workshare : Directive<"workshare"> { let allowedOnceClauses = [ - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let association = AS_Block; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_loop]; } -def OMP_ParallelWorkshare : Directive<"parallel workshare"> { +def OMP_TaskLoopSimd : Directive<"taskloop simd"> { let allowedClauses = [ + VersionedClause, VersionedClause, - VersionedClause, VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause - ]; - let allowedOnceClauses = [ - VersionedClause, - VersionedClause, - VersionedClause - ]; - let leafConstructs = [OMP_Parallel, OMP_Workshare]; -} -def OMP_EndDo : Directive<"end do"> { - let allowedOnceClauses = [ - VersionedClause + VersionedClause, + VersionedClause, ]; - // Needed for association computation, since OMP_Do has it "from leafConstructs". - let leafConstructs = OMP_Do.leafConstructs; - let association = OMP_Do.association; -} -def OMP_EndDoSimd : Directive<"end do simd"> { let allowedOnceClauses = [ - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = OMP_DoSimd.leafConstructs; - let association = OMP_DoSimd.association; -} -def OMP_EndSections : Directive<"end sections"> { - let allowedOnceClauses = [ - VersionedClause + let allowedExclusiveClauses = [ + VersionedClause, + VersionedClause, ]; - let leafConstructs = OMP_Sections.leafConstructs; - let association = OMP_Sections.association; + let leafConstructs = [OMP_TaskLoop, OMP_Simd]; } -def OMP_EndSingle : Directive<"end single"> { +def OMP_TeamsDistribute : Directive<"teams distribute"> { let allowedClauses = [ - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause - ]; - let leafConstructs = OMP_Single.leafConstructs; - let association = OMP_Single.association; -} -def OMP_EndWorkshare : Directive<"end workshare"> { - let allowedClauses = [ - VersionedClause - ]; - let leafConstructs = OMP_Workshare.leafConstructs; - let association = OMP_Workshare.association; -} -def OMP_interop : Directive<"interop"> { - let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - ]; - let association = AS_None; -} -def OMP_dispatch : Directive<"dispatch"> { - let allowedClauses = [ - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause + VersionedClause, ]; - let association = AS_Block; + let leafConstructs = [OMP_Teams, OMP_Distribute]; } -def OMP_loop : Directive<"loop"> { +def OMP_TeamsDistributeParallelDo : + Directive<"teams distribute parallel do"> { let allowedClauses = [ + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, VersionedClause, + VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, VersionedClause, - VersionedClause + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let association = AS_Loop; + let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_Do]; } -def OMP_teams_loop : Directive<"teams loop"> { +def OMP_TeamsDistributeParallelDoSimd : + Directive<"teams distribute parallel do simd"> { let allowedClauses = [ + VersionedClause, VersionedClause, VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, VersionedClause, - VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, ]; - let leafConstructs = [OMP_Teams, OMP_loop]; + let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_Do, OMP_Simd]; } -def OMP_target_teams_loop : Directive<"target teams loop"> { +def OMP_TeamsDistributeParallelFor : + Directive<"teams distribute parallel for"> { let allowedClauses = [ - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, ]; - let allowedOnceClauses = [ - VersionedClause, + let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_For]; +} +def OMP_TeamsDistributeParallelForSimd : + Directive<"teams distribute parallel for simd"> { + let allowedClauses = [ + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, ]; - let leafConstructs = [OMP_Target, OMP_Teams, OMP_loop]; + let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_For, OMP_Simd]; } -def OMP_parallel_loop : Directive<"parallel loop"> { +def OMP_TeamsDistributeSimd : Directive<"teams distribute simd"> { let allowedClauses = [ + VersionedClause, VersionedClause, - VersionedClause, VersionedClause, + VersionedClause, VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, VersionedClause, VersionedClause, VersionedClause, - VersionedClause, ]; let allowedOnceClauses = [ - VersionedClause, VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Parallel, OMP_loop]; + let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Simd]; } -def OMP_target_parallel_loop : Directive<"target parallel loop"> { +def OMP_teams_loop : Directive<"teams loop"> { let allowedClauses = [ - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, ]; let allowedOnceClauses = [ VersionedClause, VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, VersionedClause, - VersionedClause, - VersionedClause, - VersionedClause, + VersionedClause, ]; - let leafConstructs = [OMP_Target, OMP_Parallel, OMP_loop]; -} -def OMP_Metadirective : Directive<"metadirective"> { - let allowedClauses = [VersionedClause]; - let allowedOnceClauses = [VersionedClause]; - let association = AS_None; -} - -def OMP_Unknown : Directive<"unknown"> { - let isDefault = true; - let association = AS_None; + let leafConstructs = [OMP_Teams, OMP_loop]; } -- GitLab From 913a8244fe8687df1f27b61c87aa23cf4fcbe84e Mon Sep 17 00:00:00 2001 From: Kareem Ergawy Date: Fri, 7 Jun 2024 14:44:01 +0200 Subject: [PATCH 242/462] [flang][OpenMP] Lower `target .. private(..)` to `omp.private` ops (#94195) Extends delayed privatization support to `taraget .. private(..)`. With this PR, `private` is support for `target` **only** is delayed privatization mode. --- .../lib/Lower/OpenMP/DataSharingProcessor.cpp | 21 +-- flang/lib/Lower/OpenMP/DataSharingProcessor.h | 18 +- flang/lib/Lower/OpenMP/OpenMP.cpp | 118 +++++++++--- .../target-private-allocatable.f90 | 71 +++++++ .../target-private-multiple-variables.f90 | 175 ++++++++++++++++++ .../target-private-simple.f90 | 40 ++++ 6 files changed, 396 insertions(+), 47 deletions(-) create mode 100644 flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90 create mode 100644 flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 create mode 100644 flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90 diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp index b206040c237c..e2b55fcc6406 100644 --- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp +++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp @@ -48,14 +48,13 @@ DataSharingProcessor::DataSharingProcessor( } void DataSharingProcessor::processStep1( - mlir::omp::PrivateClauseOps *clauseOps, - llvm::SmallVectorImpl *privateSyms) { + mlir::omp::PrivateClauseOps *clauseOps) { collectSymbolsForPrivatization(); collectDefaultSymbols(); collectImplicitSymbols(); collectPreDeterminedSymbols(); - privatize(clauseOps, privateSyms); + privatize(clauseOps); insertBarrier(); } @@ -415,16 +414,14 @@ void DataSharingProcessor::collectPreDeterminedSymbols() { preDeterminedSymbols); } -void DataSharingProcessor::privatize( - mlir::omp::PrivateClauseOps *clauseOps, - llvm::SmallVectorImpl *privateSyms) { +void DataSharingProcessor::privatize(mlir::omp::PrivateClauseOps *clauseOps) { for (const semantics::Symbol *sym : allPrivatizedSymbols) { if (const auto *commonDet = sym->detailsIf()) { for (const auto &mem : commonDet->objects()) - doPrivatize(&*mem, clauseOps, privateSyms); + doPrivatize(&*mem, clauseOps); } else - doPrivatize(sym, clauseOps, privateSyms); + doPrivatize(sym, clauseOps); } } @@ -441,9 +438,8 @@ void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) { } } -void DataSharingProcessor::doPrivatize( - const semantics::Symbol *sym, mlir::omp::PrivateClauseOps *clauseOps, - llvm::SmallVectorImpl *privateSyms) { +void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym, + mlir::omp::PrivateClauseOps *clauseOps) { if (!useDelayedPrivatization) { cloneSymbol(sym); copyFirstPrivateSymbol(sym); @@ -548,9 +544,6 @@ void DataSharingProcessor::doPrivatize( clauseOps->privateVars.push_back(hsb.getAddr()); } - if (privateSyms) - privateSyms->push_back(sym); - symToPrivatizer[sym] = privatizerOp; } diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h index fb340e6fdb10..7c0c831ed43a 100644 --- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h +++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h @@ -105,18 +105,15 @@ private: void collectDefaultSymbols(); void collectImplicitSymbols(); void collectPreDeterminedSymbols(); - void privatize(mlir::omp::PrivateClauseOps *clauseOps, - llvm::SmallVectorImpl *privateSyms); + void privatize(mlir::omp::PrivateClauseOps *clauseOps); void defaultPrivatize( mlir::omp::PrivateClauseOps *clauseOps, llvm::SmallVectorImpl *privateSyms); void implicitPrivatize( mlir::omp::PrivateClauseOps *clauseOps, llvm::SmallVectorImpl *privateSyms); - void - doPrivatize(const semantics::Symbol *sym, - mlir::omp::PrivateClauseOps *clauseOps, - llvm::SmallVectorImpl *privateSyms); + void doPrivatize(const semantics::Symbol *sym, + mlir::omp::PrivateClauseOps *clauseOps); void copyLastPrivatize(mlir::Operation *op); void insertLastPrivateCompare(mlir::Operation *op); void cloneSymbol(const semantics::Symbol *sym); @@ -147,15 +144,18 @@ public: // Step2 performs the copying for lastprivates and requires knowledge of the // MLIR operation to insert the last private update. Step2 adds // dealocation code as well. - void processStep1( - mlir::omp::PrivateClauseOps *clauseOps = nullptr, - llvm::SmallVectorImpl *privateSyms = nullptr); + void processStep1(mlir::omp::PrivateClauseOps *clauseOps = nullptr); void processStep2(mlir::Operation *op, bool isLoop); void setLoopIV(mlir::Value iv) { assert(!loopIV && "Loop iteration variable already set"); loopIV = iv; } + + const llvm::SetVector & + getAllSymbolsToPrivatize() const { + return allPrivatizedSymbols; + } }; } // namespace omp diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index f84440d95ec1..184c43ff9fe9 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -456,6 +456,33 @@ markDeclareTarget(mlir::Operation *op, lower::AbstractConverter &converter, declareTargetOp.setDeclareTarget(deviceType, captureClause); } +/// For an operation that takes `omp.private` values as region args, this util +/// merges the private vars info into the region arguments list. +/// +/// \tparam OMPOP - the OpenMP op that takes `omp.private` inputs. +/// \tparam InfoTy - the type of private info we want to merge; e.g. mlir::Type +/// or mlir::Location fields of the private var list. +/// +/// \param [in] op - the op accepting `omp.private` inputs. +/// \param [in] currentList - the current list of region info that we +/// want to merge private info with. For example this could be the list of types +/// or locations of previous arguments to \op's region. +/// \param [in] infoAccessor - for a private variable, this returns the +/// data we want to merge: type or location. +/// \param [out] allRegionArgsInfo - the merged list of region info. +template +static void +mergePrivateVarsInfo(OMPOp op, llvm::ArrayRef currentList, + llvm::function_ref infoAccessor, + llvm::SmallVectorImpl &allRegionArgsInfo) { + mlir::OperandRange privateVars = op.getPrivateVars(); + + llvm::transform(currentList, std::back_inserter(allRegionArgsInfo), + [](InfoTy i) { return i; }); + llvm::transform(privateVars, std::back_inserter(allRegionArgsInfo), + infoAccessor); +} + //===----------------------------------------------------------------------===// // Op body generation helper structures and functions //===----------------------------------------------------------------------===// @@ -758,6 +785,7 @@ genBodyOfTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable, llvm::ArrayRef mapSyms, llvm::ArrayRef mapSymLocs, llvm::ArrayRef mapSymTypes, + DataSharingProcessor &dsp, const mlir::Location ¤tLocation, const ConstructQueue &queue, ConstructQueue::iterator item) { assert(mapSymTypes.size() == mapSymLocs.size()); @@ -765,8 +793,20 @@ genBodyOfTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable, fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); mlir::Region ®ion = targetOp.getRegion(); - auto *regionBlock = - firOpBuilder.createBlock(®ion, {}, mapSymTypes, mapSymLocs); + llvm::SmallVector allRegionArgTypes; + mergePrivateVarsInfo(targetOp, mapSymTypes, + llvm::function_ref{ + [](mlir::Value v) { return v.getType(); }}, + allRegionArgTypes); + + llvm::SmallVector allRegionArgLocs; + mergePrivateVarsInfo(targetOp, mapSymLocs, + llvm::function_ref{ + [](mlir::Value v) { return v.getLoc(); }}, + allRegionArgLocs); + + auto *regionBlock = firOpBuilder.createBlock(®ion, {}, allRegionArgTypes, + allRegionArgLocs); // Clones the `bounds` placing them inside the target region and returns them. auto cloneBound = [&](mlir::Value bound) { @@ -830,6 +870,20 @@ genBodyOfTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable, }); } + for (auto [argIndex, argSymbol] : + llvm::enumerate(dsp.getAllSymbolsToPrivatize())) { + argIndex = mapSyms.size() + argIndex; + + const mlir::BlockArgument &arg = region.getArgument(argIndex); + converter.bindSymbol(*argSymbol, + hlfir::translateToExtendedValue( + currentLocation, firOpBuilder, hlfir::Entity{arg}, + /*contiguousHint=*/ + evaluate::IsSimplyContiguous( + *argSymbol, converter.getFoldingContext())) + .first); + } + // Check if cloning the bounds introduced any dependency on the outer region. // If so, then either clone them as well if they are MemoryEffectFree, or else // copy them to a new temporary and add them to the map and block_argument @@ -907,6 +961,8 @@ genBodyOfTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable, } else { genNestedEvaluations(converter, eval); } + + dsp.processStep2(targetOp, /*isLoop=*/false); } template @@ -1048,15 +1104,18 @@ static void genTargetClauses( devicePtrSyms); cp.processMap(loc, stmtCtx, clauseOps, &mapSyms, &mapLocs, &mapTypes); cp.processThreadLimit(stmtCtx, clauseOps); - // TODO Support delayed privatization. if (processHostOnlyClauses) cp.processNowait(clauseOps); cp.processTODO(loc, llvm::omp::Directive::OMPD_target); + + // `target private(..)` is only supported in delayed privatization mode. + if (!enableDelayedPrivatization) + cp.processTODO(loc, llvm::omp::Directive::OMPD_target); } static void genTargetDataClauses( @@ -1289,7 +1348,6 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable, fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); lower::StatementContext stmtCtx; mlir::omp::ParallelClauseOps clauseOps; - llvm::SmallVector privateSyms; llvm::SmallVector reductionTypes; llvm::SmallVector reductionSyms; genParallelClauses(converter, semaCtx, stmtCtx, item->clauses, loc, @@ -1319,7 +1377,7 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable, /*useDelayedPrivatization=*/true, &symTable); if (privatize) - dsp.processStep1(&clauseOps, &privateSyms); + dsp.processStep1(&clauseOps); auto genRegionEntryCB = [&](mlir::Operation *op) { auto parallelOp = llvm::cast(op); @@ -1327,26 +1385,27 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable, llvm::SmallVector reductionLocs( clauseOps.reductionVars.size(), loc); - mlir::OperandRange privateVars = parallelOp.getPrivateVars(); - mlir::Region ®ion = parallelOp.getRegion(); + llvm::SmallVector allRegionArgTypes; + mergePrivateVarsInfo(parallelOp, llvm::ArrayRef(reductionTypes), + llvm::function_ref{ + [](mlir::Value v) { return v.getType(); }}, + allRegionArgTypes); - llvm::SmallVector privateVarTypes = reductionTypes; - privateVarTypes.reserve(privateVarTypes.size() + privateVars.size()); - llvm::transform(privateVars, std::back_inserter(privateVarTypes), - [](mlir::Value v) { return v.getType(); }); + llvm::SmallVector allRegionArgLocs; + mergePrivateVarsInfo(parallelOp, llvm::ArrayRef(reductionLocs), + llvm::function_ref{ + [](mlir::Value v) { return v.getLoc(); }}, + allRegionArgLocs); - llvm::SmallVector privateVarLocs = reductionLocs; - privateVarLocs.reserve(privateVarLocs.size() + privateVars.size()); - llvm::transform(privateVars, std::back_inserter(privateVarLocs), - [](mlir::Value v) { return v.getLoc(); }); - - firOpBuilder.createBlock(®ion, /*insertPt=*/{}, privateVarTypes, - privateVarLocs); + mlir::Region ®ion = parallelOp.getRegion(); + firOpBuilder.createBlock(®ion, /*insertPt=*/{}, allRegionArgTypes, + allRegionArgLocs); llvm::SmallVector allSymbols = reductionSyms; - allSymbols.append(privateSyms); + allSymbols.append(dsp.getAllSymbolsToPrivatize().begin(), + dsp.getAllSymbolsToPrivatize().end()); + for (auto [arg, prv] : llvm::zip_equal(allSymbols, region.getArguments())) { - fir::ExtendedValue hostExV = converter.getSymbolExtendedValue(*arg); converter.bindSymbol(*arg, hlfir::translateToExtendedValue( loc, firOpBuilder, hlfir::Entity{prv}, /*contiguousHint=*/ @@ -1541,11 +1600,22 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable, deviceAddrLocs, deviceAddrTypes, devicePtrSyms, devicePtrLocs, devicePtrTypes); + llvm::SmallVector privateSyms; + DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval, + /*shouldCollectPreDeterminedSymbols=*/ + lower::omp::isLastItemInQueue(item, queue), + /*useDelayedPrivatization=*/true, &symTable); + dsp.processStep1(&clauseOps); + // 5.8.1 Implicit Data-Mapping Attribute Rules // The following code follows the implicit data-mapping rules to map all the - // symbols used inside the region that have not been explicitly mapped using - // the map clause. + // symbols used inside the region that do not have explicit data-environment + // attribute clauses (neither data-sharing; e.g. `private`, nor `map` + // clauses). auto captureImplicitMap = [&](const semantics::Symbol &sym) { + if (dsp.getAllSymbolsToPrivatize().contains(&sym)) + return; + if (llvm::find(mapSyms, &sym) == mapSyms.end()) { mlir::Value baseOp = converter.getSymbolAddress(sym); if (!baseOp) @@ -1632,7 +1702,7 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable, auto targetOp = firOpBuilder.create(loc, clauseOps); genBodyOfTargetOp(converter, symTable, semaCtx, eval, targetOp, mapSyms, - mapLocs, mapTypes, loc, queue, item); + mapLocs, mapTypes, dsp, loc, queue, item); return targetOp; } diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90 new file mode 100644 index 000000000000..17a28c6a5ab7 --- /dev/null +++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90 @@ -0,0 +1,71 @@ +! Tests delayed privatization for `targets ... private(..)` for allocatables. + +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization \ +! RUN: -o - %s 2>&1 | FileCheck %s +! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 \ +! RUN: | FileCheck %s + +subroutine target_allocatable + implicit none + integer, allocatable :: alloc_var + + !$omp target private(alloc_var) + alloc_var = 10 + !$omp end target +end subroutine target_allocatable + +! CHECK-LABEL: omp.private {type = private} +! CHECK-SAME: @[[VAR_PRIVATIZER_SYM:.*]] : +! CHECK-SAME: [[TYPE:!fir.ref>>]] alloc { +! CHECK: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]): +! CHECK: %[[PRIV_ALLOC:.*]] = fir.alloca !fir.box> {bindc_name = "alloc_var", {{.*}}} + +! CHECK-NEXT: %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]] : !fir.ref>> +! CHECK-NEXT: %[[PRIV_ARG_BOX:.*]] = fir.box_addr %[[PRIV_ARG_VAL]] : (!fir.box>) -> !fir.heap +! CHECK-NEXT: %[[PRIV_ARG_ADDR:.*]] = fir.convert %[[PRIV_ARG_BOX]] : (!fir.heap) -> i64 +! CHECK-NEXT: %[[C0:.*]] = arith.constant 0 : i64 +! CHECK-NEXT: %[[ALLOC_COND:.*]] = arith.cmpi ne, %[[PRIV_ARG_ADDR]], %[[C0]] : i64 + +! CHECK-NEXT: fir.if %[[ALLOC_COND]] { +! CHECK: %[[PRIV_ALLOCMEM:.*]] = fir.allocmem i32 {fir.must_be_heap = true, {{.*}}} +! CHECK-NEXT: %[[PRIV_ALLOCMEM_BOX:.*]] = fir.embox %[[PRIV_ALLOCMEM]] : (!fir.heap) -> !fir.box> +! CHECK-NEXT: fir.store %[[PRIV_ALLOCMEM_BOX]] to %[[PRIV_ALLOC]] : !fir.ref>> +! CHECK-NEXT: } else { +! CHECK-NEXT: %[[ZERO_BITS:.*]] = fir.zero_bits !fir.heap +! CHECK-NEXT: %[[ZERO_BOX:.*]] = fir.embox %[[ZERO_BITS]] : (!fir.heap) -> !fir.box> +! CHECK-NEXT: fir.store %[[ZERO_BOX]] to %[[PRIV_ALLOC]] : !fir.ref>> +! CHECK-NEXT: } + +! CHECK-NEXT: %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]] +! CHECK-NEXT: omp.yield(%[[PRIV_DECL]]#0 : [[TYPE]]) + +! CHECK-NEXT: } dealloc { +! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE]]): + +! CHECK-NEXT: %[[PRIV_VAL:.*]] = fir.load %[[PRIV_ARG]] +! CHECK-NEXT: %[[PRIV_ADDR:.*]] = fir.box_addr %[[PRIV_VAL]] +! CHECK-NEXT: %[[PRIV_ADDR_I64:.*]] = fir.convert %[[PRIV_ADDR]] +! CHECK-NEXT: %[[C0:.*]] = arith.constant 0 : i64 +! CHECK-NEXT: %[[PRIV_NULL_COND:.*]] = arith.cmpi ne, %[[PRIV_ADDR_I64]], %[[C0]] : i64 + +! CHECK-NEXT: fir.if %[[PRIV_NULL_COND]] { +! CHECK: %[[PRIV_VAL_2:.*]] = fir.load %[[PRIV_ARG]] +! CHECK-NEXT: %[[PRIV_ADDR_2:.*]] = fir.box_addr %[[PRIV_VAL_2]] +! CHECK-NEXT: fir.freemem %[[PRIV_ADDR_2]] +! CHECK-NEXT: %[[ZEROS:.*]] = fir.zero_bits +! CHECK-NEXT: %[[ZEROS_BOX:.*]] = fir.embox %[[ZEROS]] +! CHECK-NEXT: fir.store %[[ZEROS_BOX]] to %[[PRIV_ARG]] +! CHECK-NEXT: } + +! CHECK-NEXT: omp.yield +! CHECK-NEXT: } + + +! CHECK-LABEL: func.func @_QPtarget_allocatable() { + +! CHECK: %[[VAR_ALLOC:.*]] = fir.alloca !fir.box> +! CHECK-SAME: {bindc_name = "alloc_var", {{.*}}} +! CHECK: %[[VAR_DECL:.*]]:2 = hlfir.declare %[[VAR_ALLOC]] + +! CHECK: omp.target private( +! CHECK-SAME: @[[VAR_PRIVATIZER_SYM]] %[[VAR_DECL]]#0 -> %{{.*}} : [[TYPE]]) { diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 new file mode 100644 index 000000000000..8682a420e89d --- /dev/null +++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 @@ -0,0 +1,175 @@ +! Tests delayed privatization for `targets ... private(..)` for allocatables. + +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization \ +! RUN: -o - %s 2>&1 | FileCheck %s +! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 \ +! RUN: | FileCheck %s + +subroutine target_allocatable(lb, ub, l) + implicit none + integer mapped_var + integer, allocatable :: alloc_var + real :: real_var + + integer(8) :: lb, ub + real, dimension(lb:ub) :: real_arr + + complex :: comp_var + + integer(8):: l + character(len = l) :: char_var + + !$omp target private(alloc_var, real_var) private(lb, real_arr) & + !$omp& private(comp_var) private(char_var) + mapped_var = 5 + + alloc_var = 10 + real_var = 3.14 + + real_arr(lb + 1) = 6.28 + + comp_var = comp_var * comp_var + + char_var = "hello" + !$omp end target +end subroutine target_allocatable + +! Test the privatizer for `character` +! +! CHECK: omp.private {type = private} +! CHECK-SAME: @[[CHAR_PRIVATIZER_SYM:[^[:space:]]+char_var[^[:space:]]+]] +! CHECK-SAME: : [[CHAR_TYPE:!fir.boxchar<1>]] alloc { +! +! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[CHAR_TYPE]]): +! CHECK-NEXT: %[[UNBOX:.*]]:2 = fir.unboxchar %[[PRIV_ARG]] +! CHECK: %[[PRIV_ALLOC:.*]] = fir.alloca !fir.char<1,?>(%[[UNBOX]]#1 : index) +! CHECK-NEXT: %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]] typeparams %[[UNBOX]]#1 +! CHECK-NEXT: omp.yield(%[[PRIV_DECL]]#0 : [[CHAR_TYPE]]) +! CHECK-NEXT: } + +! Test the privatizer for `complex` +! +! CHECK: omp.private {type = private} +! CHECK-SAME: @[[COMP_PRIVATIZER_SYM:[^[:space:]]+comp_var[^[:space:]]+]] +! CHECK-SAME: : [[COMP_TYPE:!fir.ref>]] alloc { +! +! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[COMP_TYPE]]): +! CHECK-NEXT: %[[PRIV_ALLOC:.*]] = fir.alloca !fir.complex<4> +! CHECK-NEXT: %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]] +! CHECK-NEXT: omp.yield(%[[PRIV_DECL]]#0 : [[COMP_TYPE]]) +! CHECK-NEXT: } + +! Test the privatizer for `real(:)` +! +! CHECK: omp.private {type = private} +! CHECK-SAME: @[[ARR_PRIVATIZER_SYM:[^[:space:]]+real_arr[^[:space:]]+]] +! CHECK-SAME: : [[ARR_TYPE:!fir.box>]] alloc { +! +! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[ARR_TYPE]]): +! CHECK: %[[C0:.*]] = arith.constant 0 : index +! CHECK-NEXT: %[[DIMS:.*]]:3 = fir.box_dims %[[PRIV_ARG]], %[[C0]] : ([[ARR_TYPE]], index) +! CHECK: %[[PRIV_ALLOCA:.*]] = fir.alloca !fir.array<{{\?}}xf32> +! CHECK-NEXT: %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 +! CHECK-NEXT: %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOCA]](%[[SHAPE_SHIFT]]) +! CHECK-NEXT: omp.yield(%[[PRIV_DECL]]#0 : [[ARR_TYPE]]) +! CHECK-NEXT: } + +! Test the privatizer for `real(:)`'s lower bound +! +! CHECK: omp.private {type = private} +! CHECK-SAME: @[[LB_PRIVATIZER_SYM:[^[:space:]]+lb[^[:space:]]+]] +! CHECK-SAME: : [[LB_TYPE:!fir.ref]] alloc { + +! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[LB_TYPE]]): +! CHECK-NEXT: %[[PRIV_ALLOCA:.*]] = fir.alloca i64 +! CHECK-NEXT: %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOCA]] +! CHECK-NEXT: omp.yield(%[[PRIV_DECL]]#0 : [[LB_TYPE]]) +! CHECK-NEXT: } + +! Test the privatizer for `real` +! +! CHECK: omp.private {type = private} +! CHECK-SAME: @[[REAL_PRIVATIZER_SYM:[^[:space:]]+real_var[^[:space:]]+]] +! CHECK-SAME: : [[REAL_TYPE:!fir.ref]] alloc { + +! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[REAL_TYPE]]): +! CHECK-NEXT: %[[PRIV_ALLOCA:.*]] = fir.alloca f32 +! CHECK-NEXT: %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOCA]] +! CHECK-NEXT: omp.yield(%[[PRIV_DECL]]#0 : [[REAL_TYPE]]) +! CHECK-NEXT: } + +! Test the privatizer for `allocatable` +! +! CHECK: omp.private {type = private} +! CHECK-SAME: @[[ALLOC_PRIVATIZER_SYM:[^[:space:]]+alloc_var[^[:space:]]+]] +! CHECK-SAME: : [[ALLOC_TYPE:!fir.ref>>]] alloc { +! +! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[ALLOC_TYPE]]): +! CHECK: %[[PRIV_ALLOC:.*]] = fir.alloca !fir.box> +! CHECK-NEXT: %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]] : !fir.ref>> +! CHECK-NEXT: %[[PRIV_ARG_BOX:.*]] = fir.box_addr %[[PRIV_ARG_VAL]] : (!fir.box>) -> !fir.heap +! CHECK-NEXT: %[[PRIV_ARG_ADDR:.*]] = fir.convert %[[PRIV_ARG_BOX]] : (!fir.heap) -> i64 +! CHECK-NEXT: %[[C0:.*]] = arith.constant 0 : i64 +! CHECK-NEXT: %[[ALLOC_COND:.*]] = arith.cmpi ne, %[[PRIV_ARG_ADDR]], %[[C0]] : i64 +! +! CHECK-NEXT: fir.if %[[ALLOC_COND]] { +! CHECK: %[[PRIV_ALLOCMEM:.*]] = fir.allocmem i32 {fir.must_be_heap = true, {{.*}}} +! CHECK-NEXT: %[[PRIV_ALLOCMEM_BOX:.*]] = fir.embox %[[PRIV_ALLOCMEM]] : (!fir.heap) -> !fir.box> +! CHECK-NEXT: fir.store %[[PRIV_ALLOCMEM_BOX]] to %[[PRIV_ALLOC]] : !fir.ref>> +! CHECK-NEXT: } else { +! CHECK-NEXT: %[[ZERO_BITS:.*]] = fir.zero_bits !fir.heap +! CHECK-NEXT: %[[ZERO_BOX:.*]] = fir.embox %[[ZERO_BITS]] : (!fir.heap) -> !fir.box> +! CHECK-NEXT: fir.store %[[ZERO_BOX]] to %[[PRIV_ALLOC]] : !fir.ref>> +! CHECK-NEXT: } +! +! CHECK-NEXT: %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]] +! CHECK-NEXT: omp.yield(%[[PRIV_DECL]]#0 : [[ALLOC_TYPE]]) +! +! CHECK-NEXT: } dealloc { +! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[ALLOC_TYPE]]): +! +! CHECK-NEXT: %[[PRIV_VAL:.*]] = fir.load %[[PRIV_ARG]] +! CHECK-NEXT: %[[PRIV_ADDR:.*]] = fir.box_addr %[[PRIV_VAL]] +! CHECK-NEXT: %[[PRIV_ADDR_I64:.*]] = fir.convert %[[PRIV_ADDR]] +! CHECK-NEXT: %[[C0:.*]] = arith.constant 0 : i64 +! CHECK-NEXT: %[[PRIV_NULL_COND:.*]] = arith.cmpi ne, %[[PRIV_ADDR_I64]], %[[C0]] : i64 +! +! CHECK-NEXT: fir.if %[[PRIV_NULL_COND]] { +! CHECK: %[[PRIV_VAL_2:.*]] = fir.load %[[PRIV_ARG]] +! CHECK-NEXT: %[[PRIV_ADDR_2:.*]] = fir.box_addr %[[PRIV_VAL_2]] +! CHECK-NEXT: fir.freemem %[[PRIV_ADDR_2]] +! CHECK-NEXT: %[[ZEROS:.*]] = fir.zero_bits +! CHECK-NEXT: %[[ZEROS_BOX:.*]] = fir.embox %[[ZEROS]] +! CHECK-NEXT: fir.store %[[ZEROS_BOX]] to %[[PRIV_ARG]] +! CHECK-NEXT: } +! +! CHECK-NEXT: omp.yield +! CHECK-NEXT: } + +! CHECK: func.func @_QPtarget_allocatable +! CHECK: %[[MAPPED_ALLOC:.*]] = fir.alloca i32 {bindc_name = "mapped_var", {{.*}}} +! CHECK-NEXT: %[[MAPPED_DECL:.*]]:2 = hlfir.declare %[[MAPPED_ALLOC]] +! CHECK: %[[MAPPED_MI:.*]] = omp.map.info var_ptr(%[[MAPPED_DECL]]#1 : !fir.ref, i32) + +! CHECK: omp.target +! CHECK-SAME: map_entries(%[[MAPPED_MI]] -> %[[MAPPED_ARG:.*]] : !fir.ref) +! CHECK-SAME: private( +! CHECK-SAME: @[[ALLOC_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ALLOC_ARG:.*]] : !fir.ref>>, +! CHECK-SAME: @[[REAL_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[REAL_ARG:.*]] : !fir.ref, +! CHECK-SAME: @[[LB_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[LB_ARG:.*]] : !fir.ref, +! CHECK-SAME: @[[ARR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ARR_ARG:.*]] : !fir.box>, +! CHECK-SAME: @[[COMP_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[COMP_ARG:.*]] : !fir.ref>, +! CHECK-SAME: @[[CHAR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[CHAR_ARG:.*]] : !fir.boxchar<1>) { +! CHECK-NOT: fir.alloca +! CHECK: hlfir.declare %[[MAPPED_ARG]] +! CHECK: hlfir.declare %[[ALLOC_ARG]] +! CHECK: hlfir.declare %[[REAL_ARG]] +! CHECK: hlfir.declare %[[LB_ARG]] +! CHECK: %[[ARR_ARG_ADDR:.*]] = fir.box_addr %[[ARR_ARG]] +! CHECK: hlfir.declare %[[ARR_ARG_ADDR]] +! CHECK: hlfir.declare %[[COMP_ARG]] +! CHECK: %[[CHAR_ARG_UNBOX:.*]]:2 = fir.unboxchar %[[CHAR_ARG]] +! CHECK: hlfir.declare %[[CHAR_ARG_UNBOX]] +! CHECK: omp.terminator +! CHECK-NEXT: } + diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90 new file mode 100644 index 000000000000..94edeaa5a7ef --- /dev/null +++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90 @@ -0,0 +1,40 @@ +! Tests delayed privatization for `targets ... private(..)` for simple variables. + +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization \ +! RUN: -o - %s 2>&1 | FileCheck %s +! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 \ +! RUN: | FileCheck %s + +subroutine target_simple + implicit none + integer :: simple_var + + !$omp target private(simple_var) + simple_var = 10 + !$omp end target +end subroutine target_simple + +! CHECK-LABEL: omp.private {type = private} +! CHECK-SAME: @[[VAR_PRIVATIZER_SYM:.*]] : !fir.ref alloc { +! CHECK: ^bb0(%[[PRIV_ARG:.*]]: !fir.ref): +! CHECK: %[[PRIV_ALLOC:.*]] = fir.alloca i32 {bindc_name = "simple_var", {{.*}}} +! CHECK: %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]] +! CHECK: omp.yield(%[[PRIV_DECL]]#0 : !fir.ref) +! CHECK: } + +! CHECK-LABEL: func.func @_QPtarget_simple() { +! CHECK: %[[VAR_ALLOC:.*]] = fir.alloca i32 {bindc_name = "simple_var", {{.*}}} +! CHECK: %[[VAR_DECL:.*]]:2 = hlfir.declare %[[VAR_ALLOC]] + +! CHECK: omp.target private( +! CHECK-SAME: @[[VAR_PRIVATIZER_SYM]] %[[VAR_DECL]]#0 -> %{{.*}} : !fir.ref) { +! CHECK: ^bb0(%[[REG_ARG:.*]]: !fir.ref): +! CHECK: %[[REG_DECL:.*]]:2 = hlfir.declare %[[REG_ARG]] +! CHECK: %[[C10:.*]] = arith.constant 10 +! CHECK: hlfir.assign %[[C10]] to %[[REG_DECL]]#0 +! CHECK: omp.terminator +! CHECK: } + +! CHECK: return +! CHECK: } + -- GitLab From 2c3723d321ae000dfc9dfab064076799a90c322e Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 7 Jun 2024 07:07:00 -0500 Subject: [PATCH 243/462] [libc] Correctly pass the C++ standard to NVPTX internal builds Summary: The NVPTX build wasn't getting the `C++20` standard necessary for a few files. --- libc/cmake/modules/LLVMLibCObjectRules.cmake | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/libc/cmake/modules/LLVMLibCObjectRules.cmake b/libc/cmake/modules/LLVMLibCObjectRules.cmake index 134c5143d6d6..1f80e7f4e57c 100644 --- a/libc/cmake/modules/LLVMLibCObjectRules.cmake +++ b/libc/cmake/modules/LLVMLibCObjectRules.cmake @@ -44,6 +44,10 @@ function(create_object_library fq_target_name) message(FATAL_ERROR "'add_object_library' rule requires SRCS to be specified.") endif() + if(NOT ADD_OBJECT_CXX_STANDARD) + set(ADD_OBJECT_CXX_STANDARD ${CMAKE_CXX_STANDARD}) + endif() + set(internal_target_name ${fq_target_name}.__internal__) set(public_packaging_for_internal "-DLIBC_COPT_PUBLIC_PACKAGING") @@ -75,6 +79,9 @@ function(create_object_library fq_target_name) target_include_directories(${internal_target_name} PRIVATE ${LIBC_SOURCE_DIR}) target_compile_options(${internal_target_name} PRIVATE ${compile_options} -fno-lto -march=${LIBC_GPU_TARGET_ARCHITECTURE}) + set_target_properties(${internal_target_name} + PROPERTIES + CXX_STANDARD ${ADD_OBJECT_CXX_STANDARD}) endif() if(SHOW_INTERMEDIATE_OBJECTS) @@ -92,9 +99,6 @@ function(create_object_library fq_target_name) target_link_libraries(${fq_target_name} PUBLIC ${fq_deps_list}) endif() - if(NOT ADD_OBJECT_CXX_STANDARD) - set(ADD_OBJECT_CXX_STANDARD ${CMAKE_CXX_STANDARD}) - endif() set_target_properties( ${fq_target_name} PROPERTIES -- GitLab From 5b2f7a19711710df1469ab8a0096fe2f8052c8b1 Mon Sep 17 00:00:00 2001 From: Ryan Holt Date: Fri, 7 Jun 2024 09:00:28 -0400 Subject: [PATCH 244/462] [mlir][linalg] Support lowering unpack with outer_dims_perm (#94477) This commit adds support for lowering `tensor.unpack` with a non-identity `outer_dims_perm`. This was previously left as a not-yet-implemented case. --- .../Dialect/Linalg/Transforms/Transforms.cpp | 48 ++++++++----------- .../Dialect/Linalg/transform-lower-pack.mlir | 18 +++++-- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp index 91dfac802ad6..f30ef235e9cd 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp @@ -356,13 +356,6 @@ FailureOr linalg::lowerPack(RewriterBase &rewriter, FailureOr linalg::lowerUnPack(RewriterBase &rewriter, tensor::UnPackOp unPackOp) { - // 1. Filter out NYI cases. - if (!unPackOp.getOuterDimsPerm().empty() && - !isIdentityPermutation(unPackOp.getOuterDimsPerm())) { - return rewriter.notifyMatchFailure(unPackOp, - "non-identity outer dims perm NYI"); - } - Location loc = unPackOp->getLoc(); OpBuilder::InsertionGuard g(rewriter); rewriter.setInsertionPoint(unPackOp); @@ -391,36 +384,33 @@ FailureOr linalg::lowerUnPack(RewriterBase &rewriter, return LowerUnPackOpResult{/*emptyOp=*/nullptr, /*transposeOp=*/nullptr, /*reshapeOp=*/nullptr, extractSliceOp}; } - // 2. Compute the permutation vector to move the last `numPackedDims` into - // the `innerPosDims` of a shape of rank `packedRank`. - int64_t numPackedDims = unPackOp.getInnerDimsPos().size(); - auto lastDims = llvm::to_vector( - llvm::seq(packedRank - numPackedDims, packedRank)); - PackingMetadata packingMetadata = - computePackingMetadata(packedRank, unPackOp.getInnerDimsPos()); - SmallVector lastDimsToInsertPositionsPerm = computePermutationVector( - packedRank, lastDims, packingMetadata.insertPositions); - - // 3. Compute the stripMinedShape: this is the packed shape without outer and + + // 1. Compute the permutation vector to shuffle packed shape into the shape + // before any outer or inner permutations have been applied. + PackingMetadata packingMetadata; + SmallVector packedToStripMinedShapePerm = + tensor::getUnPackInverseSrcPerm(unPackOp, packingMetadata); + + // 2. Compute the stripMinedShape: this is the packed shape without outer and // inner permutations. SmallVector stripMinedShape(packedTensorType.getShape()); - applyPermutationToVector(stripMinedShape, lastDimsToInsertPositionsPerm); + applyPermutationToVector(stripMinedShape, packedToStripMinedShapePerm); - // 4. Transpose packedShape to stripMinedShape. + // 3. Transpose packedShape to stripMinedShape. RankedTensorType stripMinedTensorType = RankedTensorType::Builder(packedTensorType).setShape(stripMinedShape); RankedTensorType collapsedType = tensor::CollapseShapeOp::inferCollapsedType( stripMinedTensorType, packingMetadata.reassociations); - // Get dynamic dims from input tensor based on lastDimsToInsertPositionsPerm + // Get dynamic dims from input tensor based on packedToStripMinedShapePerm // permutation. SmallVector dims = tensor::getMixedSizes(rewriter, loc, unPackOp.getSource()); - applyPermutationToVector(dims, lastDimsToInsertPositionsPerm); + applyPermutationToVector(dims, packedToStripMinedShapePerm); auto emptyOp = rewriter.create( loc, dims, stripMinedTensorType.getElementType()); auto transposeOp = rewriter.create( - loc, unPackOp.getSource(), emptyOp, lastDimsToInsertPositionsPerm); + loc, unPackOp.getSource(), emptyOp, packedToStripMinedShapePerm); LLVM_DEBUG( DBGSNL(); DBGSNL(); llvm::interleaveComma(packingMetadata.insertPositions, @@ -428,8 +418,8 @@ FailureOr linalg::lowerUnPack(RewriterBase &rewriter, DBGSNL(); llvm::interleaveComma(packedTensorType.getShape(), DBGS() << "packedShape: "); DBGSNL(); - llvm::interleaveComma(lastDimsToInsertPositionsPerm, - DBGS() << "lastDimsToInsertPositionsPerm: "); + llvm::interleaveComma(packedToStripMinedShapePerm, + DBGS() << "packedToStripMinedShapePerm: "); DBGSNL(); llvm::interleaveComma( packingMetadata.reassociations, DBGS() << "reassociations: ", [&](ReassociationIndices ri) { @@ -439,12 +429,12 @@ FailureOr linalg::lowerUnPack(RewriterBase &rewriter, llvm::interleaveComma(stripMinedShape, DBGS() << "stripMinedShape: "); DBGSNL(); DBGS() << "collapsed type: " << collapsedType; DBGSNL();); - // 5. Collapse from the stripMinedShape to the padded result. + // 4. Collapse from the stripMinedShape to the padded result. auto reshapeOp = rewriter.create( loc, collapsedType, transposeOp->getResult(0), packingMetadata.reassociations); - // 6. ExtractSlice. + // 5. ExtractSlice. int64_t destRank = destTensorType.getRank(); auto extractSliceOp = rewriter.create( loc, destTensorType, reshapeOp->getResult(0), @@ -452,11 +442,11 @@ FailureOr linalg::lowerUnPack(RewriterBase &rewriter, tensor::getMixedSizes(rewriter, loc, unPackOp.getDest()), SmallVector(destRank, one)); - // 7. Inject a copy to preserve DPS. + // 6. Inject a copy to preserve DPS. auto copyOp = rewriter.create( loc, extractSliceOp->getResult(0), unPackOp.getDest()); - // 8. Replace unPackOp by extractSliceOp. + // 7. Replace unPackOp by copyOp. rewriter.replaceOp(unPackOp, copyOp->getResults()); return LowerUnPackOpResult{emptyOp, transposeOp, reshapeOp, extractSliceOp}; diff --git a/mlir/test/Dialect/Linalg/transform-lower-pack.mlir b/mlir/test/Dialect/Linalg/transform-lower-pack.mlir index 926969bfc738..f34ef4f96148 100644 --- a/mlir/test/Dialect/Linalg/transform-lower-pack.mlir +++ b/mlir/test/Dialect/Linalg/transform-lower-pack.mlir @@ -622,9 +622,20 @@ module attributes {transform.with_named_sequence} { // ----- -// At the moment, we cannot lower tensor.unpack with outer_dims_perm. -func.func @diagnostic_unpack(%arg0: tensor<32x64xf32>, %arg1: tensor<2x4x32x8xf32>) -> tensor<32x64xf32> { - // expected-note @below {{target payload op}} +// CHECK-LABEL: @unpack_with_outer_dims_perm +// CHECK-SAME: %[[ARG0:.*]]: tensor<32x64xf32>, %[[ARG1:.*]]: tensor<2x4x32x8xf32> +// CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<4x8x2x32xf32> +// CHECK: %[[TRAN:.*]] = linalg.transpose +// CHECK-SAME: ins(%[[ARG1]] : tensor<2x4x32x8xf32>) +// CHECK-SAME: outs(%[[EMPTY]] : tensor<4x8x2x32xf32>) +// CHECK-SAME: permutation = [1, 3, 0, 2] +// CHECK: %[[CLP:.*]] = tensor.collapse_shape %[[TRAN]] {{\[}}[0, 1], [2, 3]] +// CHECK-SAME: : tensor<4x8x2x32xf32> into tensor<32x64xf32> +// CHECK: %[[SLICE:.*]] = tensor.extract_slice %[[CLP]][0, 0] [32, 64] [1, 1] +// CHECK-SAME: : tensor<32x64xf32> to tensor<32x64xf32> +// CHECK: linalg.copy ins(%[[SLICE]] +// CHECK-SAME: : tensor<32x64xf32>) outs(%[[ARG0]] : tensor<32x64xf32>) -> tensor<32x64xf32> +func.func @unpack_with_outer_dims_perm(%arg0: tensor<32x64xf32>, %arg1: tensor<2x4x32x8xf32>) -> tensor<32x64xf32> { %unpack = tensor.unpack %arg1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [32, 8] into %arg0 : tensor<2x4x32x8xf32> -> tensor<32x64xf32> return %unpack : tensor<32x64xf32> @@ -634,7 +645,6 @@ module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) { %unpack = transform.structured.match ops{["tensor.unpack"]} in %module_op : (!transform.any_op) -> !transform.op<"tensor.unpack"> - // expected-error @below {{cannot lower to transpose + collapse + extract}} transform.structured.lower_unpack %unpack : (!transform.op<"tensor.unpack">) -> (!transform.op<"tensor.empty">, !transform.op<"linalg.transpose">, -- GitLab From c886d66da03bdf26e6fca68a1b730ae6eb923194 Mon Sep 17 00:00:00 2001 From: Max191 <44243577+Max191@users.noreply.github.com> Date: Fri, 7 Jun 2024 06:09:06 -0700 Subject: [PATCH 245/462] [mlir] Add reshape propagation patterns for tensor.pad (#94489) This PR adds fusion by collapsing and fusion by expansion patterns for `tensor.pad` ops in ElementwiseOpFusion. Pad ops can be expanded or collapsed as long as none of the padded dimensions will be expanded or collapsed. --- .../Linalg/Transforms/ElementwiseOpFusion.cpp | 146 ++++++++++++++++++ .../fuse-with-reshape-by-collapsing.mlir | 68 ++++++++ mlir/test/Dialect/Linalg/reshape_fusion.mlir | 61 ++++++++ 3 files changed, 275 insertions(+) diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp index ad313c2d5ce6..e73df61c9643 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp @@ -956,6 +956,69 @@ private: ControlFusionFn controlFoldingReshapes; }; +class FoldPadWithProducerReshapeOpByExpansion + : public OpRewritePattern { +public: + FoldPadWithProducerReshapeOpByExpansion(MLIRContext *context, + ControlFusionFn foldReshapes, + PatternBenefit benefit = 1) + : OpRewritePattern(context, benefit), + controlFoldingReshapes(std::move(foldReshapes)) {} + + LogicalResult matchAndRewrite(tensor::PadOp padOp, + PatternRewriter &rewriter) const override { + tensor::CollapseShapeOp reshapeOp = + padOp.getSource().getDefiningOp(); + if (!reshapeOp) + return failure(); + if (!reshapeOp->hasOneUse()) + return failure(); + + if (!controlFoldingReshapes(&padOp.getSourceMutable())) { + return rewriter.notifyMatchFailure(padOp, + "fusion blocked by control function"); + } + + ArrayRef low = padOp.getStaticLow(); + ArrayRef high = padOp.getStaticHigh(); + SmallVector reassociations = + reshapeOp.getReassociationIndices(); + + for (auto [reInd, l, h] : llvm::zip_equal(reassociations, low, high)) { + if (reInd.size() != 1 && (l != 0 || h != 0)) + return failure(); + } + + SmallVector newLow, newHigh; + RankedTensorType expandedType = reshapeOp.getSrcType(); + RankedTensorType paddedType = padOp.getResultType(); + SmallVector expandedPaddedShape(expandedType.getShape()); + for (auto [idx, reInd] : llvm::enumerate(reassociations)) { + if (reInd.size() == 1) { + expandedPaddedShape[reInd[0]] = paddedType.getShape()[idx]; + } + for (size_t i = 0; i < reInd.size(); ++i) { + newLow.push_back(padOp.getMixedLowPad()[idx]); + newHigh.push_back(padOp.getMixedHighPad()[idx]); + } + } + + Location loc = padOp->getLoc(); + RankedTensorType expandedPaddedType = paddedType.clone(expandedPaddedShape); + auto newPadOp = rewriter.create( + loc, expandedPaddedType, reshapeOp.getSrc(), newLow, newHigh, + padOp.getConstantPaddingValue(), padOp.getNofold()); + + rewriter.replaceOpWithNewOp( + padOp, padOp.getResultType(), newPadOp.getResult(), reassociations); + + return success(); + } + +private: + ControlFusionFn controlFoldingReshapes; +}; + /// Pattern to fold a tensor.expand_shape op with its producer generic op /// by expanding the dimensionality of the loop in the producer op. struct FoldReshapeWithGenericOpByExpansion @@ -1702,6 +1765,85 @@ private: ControlFusionFn controlFoldingReshapes; }; +class FoldPadWithProducerReshapeOpByCollapsing + : public OpRewritePattern { +public: + FoldPadWithProducerReshapeOpByCollapsing(MLIRContext *context, + ControlFusionFn foldReshapes, + PatternBenefit benefit = 1) + : OpRewritePattern(context, benefit), + controlFoldingReshapes(std::move(foldReshapes)) {} + + LogicalResult matchAndRewrite(tensor::PadOp padOp, + PatternRewriter &rewriter) const override { + tensor::ExpandShapeOp reshapeOp = + padOp.getSource().getDefiningOp(); + if (!reshapeOp) + return failure(); + if (!reshapeOp->hasOneUse()) + return failure(); + + if (!controlFoldingReshapes(&padOp.getSourceMutable())) { + return rewriter.notifyMatchFailure(padOp, + "fusion blocked by control function"); + } + + ArrayRef low = padOp.getStaticLow(); + ArrayRef high = padOp.getStaticHigh(); + SmallVector reassociations = + reshapeOp.getReassociationIndices(); + + for (auto reInd : reassociations) { + if (reInd.size() == 1) + continue; + if (llvm::any_of(reInd, [&](int64_t ind) { + return low[ind] != 0 || high[ind] != 0; + })) { + return failure(); + } + } + + SmallVector newLow, newHigh; + RankedTensorType collapsedType = reshapeOp.getSrcType(); + RankedTensorType paddedType = padOp.getResultType(); + SmallVector collapsedPaddedShape(collapsedType.getShape()); + SmallVector expandedPaddedSizes( + getMixedValues(reshapeOp.getStaticOutputShape(), + reshapeOp.getOutputShape(), rewriter)); + AffineExpr d0, d1, d2; + bindDims(rewriter.getContext(), d0, d1, d2); + auto addMap = AffineMap::get(3, 0, {d0 + d1 + d2}); + Location loc = reshapeOp->getLoc(); + for (auto [idx, reInd] : llvm::enumerate(reassociations)) { + OpFoldResult l = padOp.getMixedLowPad()[reInd[0]]; + OpFoldResult h = padOp.getMixedHighPad()[reInd[0]]; + if (reInd.size() == 1) { + collapsedPaddedShape[idx] = paddedType.getShape()[reInd[0]]; + OpFoldResult paddedSize = affine::makeComposedFoldedAffineApply( + rewriter, loc, addMap, {l, h, expandedPaddedSizes[reInd[0]]}); + expandedPaddedSizes[reInd[0]] = paddedSize; + } + newLow.push_back(l); + newHigh.push_back(h); + } + + RankedTensorType collapsedPaddedType = + paddedType.clone(collapsedPaddedShape); + auto newPadOp = rewriter.create( + loc, collapsedPaddedType, reshapeOp.getSrc(), newLow, newHigh, + padOp.getConstantPaddingValue(), padOp.getNofold()); + + rewriter.replaceOpWithNewOp( + padOp, padOp.getResultType(), newPadOp.getResult(), reassociations, + expandedPaddedSizes); + + return success(); + } + +private: + ControlFusionFn controlFoldingReshapes; +}; + /// Pattern to collapse dimensions. template class CollapseLinalgDimensions : public OpRewritePattern { @@ -1937,6 +2079,8 @@ void mlir::linalg::populateFoldReshapeOpsByExpansionPatterns( const ControlFusionFn &controlFoldingReshapes) { patterns.add(patterns.getContext(), controlFoldingReshapes); + patterns.add(patterns.getContext(), + controlFoldingReshapes); patterns.add(patterns.getContext(), controlFoldingReshapes); } @@ -1946,6 +2090,8 @@ void mlir::linalg::populateFoldReshapeOpsByCollapsingPatterns( const ControlFusionFn &controlFoldingReshapes) { patterns.add(patterns.getContext(), controlFoldingReshapes); + patterns.add( + patterns.getContext(), controlFoldingReshapes); } void mlir::linalg::populateElementwiseOpsFusionPatterns( diff --git a/mlir/test/Dialect/Linalg/fuse-with-reshape-by-collapsing.mlir b/mlir/test/Dialect/Linalg/fuse-with-reshape-by-collapsing.mlir index 0d40df534a3b..600f0dea31f4 100644 --- a/mlir/test/Dialect/Linalg/fuse-with-reshape-by-collapsing.mlir +++ b/mlir/test/Dialect/Linalg/fuse-with-reshape-by-collapsing.mlir @@ -537,3 +537,71 @@ func.func @no_fold_non_consecutive_reduction_dims(%arg0 : tensor, %sz0: // CHECK: %[[GENERIC:.+]] = linalg.generic // CHECK-SAME: ins(%[[EXPAND_ARG0]] : // CHECK: return %[[GENERIC]] + +// ----- + +func.func @fuse_by_collapsing_pad(%arg0 : tensor<2x12x5x336x9xi32>) -> tensor<8x3x4x17x6x7x8x14xi32> { + %expand = tensor.expand_shape %arg0 [[0], [1, 2], [3], [4, 5, 6], [7]] output_shape [2, 3, 4, 5, 6, 7, 8, 9] : tensor<2x12x5x336x9xi32> into tensor<2x3x4x5x6x7x8x9xi32> + %cst = arith.constant 0 : i32 + %padded_0 = tensor.pad %expand low[1, 0, 0, 8, 0, 0, 0, 3] high[5, 0, 0, 4, 0, 0, 0, 2] { + ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index, + %arg5: index, %arg6: index, %arg7: index, %arg8: index): + tensor.yield %cst : i32 + } : tensor<2x3x4x5x6x7x8x9xi32> to tensor<8x3x4x17x6x7x8x14xi32> + return %padded_0 : tensor<8x3x4x17x6x7x8x14xi32> +} +// CHECK: func @fuse_by_collapsing_pad( +// CHECK-SAME: %[[ARG0:.+]]: tensor<2x12x5x336x9xi32>) +// CHECK: %[[PAD:.+]] = tensor.pad %[[ARG0]] +// CHECK-SAME: low[1, 0, 8, 0, 3] high[5, 0, 4, 0, 2] +// CHECK: tensor<2x12x5x336x9xi32> to tensor<8x12x17x336x14xi32> +// CHECK: %[[EXPAND:.+]] = tensor.expand_shape %[[PAD]] {{\[}}[0], [1, 2], [3], [4, 5, 6], [7]] +// CHECK-SAME: output_shape [8, 3, 4, 17, 6, 7, 8, 14] : tensor<8x12x17x336x14xi32> into tensor<8x3x4x17x6x7x8x14xi32> +// CHECK: return %[[EXPAND]] + +// ----- + +func.func @no_fuse_by_collapsing_pad(%arg0 : tensor<2x12x5x336x9xi32>) -> tensor<8x5x4x17x6x7x8x14xi32> { + %expand = tensor.expand_shape %arg0 [[0], [1, 2], [3], [4, 5, 6], [7]] output_shape [2, 3, 4, 5, 6, 7, 8, 9] : tensor<2x12x5x336x9xi32> into tensor<2x3x4x5x6x7x8x9xi32> + %cst = arith.constant 0 : i32 + %padded_0 = tensor.pad %expand low[1, 2, 0, 8, 0, 0, 0, 3] high[5, 0, 0, 4, 0, 0, 0, 2] { + ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index, + %arg5: index, %arg6: index, %arg7: index, %arg8: index): + tensor.yield %cst : i32 + } : tensor<2x3x4x5x6x7x8x9xi32> to tensor<8x5x4x17x6x7x8x14xi32> + return %padded_0 : tensor<8x5x4x17x6x7x8x14xi32> +} +// CHECK: func @no_fuse_by_collapsing_pad( +// CHECK-SAME: %[[ARG0:.+]]: tensor<2x12x5x336x9xi32>) +// CHECK: %[[EXPAND_ARG0:.+]] = tensor.expand_shape %[[ARG0]] {{\[}}[0], [1, 2], [3], [4, 5, 6], [7]] +// CHECK-SAME: output_shape [2, 3, 4, 5, 6, 7, 8, 9] : tensor<2x12x5x336x9xi32> into tensor<2x3x4x5x6x7x8x9xi32> +// CHECK: %[[PAD:.+]] = tensor.pad %[[EXPAND_ARG0]] +// CHECK-SAME: low[1, 2, 0, 8, 0, 0, 0, 3] high[5, 0, 0, 4, 0, 0, 0, 2] +// CHECK: tensor<2x3x4x5x6x7x8x9xi32> to tensor<8x5x4x17x6x7x8x14xi32> +// CHECK: return %[[PAD]] + +// ----- + +func.func @fuse_by_collapsing_dynamic_pad(%arg0 : tensor, + %s0 : index, %s1 : index, %s2 : index, %s3 : index, %s4 : index, %s5 : index, + %l0 : index, %l1 : index, %h0 : index, %h1 : index) -> tensor { + %expand = tensor.expand_shape %arg0 [[0], [1, 2], [3], [4, 5]] output_shape [%s0, %s1, %s2, %s3, %s4, %s5] : tensor into tensor + %cst = arith.constant 0.0 : f32 + %padded_0 = tensor.pad %expand low[%l0, 0, 0, %l1, 0, 0] high[%h0, 0, 0, %h1, 0, 0] { + ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: index): + tensor.yield %cst : f32 + } : tensor to tensor + return %padded_0 : tensor +} +// CHECK-DAG: #[[MAP:.+]] = affine_map<()[s0, s1, s2] -> (s0 + s1 + s2)> +// CHECK: func @fuse_by_collapsing_dynamic_pad( +// CHECK-SAME: %[[ARG0:.+]]: tensor +// CHECK-SAME: %[[S0:.+]]: index, %[[S1:.+]]: index, %[[S2:.+]]: index, %[[S3:.+]]: index, %[[S4:.+]]: index, %[[S5:.+]]: index, %[[L0:.+]]: index, %[[L1:.+]]: index, %[[H0:.+]]: index, %[[H1:.+]]: index +// CHECK: %[[PAD_SIZE0:.+]] = affine.apply #[[MAP]]()[%[[L0]], %[[H0]], %[[S0]]] +// CHECK: %[[PAD_SIZE1:.+]] = affine.apply #[[MAP]]()[%[[L1]], %[[H1]], %[[S3]]] +// CHECK: %[[PAD:.+]] = tensor.pad %[[ARG0]] +// CHECK-SAME: low[%[[L0]], 0, %[[L1]], 0] high[%[[H0]], 0, %[[H1]], 0] +// CHECK: tensor to tensor +// CHECK: %[[EXPAND:.+]] = tensor.expand_shape %[[PAD]] {{\[}}[0], [1, 2], [3], [4, 5]] +// CHECK-SAME: output_shape [%[[PAD_SIZE0]], %[[S1]], %[[S2]], %[[PAD_SIZE1]], %[[S4]], %[[S5]]] : tensor into tensor +// CHECK: return %[[EXPAND]] diff --git a/mlir/test/Dialect/Linalg/reshape_fusion.mlir b/mlir/test/Dialect/Linalg/reshape_fusion.mlir index f42666f81bba..b8df5fc88e19 100644 --- a/mlir/test/Dialect/Linalg/reshape_fusion.mlir +++ b/mlir/test/Dialect/Linalg/reshape_fusion.mlir @@ -826,3 +826,64 @@ func.func @linalg_add_reshape_producer_fusion(%arg0 : tensor, // CHECK-SAME: [0, 1], [2, 3] // CHECK-SAME: tensor into tensor // CHECK: return %[[T4]] + +// ----- + +func.func @fuse_by_expanding_pad(%arg0 : tensor<2x3x4x5x6x7x8x9xi32>) -> tensor<8x12x17x336x14xi32> { + %collapse = tensor.collapse_shape %arg0 [[0], [1, 2], [3], [4, 5, 6], [7]] : tensor<2x3x4x5x6x7x8x9xi32> into tensor<2x12x5x336x9xi32> + %cst = arith.constant 0 : i32 + %padded_0 = tensor.pad %collapse low[1, 0, 8, 0, 3] high[5, 0, 4, 0, 2] { + ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index): + tensor.yield %cst : i32 + } : tensor<2x12x5x336x9xi32> to tensor<8x12x17x336x14xi32> + return %padded_0 : tensor<8x12x17x336x14xi32> +} +// CHECK: func @fuse_by_expanding_pad( +// CHECK-SAME: %[[ARG0:.+]]: tensor<2x3x4x5x6x7x8x9xi32>) +// CHECK: %[[PAD:.+]] = tensor.pad %[[ARG0]] +// CHECK-SAME: low[1, 0, 0, 8, 0, 0, 0, 3] high[5, 0, 0, 4, 0, 0, 0, 2] +// CHECK: tensor<2x3x4x5x6x7x8x9xi32> to tensor<8x3x4x17x6x7x8x14xi32> +// CHECK: %[[COLLAPSE:.+]] = tensor.collapse_shape %[[PAD]] {{\[}}[0], [1, 2], [3], [4, 5, 6], [7]] +// CHECK-SAME: : tensor<8x3x4x17x6x7x8x14xi32> into tensor<8x12x17x336x14xi32> +// CHECK: return %[[COLLAPSE]] + +// ----- + +func.func @no_fuse_by_expanding_pad(%arg0 : tensor<2x3x4x5x6x7x8x9xi32>) -> tensor<8x12x17x339x14xi32> { + %collapse = tensor.collapse_shape %arg0 [[0], [1, 2], [3], [4, 5, 6], [7]] : tensor<2x3x4x5x6x7x8x9xi32> into tensor<2x12x5x336x9xi32> + %cst = arith.constant 0 : i32 + %padded_0 = tensor.pad %collapse low[1, 0, 8, 0, 3] high[5, 0, 4, 3, 2] { + ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index): + tensor.yield %cst : i32 + } : tensor<2x12x5x336x9xi32> to tensor<8x12x17x339x14xi32> + return %padded_0 : tensor<8x12x17x339x14xi32> +} +// CHECK: func @no_fuse_by_expanding_pad( +// CHECK-SAME: %[[ARG0:.+]]: tensor<2x3x4x5x6x7x8x9xi32>) +// CHECK: %[[COLLAPSE:.+]] = tensor.collapse_shape %[[ARG0]] {{\[}}[0], [1, 2], [3], [4, 5, 6], [7]] +// CHECK-SAME: : tensor<2x3x4x5x6x7x8x9xi32> into tensor<2x12x5x336x9xi32> +// CHECK: %[[PAD:.+]] = tensor.pad %[[COLLAPSE]] +// CHECK-SAME: low[1, 0, 8, 0, 3] high[5, 0, 4, 3, 2] +// CHECK: tensor<2x12x5x336x9xi32> to tensor<8x12x17x339x14xi32> +// CHECK: return %[[PAD]] + +// ----- + +func.func @fuse_by_expanding_dynamic_pad(%arg0 : tensor, %l0: index, %l1: index, %h0: index, %h1: index) -> tensor { + %collapse = tensor.collapse_shape %arg0 [[0], [1, 2], [3], [4, 5]] : tensor into tensor + %cst = arith.constant 0 : i32 + %padded_0 = tensor.pad %collapse low[%l0, 0, %l1, 0] high[%h0, 0, %h1, 0] { + ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index): + tensor.yield %cst : i32 + } : tensor to tensor + return %padded_0 : tensor +} +// CHECK: func @fuse_by_expanding_dynamic_pad( +// CHECK-SAME: %[[ARG0:.+]]: tensor +// CHECK-SAME: %[[L0:.+]]: index, %[[L1:.+]]: index, %[[H0:.+]]: index, %[[H1:.+]]: index +// CHECK: %[[PAD:.+]] = tensor.pad %[[ARG0]] +// CHECK-SAME: low[%[[L0]], 0, 0, %[[L1]], 0, 0] high[%[[H0]], 0, 0, %[[H1]], 0, 0] +// CHECK: tensor to tensor +// CHECK: %[[COLLAPSE:.+]] = tensor.collapse_shape %[[PAD]] {{\[}}[0], [1, 2], [3], [4, 5]] +// CHECK-SAME: : tensor into tensor +// CHECK: return %[[COLLAPSE]] -- GitLab From 2117677e304d334326f6591f3c75fb2f34dc4bcb Mon Sep 17 00:00:00 2001 From: Max191 <44243577+Max191@users.noreply.github.com> Date: Fri, 7 Jun 2024 06:09:51 -0700 Subject: [PATCH 246/462] [mlir] Fix bugs in expand_shape patterns after semantics changes (#94631) After the `output_shape` field was added to `expand_shape` ops, dynamically sized expand shapes are now possible, but this was not accounted for in the folder. This PR tightens the constraints of the folder to fix this. --- .../mlir/Dialect/Utils/ReshapeOpsUtils.h | 50 ++++++++++--- mlir/test/Dialect/Tensor/canonicalize.mlir | 72 ++++++++++++++++++- 2 files changed, 110 insertions(+), 12 deletions(-) diff --git a/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h b/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h index e8f6edc3f133..89bc57f09ec8 100644 --- a/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h +++ b/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h @@ -85,21 +85,49 @@ bool isReassociationValid(ArrayRef reassociation, template static OpFoldResult foldReshapeOp(ReshapeOpTy reshapeOp, ArrayRef operands) { - + // Fold identity reshape. if (reshapeOp.getSrcType() == reshapeOp.getType()) return reshapeOp.getSrc(); - // Fold producer-consumer reshape ops where the operand type of the - // producer is same as the return type of the consumer. - auto reshapeSrcOp = - reshapeOp.getSrc().template getDefiningOp(); - if (reshapeSrcOp && reshapeSrcOp.getSrcType() == reshapeOp.getResultType()) - return reshapeSrcOp.getSrc(); - // Reshape of a constant can be replaced with a new constant. if (auto elements = dyn_cast_or_null(operands.front())) return elements.reshape(cast(reshapeOp.getResult().getType())); + // Fold if the producer reshape source has the same shape with at most 1 + // dynamic dimension. + auto reshapeSrcOp = + reshapeOp.getSrc().template getDefiningOp(); + if (!reshapeSrcOp) + return nullptr; + auto srcType = reshapeSrcOp.getSrcType(); + auto resultType = reshapeOp.getResultType(); + if (srcType != resultType) + return nullptr; + + if (llvm::count_if(srcType.getShape(), ShapedType::isDynamic) < 2) { + return reshapeSrcOp.getSrc(); + } + + // Fold producer-consumer reshape ops when they are perfect inverses of each + // other: + // 1) Reassociation indices are equivalent. + // 2) Boundary types are equivalent. + // 3) No reassociations have more than 1 dynamic dimension, and reassociated + // shapes are equal for each reassociation. + auto reassociations = reshapeOp.getReassociationIndices(); + if (reassociations != reshapeSrcOp.getReassociationIndices()) + return nullptr; + // If the reshapes are expanding and then collapsing, the ops can be folded + // despite multiple dynamic dimensions. + if (srcType.getRank() < reshapeSrcOp.getResultType().getRank()) + return reshapeSrcOp.getSrc(); + if (llvm::all_of(reassociations, [&](auto reInd) { + ArrayRef srcSlice = + srcType.getShape().slice(reInd.front(), reInd.size()); + return llvm::count_if(srcSlice, ShapedType::isDynamic) < 2; + })) { + return reshapeSrcOp.getSrc(); + } return nullptr; } @@ -360,10 +388,12 @@ private: resultShape.slice(resultIndices.front(), resultIndices.size()); if (srcSubShape.size() == resultSubShape.size()) { - if (srcSubShape == resultSubShape) + if (srcSubShape == resultSubShape && + llvm::count_if(srcSubShape, ShapedType::isDynamic) < 2) { composedReassociation.push_back(srcIndices); - else + } else { return std::nullopt; + } } // Find reassociation to collapse `srcSubShape` into `resultSubShape`. diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir index 6b51d0b294bc..df2bea08577e 100644 --- a/mlir/test/Dialect/Tensor/canonicalize.mlir +++ b/mlir/test/Dialect/Tensor/canonicalize.mlir @@ -1139,7 +1139,7 @@ func.func @fold_collapse_of_expand(%arg0 : tensor<12x4xf32>) -> tensor<12x4xf32> return %1 : tensor<12x4xf32> } // CHECK-LABEL: @fold_collapse_of_expand -// CHECK-NOT: linalg.{{.*}}shape +// CHECK-NOT: tensor.{{.*}}_shape // ----- @@ -1152,7 +1152,75 @@ func.func @fold_collapse_of_expand_dynamic(%arg0 : tensor, %arg1: index return %1 : tensor } // CHECK-LABEL: @fold_collapse_of_expand_dynamic -// CHECK-NOT: linalg.{{.*}}_shape +// CHECK-NOT: tensor.{{.*}}_shape + +// ----- + +func.func @fold_collapse_of_expand_fully_dynamic(%arg0 : tensor, %arg1: index, %arg2: index, %arg3: index) + -> tensor { + %0 = tensor.expand_shape %arg0 [[0, 1], [2]] output_shape [%arg1, %arg2, %arg3] + : tensor into tensor + %1 = tensor.collapse_shape %0 [[0, 1], [2]] + : tensor into tensor + return %1 : tensor +} +// CHECK-LABEL: @fold_collapse_of_expand_fully_dynamic +// CHECK-NOT: tensor.{{.*}}_shape + +// ----- + +func.func @no_fold_parallel_collapse_of_expand_dynamic(%arg0 : tensor, %arg1: index, %arg2: index, %arg3: index, %arg4: index) + -> tensor { + %0 = tensor.expand_shape %arg0 [[0, 1], [2], [3]] output_shape [%arg1, %arg2, %arg3, %arg4] + : tensor into tensor + %1 = tensor.collapse_shape %0 [[0], [1], [2, 3]] + : tensor into tensor + return %1 : tensor +} +// CHECK-LABEL: @no_fold_parallel_collapse_of_expand_dynamic +// CHECK: tensor.expand_shape +// CHECK: %[[COLLAPSE:.+]] = tensor.collapse_shape +// CHECK: return %[[COLLAPSE]] + +// ----- + +func.func @fold_expand_of_collapse(%arg0 : tensor<3x4x4xf32>) -> tensor<3x4x4xf32> { + %0 = tensor.collapse_shape %arg0 [[0, 1], [2]] + : tensor<3x4x4xf32> into tensor<12x4xf32> + %1 = tensor.expand_shape %0 [[0, 1], [2]] output_shape [3, 4, 4] + : tensor<12x4xf32> into tensor<3x4x4xf32> + return %1 : tensor<3x4x4xf32> +} +// CHECK-LABEL: @fold_expand_of_collapse +// CHECK-NOT: tensor.{{.*}}_shape + +// ----- + +func.func @fold_expand_of_collapse_dynamic(%arg0 : tensor, %arg1: index, %arg2: index) + -> tensor { + %0 = tensor.collapse_shape %arg0 [[0, 1], [2]] + : tensor into tensor + %1 = tensor.expand_shape %0 [[0, 1], [2]] output_shape [%arg1, 4, %arg2] + : tensor into tensor + return %1 : tensor +} +// CHECK-LABEL: @fold_expand_of_collapse_dynamic +// CHECK-NOT: tensor.{{.*}}_shape + +// ----- + +func.func @no_fold_expand_of_collapse_dynamic(%arg0 : tensor, %arg1: index, %arg2: index, %arg3: index) + -> tensor { + %0 = tensor.collapse_shape %arg0 [[0, 1], [2]] + : tensor into tensor + %1 = tensor.expand_shape %0 [[0, 1], [2]] output_shape [%arg1, %arg2, %arg3] + : tensor into tensor + return %1 : tensor +} +// CHECK-LABEL: @no_fold_expand_of_collapse_dynamic +// CHECK: tensor.collapse_shape +// CHECK: %[[EXPAND:.+]] = tensor.expand_shape +// CHECK: return %[[EXPAND]] // ----- -- GitLab From ac02168990aa8429898d0c59fec7a78526638c5c Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 7 Jun 2024 14:31:15 +0100 Subject: [PATCH 247/462] [ARM] Clean up neon_vabd.ll, vaba.ll and vabd.ll tests a bit. NFC Change the target triple to remove some unnecessary instructions. --- llvm/test/CodeGen/ARM/neon_vabd.ll | 587 ++++++++++------------------- llvm/test/CodeGen/ARM/vaba.ll | 140 +++---- llvm/test/CodeGen/ARM/vabd.ll | 115 ++---- 3 files changed, 292 insertions(+), 550 deletions(-) diff --git a/llvm/test/CodeGen/ARM/neon_vabd.ll b/llvm/test/CodeGen/ARM/neon_vabd.ll index 907e11c0cf19..8695c3e5f3db 100644 --- a/llvm/test/CodeGen/ARM/neon_vabd.ll +++ b/llvm/test/CodeGen/ARM/neon_vabd.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s +; RUN: llc -mtriple=armv7a-eabihf -mattr=+neon %s -o - | FileCheck %s ; ; SABD @@ -8,11 +8,8 @@ define <8 x i8> @sabd_8b(<8 x i8> %a, <8 x i8> %b) { ; CHECK-LABEL: sabd_8b: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabd.s8 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s8 d0, d0, d1 +; CHECK-NEXT: bx lr %a.sext = sext <8 x i8> %a to <8 x i16> %b.sext = sext <8 x i8> %b to <8 x i16> %sub = sub <8 x i16> %a.sext, %b.sext @@ -24,14 +21,8 @@ define <8 x i8> @sabd_8b(<8 x i8> %a, <8 x i8> %b) { define <16 x i8> @sabd_16b(<16 x i8> %a, <16 x i8> %b) { ; CHECK-LABEL: sabd_16b: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.s8 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s8 q0, q0, q1 +; CHECK-NEXT: bx lr %a.sext = sext <16 x i8> %a to <16 x i16> %b.sext = sext <16 x i8> %b to <16 x i16> %sub = sub <16 x i16> %a.sext, %b.sext @@ -43,11 +34,8 @@ define <16 x i8> @sabd_16b(<16 x i8> %a, <16 x i8> %b) { define <4 x i16> @sabd_4h(<4 x i16> %a, <4 x i16> %b) { ; CHECK-LABEL: sabd_4h: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabd.s16 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s16 d0, d0, d1 +; CHECK-NEXT: bx lr %a.sext = sext <4 x i16> %a to <4 x i32> %b.sext = sext <4 x i16> %b to <4 x i32> %sub = sub <4 x i32> %a.sext, %b.sext @@ -59,15 +47,12 @@ define <4 x i16> @sabd_4h(<4 x i16> %a, <4 x i16> %b) { define <4 x i16> @sabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) { ; CHECK-LABEL: sabd_4h_promoted_ops: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vshl.i16 d16, d16, #8 -; CHECK-NEXT: vshl.i16 d17, d17, #8 +; CHECK-NEXT: vshl.i16 d16, d1, #8 +; CHECK-NEXT: vshl.i16 d17, d0, #8 ; CHECK-NEXT: vshr.s16 d16, d16, #8 ; CHECK-NEXT: vshr.s16 d17, d17, #8 -; CHECK-NEXT: vabd.s16 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s16 d0, d17, d16 +; CHECK-NEXT: bx lr %a.sext = sext <4 x i8> %a to <4 x i16> %b.sext = sext <4 x i8> %b to <4 x i16> %sub = sub <4 x i16> %a.sext, %b.sext @@ -78,14 +63,8 @@ define <4 x i16> @sabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) { define <8 x i16> @sabd_8h(<8 x i16> %a, <8 x i16> %b) { ; CHECK-LABEL: sabd_8h: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.s16 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s16 q0, q0, q1 +; CHECK-NEXT: bx lr %a.sext = sext <8 x i16> %a to <8 x i32> %b.sext = sext <8 x i16> %b to <8 x i32> %sub = sub <8 x i32> %a.sext, %b.sext @@ -97,12 +76,8 @@ define <8 x i16> @sabd_8h(<8 x i16> %a, <8 x i16> %b) { define <8 x i16> @sabd_8h_promoted_ops(<8 x i8> %a, <8 x i8> %b) { ; CHECK-LABEL: sabd_8h_promoted_ops: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabdl.s8 q8, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabdl.s8 q0, d0, d1 +; CHECK-NEXT: bx lr %a.sext = sext <8 x i8> %a to <8 x i16> %b.sext = sext <8 x i8> %b to <8 x i16> %sub = sub <8 x i16> %a.sext, %b.sext @@ -113,11 +88,8 @@ define <8 x i16> @sabd_8h_promoted_ops(<8 x i8> %a, <8 x i8> %b) { define <2 x i32> @sabd_2s(<2 x i32> %a, <2 x i32> %b) { ; CHECK-LABEL: sabd_2s: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabd.s32 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s32 d0, d0, d1 +; CHECK-NEXT: bx lr %a.sext = sext <2 x i32> %a to <2 x i64> %b.sext = sext <2 x i32> %b to <2 x i64> %sub = sub <2 x i64> %a.sext, %b.sext @@ -129,15 +101,12 @@ define <2 x i32> @sabd_2s(<2 x i32> %a, <2 x i32> %b) { define <2 x i32> @sabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: sabd_2s_promoted_ops: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vshl.i32 d16, d16, #16 -; CHECK-NEXT: vshl.i32 d17, d17, #16 +; CHECK-NEXT: vshl.i32 d16, d1, #16 +; CHECK-NEXT: vshl.i32 d17, d0, #16 ; CHECK-NEXT: vshr.s32 d16, d16, #16 ; CHECK-NEXT: vshr.s32 d17, d17, #16 -; CHECK-NEXT: vabd.s32 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s32 d0, d17, d16 +; CHECK-NEXT: bx lr %a.sext = sext <2 x i16> %a to <2 x i32> %b.sext = sext <2 x i16> %b to <2 x i32> %sub = sub <2 x i32> %a.sext, %b.sext @@ -148,14 +117,8 @@ define <2 x i32> @sabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) { define <4 x i32> @sabd_4s(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: sabd_4s: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.s32 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s32 q0, q0, q1 +; CHECK-NEXT: bx lr %a.sext = sext <4 x i32> %a to <4 x i64> %b.sext = sext <4 x i32> %b to <4 x i64> %sub = sub <4 x i64> %a.sext, %b.sext @@ -167,12 +130,8 @@ define <4 x i32> @sabd_4s(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @sabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) { ; CHECK-LABEL: sabd_4s_promoted_ops: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabdl.s16 q8, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabdl.s16 q0, d0, d1 +; CHECK-NEXT: bx lr %a.sext = sext <4 x i16> %a to <4 x i32> %b.sext = sext <4 x i16> %b to <4 x i32> %sub = sub <4 x i32> %a.sext, %b.sext @@ -183,38 +142,35 @@ define <4 x i32> @sabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) { define <2 x i64> @sabd_2d(<2 x i64> %a, <2 x i64> %b) { ; CHECK-LABEL: sabd_2d: ; CHECK: @ %bb.0: -; CHECK-NEXT: .save {r4, r5, r6, r7, r11, lr} -; CHECK-NEXT: push {r4, r5, r6, r7, r11, lr} -; CHECK-NEXT: add r12, sp, #24 -; CHECK-NEXT: asr r6, r3, #31 -; CHECK-NEXT: vld1.64 {d16, d17}, [r12] -; CHECK-NEXT: vmov r12, lr, d17 -; CHECK-NEXT: vmov r7, r5, d16 -; CHECK-NEXT: subs r2, r2, r12 -; CHECK-NEXT: sbcs r3, r3, lr -; CHECK-NEXT: sbcs r4, r6, lr, asr #31 -; CHECK-NEXT: sbc r6, r6, lr, asr #31 -; CHECK-NEXT: eor r2, r2, r6, asr #31 -; CHECK-NEXT: eor r3, r3, r6, asr #31 -; CHECK-NEXT: subs r2, r2, r6, asr #31 -; CHECK-NEXT: sbc r3, r3, r6, asr #31 -; CHECK-NEXT: subs r0, r0, r7 -; CHECK-NEXT: asr r6, r1, #31 -; CHECK-NEXT: sbcs r1, r1, r5 -; CHECK-NEXT: sbcs r7, r6, r5, asr #31 -; CHECK-NEXT: vmov.32 d17[0], r2 -; CHECK-NEXT: sbc r7, r6, r5, asr #31 -; CHECK-NEXT: eor r0, r0, r7, asr #31 -; CHECK-NEXT: subs r0, r0, r7, asr #31 -; CHECK-NEXT: vmov.32 d16[0], r0 -; CHECK-NEXT: eor r0, r1, r7, asr #31 -; CHECK-NEXT: sbc r0, r0, r7, asr #31 -; CHECK-NEXT: vmov.32 d17[1], r3 -; CHECK-NEXT: vmov.32 d16[1], r0 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: pop {r4, r5, r6, r7, r11, lr} -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: .save {r4, r5, r11, lr} +; CHECK-NEXT: push {r4, r5, r11, lr} +; CHECK-NEXT: vmov lr, r1, d1 +; CHECK-NEXT: vmov r2, r3, d3 +; CHECK-NEXT: vmov r12, r0, d0 +; CHECK-NEXT: subs lr, lr, r2 +; CHECK-NEXT: asr r4, r1, #31 +; CHECK-NEXT: sbcs r1, r1, r3 +; CHECK-NEXT: sbcs r2, r4, r3, asr #31 +; CHECK-NEXT: vmov r2, r5, d2 +; CHECK-NEXT: sbc r3, r4, r3, asr #31 +; CHECK-NEXT: eor r4, lr, r3, asr #31 +; CHECK-NEXT: eor r1, r1, r3, asr #31 +; CHECK-NEXT: subs r4, r4, r3, asr #31 +; CHECK-NEXT: sbc lr, r1, r3, asr #31 +; CHECK-NEXT: asr r3, r0, #31 +; CHECK-NEXT: vmov.32 d1[0], r4 +; CHECK-NEXT: subs r2, r12, r2 +; CHECK-NEXT: sbcs r0, r0, r5 +; CHECK-NEXT: sbcs r1, r3, r5, asr #31 +; CHECK-NEXT: sbc r1, r3, r5, asr #31 +; CHECK-NEXT: eor r2, r2, r1, asr #31 +; CHECK-NEXT: eor r0, r0, r1, asr #31 +; CHECK-NEXT: subs r2, r2, r1, asr #31 +; CHECK-NEXT: sbc r0, r0, r1, asr #31 +; CHECK-NEXT: vmov.32 d0[0], r2 +; CHECK-NEXT: vmov.32 d1[1], lr +; CHECK-NEXT: vmov.32 d0[1], r0 +; CHECK-NEXT: pop {r4, r5, r11, pc} %a.sext = sext <2 x i64> %a to <2 x i128> %b.sext = sext <2 x i64> %b to <2 x i128> %sub = sub <2 x i128> %a.sext, %b.sext @@ -226,12 +182,8 @@ define <2 x i64> @sabd_2d(<2 x i64> %a, <2 x i64> %b) { define <2 x i64> @sabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) { ; CHECK-LABEL: sabd_2d_promoted_ops: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabdl.s32 q8, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabdl.s32 q0, d0, d1 +; CHECK-NEXT: bx lr %a.sext = sext <2 x i32> %a to <2 x i64> %b.sext = sext <2 x i32> %b to <2 x i64> %sub = sub <2 x i64> %a.sext, %b.sext @@ -246,11 +198,8 @@ define <2 x i64> @sabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) { define <8 x i8> @uabd_8b(<8 x i8> %a, <8 x i8> %b) { ; CHECK-LABEL: uabd_8b: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabd.u8 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u8 d0, d0, d1 +; CHECK-NEXT: bx lr %a.zext = zext <8 x i8> %a to <8 x i16> %b.zext = zext <8 x i8> %b to <8 x i16> %sub = sub <8 x i16> %a.zext, %b.zext @@ -262,14 +211,8 @@ define <8 x i8> @uabd_8b(<8 x i8> %a, <8 x i8> %b) { define <16 x i8> @uabd_16b(<16 x i8> %a, <16 x i8> %b) { ; CHECK-LABEL: uabd_16b: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.u8 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u8 q0, q0, q1 +; CHECK-NEXT: bx lr %a.zext = zext <16 x i8> %a to <16 x i16> %b.zext = zext <16 x i8> %b to <16 x i16> %sub = sub <16 x i16> %a.zext, %b.zext @@ -281,11 +224,8 @@ define <16 x i8> @uabd_16b(<16 x i8> %a, <16 x i8> %b) { define <4 x i16> @uabd_4h(<4 x i16> %a, <4 x i16> %b) { ; CHECK-LABEL: uabd_4h: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabd.u16 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u16 d0, d0, d1 +; CHECK-NEXT: bx lr %a.zext = zext <4 x i16> %a to <4 x i32> %b.zext = zext <4 x i16> %b to <4 x i32> %sub = sub <4 x i32> %a.zext, %b.zext @@ -297,13 +237,10 @@ define <4 x i16> @uabd_4h(<4 x i16> %a, <4 x i16> %b) { define <4 x i16> @uabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) { ; CHECK-LABEL: uabd_4h_promoted_ops: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vbic.i16 d16, #0xff00 -; CHECK-NEXT: vbic.i16 d17, #0xff00 -; CHECK-NEXT: vabd.u16 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vbic.i16 d1, #0xff00 +; CHECK-NEXT: vbic.i16 d0, #0xff00 +; CHECK-NEXT: vabd.u16 d0, d0, d1 +; CHECK-NEXT: bx lr %a.zext = zext <4 x i8> %a to <4 x i16> %b.zext = zext <4 x i8> %b to <4 x i16> %sub = sub <4 x i16> %a.zext, %b.zext @@ -314,14 +251,8 @@ define <4 x i16> @uabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) { define <8 x i16> @uabd_8h(<8 x i16> %a, <8 x i16> %b) { ; CHECK-LABEL: uabd_8h: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.u16 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u16 q0, q0, q1 +; CHECK-NEXT: bx lr %a.zext = zext <8 x i16> %a to <8 x i32> %b.zext = zext <8 x i16> %b to <8 x i32> %sub = sub <8 x i32> %a.zext, %b.zext @@ -333,12 +264,8 @@ define <8 x i16> @uabd_8h(<8 x i16> %a, <8 x i16> %b) { define <8 x i16> @uabd_8h_promoted_ops(<8 x i8> %a, <8 x i8> %b) { ; CHECK-LABEL: uabd_8h_promoted_ops: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabdl.u8 q8, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabdl.u8 q0, d0, d1 +; CHECK-NEXT: bx lr %a.zext = zext <8 x i8> %a to <8 x i16> %b.zext = zext <8 x i8> %b to <8 x i16> %sub = sub <8 x i16> %a.zext, %b.zext @@ -349,11 +276,8 @@ define <8 x i16> @uabd_8h_promoted_ops(<8 x i8> %a, <8 x i8> %b) { define <2 x i32> @uabd_2s(<2 x i32> %a, <2 x i32> %b) { ; CHECK-LABEL: uabd_2s: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabd.u32 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u32 d0, d0, d1 +; CHECK-NEXT: bx lr %a.zext = zext <2 x i32> %a to <2 x i64> %b.zext = zext <2 x i32> %b to <2 x i64> %sub = sub <2 x i64> %a.zext, %b.zext @@ -366,13 +290,10 @@ define <2 x i32> @uabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: uabd_2s_promoted_ops: ; CHECK: @ %bb.0: ; CHECK-NEXT: vmov.i32 d16, #0xffff -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: vmov d18, r0, r1 -; CHECK-NEXT: vand d17, d17, d16 -; CHECK-NEXT: vand d16, d18, d16 -; CHECK-NEXT: vabd.u32 d16, d16, d17 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vand d17, d1, d16 +; CHECK-NEXT: vand d16, d0, d16 +; CHECK-NEXT: vabd.u32 d0, d16, d17 +; CHECK-NEXT: bx lr %a.zext = zext <2 x i16> %a to <2 x i32> %b.zext = zext <2 x i16> %b to <2 x i32> %sub = sub <2 x i32> %a.zext, %b.zext @@ -383,14 +304,8 @@ define <2 x i32> @uabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) { define <4 x i32> @uabd_4s(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: uabd_4s: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.u32 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u32 q0, q0, q1 +; CHECK-NEXT: bx lr %a.zext = zext <4 x i32> %a to <4 x i64> %b.zext = zext <4 x i32> %b to <4 x i64> %sub = sub <4 x i64> %a.zext, %b.zext @@ -402,12 +317,8 @@ define <4 x i32> @uabd_4s(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @uabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) { ; CHECK-LABEL: uabd_4s_promoted_ops: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabdl.u16 q8, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabdl.u16 q0, d0, d1 +; CHECK-NEXT: bx lr %a.zext = zext <4 x i16> %a to <4 x i32> %b.zext = zext <4 x i16> %b to <4 x i32> %sub = sub <4 x i32> %a.zext, %b.zext @@ -418,37 +329,34 @@ define <4 x i32> @uabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) { define <2 x i64> @uabd_2d(<2 x i64> %a, <2 x i64> %b) { ; CHECK-LABEL: uabd_2d: ; CHECK: @ %bb.0: -; CHECK-NEXT: .save {r4, r5, r6, r7, r11, lr} -; CHECK-NEXT: push {r4, r5, r6, r7, r11, lr} -; CHECK-NEXT: add r12, sp, #24 -; CHECK-NEXT: mov r6, #0 -; CHECK-NEXT: vld1.64 {d16, d17}, [r12] -; CHECK-NEXT: vmov r12, lr, d17 -; CHECK-NEXT: vmov r4, r7, d16 -; CHECK-NEXT: subs r2, r2, r12 -; CHECK-NEXT: sbcs r3, r3, lr -; CHECK-NEXT: sbcs r5, r6, #0 -; CHECK-NEXT: sbc r5, r6, #0 -; CHECK-NEXT: eor r2, r2, r5, asr #31 -; CHECK-NEXT: eor r3, r3, r5, asr #31 -; CHECK-NEXT: subs r2, r2, r5, asr #31 -; CHECK-NEXT: sbc r3, r3, r5, asr #31 -; CHECK-NEXT: subs r0, r0, r4 -; CHECK-NEXT: sbcs r1, r1, r7 -; CHECK-NEXT: vmov.32 d17[0], r2 -; CHECK-NEXT: sbcs r7, r6, #0 -; CHECK-NEXT: sbc r7, r6, #0 -; CHECK-NEXT: eor r0, r0, r7, asr #31 -; CHECK-NEXT: subs r0, r0, r7, asr #31 -; CHECK-NEXT: vmov.32 d16[0], r0 -; CHECK-NEXT: eor r0, r1, r7, asr #31 -; CHECK-NEXT: sbc r0, r0, r7, asr #31 -; CHECK-NEXT: vmov.32 d17[1], r3 -; CHECK-NEXT: vmov.32 d16[1], r0 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: pop {r4, r5, r6, r7, r11, lr} -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: .save {r4, r5, r6, lr} +; CHECK-NEXT: push {r4, r5, r6, lr} +; CHECK-NEXT: vmov r0, r12, d3 +; CHECK-NEXT: mov r1, #0 +; CHECK-NEXT: vmov r2, r3, d1 +; CHECK-NEXT: vmov lr, r6, d2 +; CHECK-NEXT: vmov r4, r5, d0 +; CHECK-NEXT: subs r0, r2, r0 +; CHECK-NEXT: sbcs r2, r3, r12 +; CHECK-NEXT: sbcs r3, r1, #0 +; CHECK-NEXT: sbc r3, r1, #0 +; CHECK-NEXT: eor r0, r0, r3, asr #31 +; CHECK-NEXT: eor r2, r2, r3, asr #31 +; CHECK-NEXT: subs r0, r0, r3, asr #31 +; CHECK-NEXT: sbc r2, r2, r3, asr #31 +; CHECK-NEXT: subs r3, r4, lr +; CHECK-NEXT: sbcs r6, r5, r6 +; CHECK-NEXT: vmov.32 d1[0], r0 +; CHECK-NEXT: sbcs r5, r1, #0 +; CHECK-NEXT: sbc r1, r1, #0 +; CHECK-NEXT: eor r3, r3, r1, asr #31 +; CHECK-NEXT: subs r0, r3, r1, asr #31 +; CHECK-NEXT: vmov.32 d0[0], r0 +; CHECK-NEXT: eor r0, r6, r1, asr #31 +; CHECK-NEXT: sbc r0, r0, r1, asr #31 +; CHECK-NEXT: vmov.32 d1[1], r2 +; CHECK-NEXT: vmov.32 d0[1], r0 +; CHECK-NEXT: pop {r4, r5, r6, pc} %a.zext = zext <2 x i64> %a to <2 x i128> %b.zext = zext <2 x i64> %b to <2 x i128> %sub = sub <2 x i128> %a.zext, %b.zext @@ -460,12 +368,8 @@ define <2 x i64> @uabd_2d(<2 x i64> %a, <2 x i64> %b) { define <2 x i64> @uabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) { ; CHECK-LABEL: uabd_2d_promoted_ops: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d16, r2, r3 -; CHECK-NEXT: vmov d17, r0, r1 -; CHECK-NEXT: vabdl.u32 q8, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabdl.u32 q0, d0, d1 +; CHECK-NEXT: bx lr %a.zext = zext <2 x i32> %a to <2 x i64> %b.zext = zext <2 x i32> %b to <2 x i64> %sub = sub <2 x i64> %a.zext, %b.zext @@ -476,15 +380,9 @@ define <2 x i64> @uabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) { define <16 x i8> @uabd_v16i8_nuw(<16 x i8> %a, <16 x i8> %b) { ; CHECK-LABEL: uabd_v16i8_nuw: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vsub.i8 q8, q8, q9 -; CHECK-NEXT: vabs.s8 q8, q8 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vsub.i8 q8, q0, q1 +; CHECK-NEXT: vabs.s8 q0, q8 +; CHECK-NEXT: bx lr %sub = sub nuw <16 x i8> %a, %b %abs = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %sub, i1 true) ret <16 x i8> %abs @@ -493,15 +391,9 @@ define <16 x i8> @uabd_v16i8_nuw(<16 x i8> %a, <16 x i8> %b) { define <8 x i16> @uabd_v8i16_nuw(<8 x i16> %a, <8 x i16> %b) { ; CHECK-LABEL: uabd_v8i16_nuw: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vsub.i16 q8, q8, q9 -; CHECK-NEXT: vabs.s16 q8, q8 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vsub.i16 q8, q0, q1 +; CHECK-NEXT: vabs.s16 q0, q8 +; CHECK-NEXT: bx lr %sub = sub nuw <8 x i16> %a, %b %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true) ret <8 x i16> %abs @@ -510,15 +402,9 @@ define <8 x i16> @uabd_v8i16_nuw(<8 x i16> %a, <8 x i16> %b) { define <4 x i32> @uabd_v4i32_nuw(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: uabd_v4i32_nuw: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vsub.i32 q8, q8, q9 -; CHECK-NEXT: vabs.s32 q8, q8 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vsub.i32 q8, q0, q1 +; CHECK-NEXT: vabs.s32 q0, q8 +; CHECK-NEXT: bx lr %sub = sub nuw <4 x i32> %a, %b %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true) ret <4 x i32> %abs @@ -527,17 +413,11 @@ define <4 x i32> @uabd_v4i32_nuw(<4 x i32> %a, <4 x i32> %b) { define <2 x i64> @uabd_v2i64_nuw(<2 x i64> %a, <2 x i64> %b) { ; CHECK-LABEL: uabd_v2i64_nuw: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vsub.i64 q8, q8, q9 +; CHECK-NEXT: vsub.i64 q8, q0, q1 ; CHECK-NEXT: vshr.s64 q9, q8, #63 ; CHECK-NEXT: veor q8, q8, q9 -; CHECK-NEXT: vsub.i64 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vsub.i64 q0, q8, q9 +; CHECK-NEXT: bx lr %sub = sub nuw <2 x i64> %a, %b %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true) ret <2 x i64> %abs @@ -546,14 +426,8 @@ define <2 x i64> @uabd_v2i64_nuw(<2 x i64> %a, <2 x i64> %b) { define <16 x i8> @sabd_v16i8_nsw(<16 x i8> %a, <16 x i8> %b) { ; CHECK-LABEL: sabd_v16i8_nsw: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.s8 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s8 q0, q0, q1 +; CHECK-NEXT: bx lr %sub = sub nsw <16 x i8> %a, %b %abs = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %sub, i1 true) ret <16 x i8> %abs @@ -562,14 +436,8 @@ define <16 x i8> @sabd_v16i8_nsw(<16 x i8> %a, <16 x i8> %b) { define <8 x i16> @sabd_v8i16_nsw(<8 x i16> %a, <8 x i16> %b) { ; CHECK-LABEL: sabd_v8i16_nsw: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.s16 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s16 q0, q0, q1 +; CHECK-NEXT: bx lr %sub = sub nsw <8 x i16> %a, %b %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true) ret <8 x i16> %abs @@ -578,14 +446,8 @@ define <8 x i16> @sabd_v8i16_nsw(<8 x i16> %a, <8 x i16> %b) { define <4 x i32> @sabd_v4i32_nsw(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: sabd_v4i32_nsw: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.s32 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s32 q0, q0, q1 +; CHECK-NEXT: bx lr %sub = sub nsw <4 x i32> %a, %b %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true) ret <4 x i32> %abs @@ -594,17 +456,11 @@ define <4 x i32> @sabd_v4i32_nsw(<4 x i32> %a, <4 x i32> %b) { define <2 x i64> @sabd_v2i64_nsw(<2 x i64> %a, <2 x i64> %b) { ; CHECK-LABEL: sabd_v2i64_nsw: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vsub.i64 q8, q8, q9 +; CHECK-NEXT: vsub.i64 q8, q0, q1 ; CHECK-NEXT: vshr.s64 q9, q8, #63 ; CHECK-NEXT: veor q8, q8, q9 -; CHECK-NEXT: vsub.i64 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vsub.i64 q0, q8, q9 +; CHECK-NEXT: bx lr %sub = sub nsw <2 x i64> %a, %b %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true) ret <2 x i64> %abs @@ -613,14 +469,8 @@ define <2 x i64> @sabd_v2i64_nsw(<2 x i64> %a, <2 x i64> %b) { define <16 x i8> @smaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) { ; CHECK-LABEL: smaxmin_v16i8: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.s8 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s8 q0, q0, q1 +; CHECK-NEXT: bx lr %a = tail call <16 x i8> @llvm.smax.v16i8(<16 x i8> %0, <16 x i8> %1) %b = tail call <16 x i8> @llvm.smin.v16i8(<16 x i8> %0, <16 x i8> %1) %sub = sub <16 x i8> %a, %b @@ -630,14 +480,8 @@ define <16 x i8> @smaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) { define <8 x i16> @smaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) { ; CHECK-LABEL: smaxmin_v8i16: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.s16 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s16 q0, q0, q1 +; CHECK-NEXT: bx lr %a = tail call <8 x i16> @llvm.smax.v8i16(<8 x i16> %0, <8 x i16> %1) %b = tail call <8 x i16> @llvm.smin.v8i16(<8 x i16> %0, <8 x i16> %1) %sub = sub <8 x i16> %a, %b @@ -647,14 +491,8 @@ define <8 x i16> @smaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) { define <4 x i32> @smaxmin_v4i32(<4 x i32> %0, <4 x i32> %1) { ; CHECK-LABEL: smaxmin_v4i32: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.s32 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s32 q0, q0, q1 +; CHECK-NEXT: bx lr %a = tail call <4 x i32> @llvm.smax.v4i32(<4 x i32> %0, <4 x i32> %1) %b = tail call <4 x i32> @llvm.smin.v4i32(<4 x i32> %0, <4 x i32> %1) %sub = sub <4 x i32> %a, %b @@ -666,57 +504,54 @@ define <2 x i64> @smaxmin_v2i64(<2 x i64> %0, <2 x i64> %1) { ; CHECK: @ %bb.0: ; CHECK-NEXT: .save {r4, r5, r6, r7, r8, lr} ; CHECK-NEXT: push {r4, r5, r6, r7, r8, lr} -; CHECK-NEXT: add r6, sp, #24 -; CHECK-NEXT: mov r8, #0 -; CHECK-NEXT: vld1.64 {d18, d19}, [r6] -; CHECK-NEXT: vmov r7, r12, d19 -; CHECK-NEXT: vmov r4, lr, d18 -; CHECK-NEXT: subs r5, r2, r7 -; CHECK-NEXT: sbcs r5, r3, r12 -; CHECK-NEXT: mov r6, r7 -; CHECK-NEXT: mov r5, #0 -; CHECK-NEXT: movlt r5, #1 -; CHECK-NEXT: cmp r5, #0 -; CHECK-NEXT: movne r6, r2 -; CHECK-NEXT: mov r5, r12 -; CHECK-NEXT: vmov.32 d17[0], r6 -; CHECK-NEXT: movne r5, r3 -; CHECK-NEXT: mov r6, r4 -; CHECK-NEXT: vmov.32 d17[1], r5 -; CHECK-NEXT: subs r5, r4, r0 -; CHECK-NEXT: sbcs r5, lr, r1 +; CHECK-NEXT: vmov r1, r0, d3 +; CHECK-NEXT: mov r12, #0 +; CHECK-NEXT: vmov r4, lr, d1 +; CHECK-NEXT: vmov r6, r8, d0 +; CHECK-NEXT: subs r2, r4, r1 +; CHECK-NEXT: mov r3, r0 +; CHECK-NEXT: sbcs r2, lr, r0 +; CHECK-NEXT: mov r2, #0 +; CHECK-NEXT: movwlt r2, #1 +; CHECK-NEXT: cmp r2, #0 +; CHECK-NEXT: mov r2, r1 +; CHECK-NEXT: movne r3, lr +; CHECK-NEXT: movne r2, r4 +; CHECK-NEXT: vmov.32 d17[0], r2 +; CHECK-NEXT: vmov.32 d17[1], r3 +; CHECK-NEXT: vmov r2, r3, d2 +; CHECK-NEXT: subs r5, r2, r6 +; CHECK-NEXT: sbcs r5, r3, r8 +; CHECK-NEXT: mov r7, r2 ; CHECK-NEXT: mov r5, #0 -; CHECK-NEXT: movlt r5, #1 +; CHECK-NEXT: movwlt r5, #1 ; CHECK-NEXT: cmp r5, #0 -; CHECK-NEXT: movne r6, r0 -; CHECK-NEXT: vmov.32 d18[0], r6 -; CHECK-NEXT: subs r6, r7, r2 -; CHECK-NEXT: sbcs r6, r12, r3 -; CHECK-NEXT: mov r6, #0 -; CHECK-NEXT: movlt r6, #1 -; CHECK-NEXT: cmp r6, #0 -; CHECK-NEXT: movne r7, r2 -; CHECK-NEXT: subs r2, r0, r4 -; CHECK-NEXT: sbcs r2, r1, lr -; CHECK-NEXT: vmov.32 d19[0], r7 -; CHECK-NEXT: movlt r8, #1 -; CHECK-NEXT: cmp r8, #0 -; CHECK-NEXT: movne r4, r0 -; CHECK-NEXT: mov r0, lr -; CHECK-NEXT: vmov.32 d16[0], r4 -; CHECK-NEXT: movne r0, r1 -; CHECK-NEXT: cmp r6, #0 -; CHECK-NEXT: movne r12, r3 +; CHECK-NEXT: movne r7, r6 +; CHECK-NEXT: vmov.32 d18[0], r7 +; CHECK-NEXT: subs r7, r1, r4 +; CHECK-NEXT: sbcs r7, r0, lr +; CHECK-NEXT: mov r7, #0 +; CHECK-NEXT: movwlt r7, #1 +; CHECK-NEXT: cmp r7, #0 +; CHECK-NEXT: movne r1, r4 +; CHECK-NEXT: vmov.32 d19[0], r1 +; CHECK-NEXT: subs r1, r6, r2 +; CHECK-NEXT: sbcs r1, r8, r3 +; CHECK-NEXT: movwlt r12, #1 +; CHECK-NEXT: cmp r12, #0 +; CHECK-NEXT: movne r2, r6 +; CHECK-NEXT: mov r1, r3 +; CHECK-NEXT: vmov.32 d16[0], r2 +; CHECK-NEXT: movne r1, r8 +; CHECK-NEXT: cmp r7, #0 +; CHECK-NEXT: movne r0, lr ; CHECK-NEXT: cmp r5, #0 -; CHECK-NEXT: vmov.32 d16[1], r0 -; CHECK-NEXT: movne lr, r1 -; CHECK-NEXT: vmov.32 d19[1], r12 -; CHECK-NEXT: vmov.32 d18[1], lr -; CHECK-NEXT: vsub.i64 q8, q9, q8 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: pop {r4, r5, r6, r7, r8, lr} -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vmov.32 d16[1], r1 +; CHECK-NEXT: movne r3, r8 +; CHECK-NEXT: vmov.32 d19[1], r0 +; CHECK-NEXT: vmov.32 d18[1], r3 +; CHECK-NEXT: vsub.i64 q0, q9, q8 +; CHECK-NEXT: pop {r4, r5, r6, r7, r8, pc} %a = tail call <2 x i64> @llvm.smax.v2i64(<2 x i64> %0, <2 x i64> %1) %b = tail call <2 x i64> @llvm.smin.v2i64(<2 x i64> %0, <2 x i64> %1) %sub = sub <2 x i64> %a, %b @@ -726,14 +561,8 @@ define <2 x i64> @smaxmin_v2i64(<2 x i64> %0, <2 x i64> %1) { define <16 x i8> @umaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) { ; CHECK-LABEL: umaxmin_v16i8: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.u8 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u8 q0, q0, q1 +; CHECK-NEXT: bx lr %a = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %0, <16 x i8> %1) %b = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %0, <16 x i8> %1) %sub = sub <16 x i8> %a, %b @@ -743,14 +572,8 @@ define <16 x i8> @umaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) { define <8 x i16> @umaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) { ; CHECK-LABEL: umaxmin_v8i16: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.u16 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u16 q0, q0, q1 +; CHECK-NEXT: bx lr %a = tail call <8 x i16> @llvm.umax.v8i16(<8 x i16> %0, <8 x i16> %1) %b = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %0, <8 x i16> %1) %sub = sub <8 x i16> %a, %b @@ -760,14 +583,8 @@ define <8 x i16> @umaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) { define <4 x i32> @umaxmin_v4i32(<4 x i32> %0, <4 x i32> %1) { ; CHECK-LABEL: umaxmin_v4i32: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.u32 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u32 q0, q0, q1 +; CHECK-NEXT: bx lr %a = tail call <4 x i32> @llvm.umax.v4i32(<4 x i32> %0, <4 x i32> %1) %b = tail call <4 x i32> @llvm.umin.v4i32(<4 x i32> %0, <4 x i32> %1) %sub = sub <4 x i32> %a, %b @@ -777,18 +594,12 @@ define <4 x i32> @umaxmin_v4i32(<4 x i32> %0, <4 x i32> %1) { define <2 x i64> @umaxmin_v2i64(<2 x i64> %0, <2 x i64> %1) { ; CHECK-LABEL: umaxmin_v2i64: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vqsub.u64 q10, q8, q9 -; CHECK-NEXT: vqsub.u64 q9, q9, q8 -; CHECK-NEXT: vsub.i64 q10, q10, q8 -; CHECK-NEXT: vadd.i64 q8, q8, q9 -; CHECK-NEXT: vadd.i64 q8, q8, q10 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vqsub.u64 q8, q0, q1 +; CHECK-NEXT: vqsub.u64 q9, q1, q0 +; CHECK-NEXT: vsub.i64 q8, q8, q0 +; CHECK-NEXT: vadd.i64 q9, q0, q9 +; CHECK-NEXT: vadd.i64 q0, q9, q8 +; CHECK-NEXT: bx lr %a = tail call <2 x i64> @llvm.umax.v2i64(<2 x i64> %0, <2 x i64> %1) %b = tail call <2 x i64> @llvm.umin.v2i64(<2 x i64> %0, <2 x i64> %1) %sub = sub <2 x i64> %a, %b @@ -798,14 +609,8 @@ define <2 x i64> @umaxmin_v2i64(<2 x i64> %0, <2 x i64> %1) { define <16 x i8> @umaxmin_v16i8_com1(<16 x i8> %0, <16 x i8> %1) { ; CHECK-LABEL: umaxmin_v16i8_com1: ; CHECK: @ %bb.0: -; CHECK-NEXT: vmov d17, r2, r3 -; CHECK-NEXT: mov r12, sp -; CHECK-NEXT: vld1.64 {d18, d19}, [r12] -; CHECK-NEXT: vmov d16, r0, r1 -; CHECK-NEXT: vabd.u8 q8, q8, q9 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u8 q0, q0, q1 +; CHECK-NEXT: bx lr %a = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %0, <16 x i8> %1) %b = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %1, <16 x i8> %0) %sub = sub <16 x i8> %a, %b diff --git a/llvm/test/CodeGen/ARM/vaba.ll b/llvm/test/CodeGen/ARM/vaba.ll index 14419a345d82..090eaca96b26 100644 --- a/llvm/test/CodeGen/ARM/vaba.ll +++ b/llvm/test/CodeGen/ARM/vaba.ll @@ -1,15 +1,14 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 -; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s +; RUN: llc -mtriple=armv7a-eabihf -mattr=+neon %s -o - | FileCheck %s define <8 x i8> @vabas8(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK-LABEL: vabas8: ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r2] ; CHECK-NEXT: vldr d17, [r1] -; CHECK-NEXT: vldr d18, [r0] -; CHECK-NEXT: vaba.s8 d18, d17, d16 -; CHECK-NEXT: vmov r0, r1, d18 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vldr d0, [r0] +; CHECK-NEXT: vaba.s8 d0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <8 x i8>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = load <8 x i8>, ptr %C @@ -23,10 +22,9 @@ define <4 x i16> @vabas16(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r2] ; CHECK-NEXT: vldr d17, [r1] -; CHECK-NEXT: vldr d18, [r0] -; CHECK-NEXT: vaba.s16 d18, d17, d16 -; CHECK-NEXT: vmov r0, r1, d18 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vldr d0, [r0] +; CHECK-NEXT: vaba.s16 d0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <4 x i16>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = load <4 x i16>, ptr %C @@ -40,10 +38,9 @@ define <2 x i32> @vabas32(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r2] ; CHECK-NEXT: vldr d17, [r1] -; CHECK-NEXT: vldr d18, [r0] -; CHECK-NEXT: vaba.s32 d18, d17, d16 -; CHECK-NEXT: vmov r0, r1, d18 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vldr d0, [r0] +; CHECK-NEXT: vaba.s32 d0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <2 x i32>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = load <2 x i32>, ptr %C @@ -57,10 +54,9 @@ define <8 x i8> @vabau8(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r2] ; CHECK-NEXT: vldr d17, [r1] -; CHECK-NEXT: vldr d18, [r0] -; CHECK-NEXT: vaba.u8 d18, d17, d16 -; CHECK-NEXT: vmov r0, r1, d18 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vldr d0, [r0] +; CHECK-NEXT: vaba.u8 d0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <8 x i8>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = load <8 x i8>, ptr %C @@ -74,10 +70,9 @@ define <4 x i16> @vabau16(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r2] ; CHECK-NEXT: vldr d17, [r1] -; CHECK-NEXT: vldr d18, [r0] -; CHECK-NEXT: vaba.u16 d18, d17, d16 -; CHECK-NEXT: vmov r0, r1, d18 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vldr d0, [r0] +; CHECK-NEXT: vaba.u16 d0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <4 x i16>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = load <4 x i16>, ptr %C @@ -91,10 +86,9 @@ define <2 x i32> @vabau32(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r2] ; CHECK-NEXT: vldr d17, [r1] -; CHECK-NEXT: vldr d18, [r0] -; CHECK-NEXT: vaba.u32 d18, d17, d16 -; CHECK-NEXT: vmov r0, r1, d18 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vldr d0, [r0] +; CHECK-NEXT: vaba.u32 d0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <2 x i32>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = load <2 x i32>, ptr %C @@ -108,11 +102,9 @@ define <16 x i8> @vabaQs8(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r2] ; CHECK-NEXT: vld1.64 {d18, d19}, [r1] -; CHECK-NEXT: vld1.64 {d20, d21}, [r0] -; CHECK-NEXT: vaba.s8 q10, q9, q8 -; CHECK-NEXT: vmov r0, r1, d20 -; CHECK-NEXT: vmov r2, r3, d21 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vld1.64 {d0, d1}, [r0] +; CHECK-NEXT: vaba.s8 q0, q9, q8 +; CHECK-NEXT: bx lr %tmp1 = load <16 x i8>, ptr %A %tmp2 = load <16 x i8>, ptr %B %tmp3 = load <16 x i8>, ptr %C @@ -126,11 +118,9 @@ define <8 x i16> @vabaQs16(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r2] ; CHECK-NEXT: vld1.64 {d18, d19}, [r1] -; CHECK-NEXT: vld1.64 {d20, d21}, [r0] -; CHECK-NEXT: vaba.s16 q10, q9, q8 -; CHECK-NEXT: vmov r0, r1, d20 -; CHECK-NEXT: vmov r2, r3, d21 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vld1.64 {d0, d1}, [r0] +; CHECK-NEXT: vaba.s16 q0, q9, q8 +; CHECK-NEXT: bx lr %tmp1 = load <8 x i16>, ptr %A %tmp2 = load <8 x i16>, ptr %B %tmp3 = load <8 x i16>, ptr %C @@ -144,11 +134,9 @@ define <4 x i32> @vabaQs32(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r2] ; CHECK-NEXT: vld1.64 {d18, d19}, [r1] -; CHECK-NEXT: vld1.64 {d20, d21}, [r0] -; CHECK-NEXT: vaba.s32 q10, q9, q8 -; CHECK-NEXT: vmov r0, r1, d20 -; CHECK-NEXT: vmov r2, r3, d21 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vld1.64 {d0, d1}, [r0] +; CHECK-NEXT: vaba.s32 q0, q9, q8 +; CHECK-NEXT: bx lr %tmp1 = load <4 x i32>, ptr %A %tmp2 = load <4 x i32>, ptr %B %tmp3 = load <4 x i32>, ptr %C @@ -162,11 +150,9 @@ define <16 x i8> @vabaQu8(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r2] ; CHECK-NEXT: vld1.64 {d18, d19}, [r1] -; CHECK-NEXT: vld1.64 {d20, d21}, [r0] -; CHECK-NEXT: vaba.u8 q10, q9, q8 -; CHECK-NEXT: vmov r0, r1, d20 -; CHECK-NEXT: vmov r2, r3, d21 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vld1.64 {d0, d1}, [r0] +; CHECK-NEXT: vaba.u8 q0, q9, q8 +; CHECK-NEXT: bx lr %tmp1 = load <16 x i8>, ptr %A %tmp2 = load <16 x i8>, ptr %B %tmp3 = load <16 x i8>, ptr %C @@ -180,11 +166,9 @@ define <8 x i16> @vabaQu16(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r2] ; CHECK-NEXT: vld1.64 {d18, d19}, [r1] -; CHECK-NEXT: vld1.64 {d20, d21}, [r0] -; CHECK-NEXT: vaba.u16 q10, q9, q8 -; CHECK-NEXT: vmov r0, r1, d20 -; CHECK-NEXT: vmov r2, r3, d21 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vld1.64 {d0, d1}, [r0] +; CHECK-NEXT: vaba.u16 q0, q9, q8 +; CHECK-NEXT: bx lr %tmp1 = load <8 x i16>, ptr %A %tmp2 = load <8 x i16>, ptr %B %tmp3 = load <8 x i16>, ptr %C @@ -198,11 +182,9 @@ define <4 x i32> @vabaQu32(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r2] ; CHECK-NEXT: vld1.64 {d18, d19}, [r1] -; CHECK-NEXT: vld1.64 {d20, d21}, [r0] -; CHECK-NEXT: vaba.u32 q10, q9, q8 -; CHECK-NEXT: vmov r0, r1, d20 -; CHECK-NEXT: vmov r2, r3, d21 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vld1.64 {d0, d1}, [r0] +; CHECK-NEXT: vaba.u32 q0, q9, q8 +; CHECK-NEXT: bx lr %tmp1 = load <4 x i32>, ptr %A %tmp2 = load <4 x i32>, ptr %B %tmp3 = load <4 x i32>, ptr %C @@ -232,11 +214,9 @@ define <8 x i16> @vabals8(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r2] ; CHECK-NEXT: vldr d17, [r1] -; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vabal.s8 q9, d17, d16 -; CHECK-NEXT: vmov r0, r1, d18 -; CHECK-NEXT: vmov r2, r3, d19 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vld1.64 {d0, d1}, [r0] +; CHECK-NEXT: vabal.s8 q0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <8 x i16>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = load <8 x i8>, ptr %C @@ -251,11 +231,9 @@ define <4 x i32> @vabals16(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r2] ; CHECK-NEXT: vldr d17, [r1] -; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vabal.s16 q9, d17, d16 -; CHECK-NEXT: vmov r0, r1, d18 -; CHECK-NEXT: vmov r2, r3, d19 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vld1.64 {d0, d1}, [r0] +; CHECK-NEXT: vabal.s16 q0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <4 x i32>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = load <4 x i16>, ptr %C @@ -270,11 +248,9 @@ define <2 x i64> @vabals32(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r2] ; CHECK-NEXT: vldr d17, [r1] -; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vabal.s32 q9, d17, d16 -; CHECK-NEXT: vmov r0, r1, d18 -; CHECK-NEXT: vmov r2, r3, d19 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vld1.64 {d0, d1}, [r0] +; CHECK-NEXT: vabal.s32 q0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <2 x i64>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = load <2 x i32>, ptr %C @@ -289,11 +265,9 @@ define <8 x i16> @vabalu8(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r2] ; CHECK-NEXT: vldr d17, [r1] -; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vabal.u8 q9, d17, d16 -; CHECK-NEXT: vmov r0, r1, d18 -; CHECK-NEXT: vmov r2, r3, d19 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vld1.64 {d0, d1}, [r0] +; CHECK-NEXT: vabal.u8 q0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <8 x i16>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = load <8 x i8>, ptr %C @@ -308,11 +282,9 @@ define <4 x i32> @vabalu16(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r2] ; CHECK-NEXT: vldr d17, [r1] -; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vabal.u16 q9, d17, d16 -; CHECK-NEXT: vmov r0, r1, d18 -; CHECK-NEXT: vmov r2, r3, d19 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vld1.64 {d0, d1}, [r0] +; CHECK-NEXT: vabal.u16 q0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <4 x i32>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = load <4 x i16>, ptr %C @@ -327,11 +299,9 @@ define <2 x i64> @vabalu32(ptr %A, ptr %B, ptr %C) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r2] ; CHECK-NEXT: vldr d17, [r1] -; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vabal.u32 q9, d17, d16 -; CHECK-NEXT: vmov r0, r1, d18 -; CHECK-NEXT: vmov r2, r3, d19 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vld1.64 {d0, d1}, [r0] +; CHECK-NEXT: vabal.u32 q0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <2 x i64>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = load <2 x i32>, ptr %C diff --git a/llvm/test/CodeGen/ARM/vabd.ll b/llvm/test/CodeGen/ARM/vabd.ll index 4184e9275a25..398f7963535c 100644 --- a/llvm/test/CodeGen/ARM/vabd.ll +++ b/llvm/test/CodeGen/ARM/vabd.ll @@ -1,14 +1,13 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 -; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s +; RUN: llc -mtriple=armv7a-eabihf -mattr=+neon %s -o - | FileCheck %s define <8 x i8> @vabds8(ptr %A, ptr %B) nounwind { ; CHECK-LABEL: vabds8: ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] -; CHECK-NEXT: vabd.s8 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s8 d0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <8 x i8>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = call <8 x i8> @llvm.arm.neon.vabds.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2) @@ -20,9 +19,8 @@ define <4 x i16> @vabds16(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] -; CHECK-NEXT: vabd.s16 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s16 d0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <4 x i16>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = call <4 x i16> @llvm.arm.neon.vabds.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2) @@ -34,9 +32,8 @@ define <2 x i32> @vabds32(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] -; CHECK-NEXT: vabd.s32 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s32 d0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <2 x i32>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = call <2 x i32> @llvm.arm.neon.vabds.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2) @@ -48,9 +45,8 @@ define <8 x i8> @vabdu8(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] -; CHECK-NEXT: vabd.u8 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u8 d0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <8 x i8>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = call <8 x i8> @llvm.arm.neon.vabdu.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2) @@ -62,9 +58,8 @@ define <4 x i16> @vabdu16(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] -; CHECK-NEXT: vabd.u16 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u16 d0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <4 x i16>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = call <4 x i16> @llvm.arm.neon.vabdu.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2) @@ -76,9 +71,8 @@ define <2 x i32> @vabdu32(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] -; CHECK-NEXT: vabd.u32 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u32 d0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <2 x i32>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = call <2 x i32> @llvm.arm.neon.vabdu.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2) @@ -90,9 +84,8 @@ define <2 x float> @vabdf32(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] -; CHECK-NEXT: vabd.f32 d16, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.f32 d0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <2 x float>, ptr %A %tmp2 = load <2 x float>, ptr %B %tmp3 = call <2 x float> @llvm.arm.neon.vabds.v2f32(<2 x float> %tmp1, <2 x float> %tmp2) @@ -104,10 +97,8 @@ define <16 x i8> @vabdQs8(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r1] ; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vabd.s8 q8, q9, q8 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s8 q0, q9, q8 +; CHECK-NEXT: bx lr %tmp1 = load <16 x i8>, ptr %A %tmp2 = load <16 x i8>, ptr %B %tmp3 = call <16 x i8> @llvm.arm.neon.vabds.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2) @@ -119,10 +110,8 @@ define <8 x i16> @vabdQs16(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r1] ; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vabd.s16 q8, q9, q8 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s16 q0, q9, q8 +; CHECK-NEXT: bx lr %tmp1 = load <8 x i16>, ptr %A %tmp2 = load <8 x i16>, ptr %B %tmp3 = call <8 x i16> @llvm.arm.neon.vabds.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2) @@ -134,10 +123,8 @@ define <4 x i32> @vabdQs32(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r1] ; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vabd.s32 q8, q9, q8 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.s32 q0, q9, q8 +; CHECK-NEXT: bx lr %tmp1 = load <4 x i32>, ptr %A %tmp2 = load <4 x i32>, ptr %B %tmp3 = call <4 x i32> @llvm.arm.neon.vabds.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2) @@ -149,10 +136,8 @@ define <16 x i8> @vabdQu8(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r1] ; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vabd.u8 q8, q9, q8 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u8 q0, q9, q8 +; CHECK-NEXT: bx lr %tmp1 = load <16 x i8>, ptr %A %tmp2 = load <16 x i8>, ptr %B %tmp3 = call <16 x i8> @llvm.arm.neon.vabdu.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2) @@ -164,10 +149,8 @@ define <8 x i16> @vabdQu16(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r1] ; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vabd.u16 q8, q9, q8 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u16 q0, q9, q8 +; CHECK-NEXT: bx lr %tmp1 = load <8 x i16>, ptr %A %tmp2 = load <8 x i16>, ptr %B %tmp3 = call <8 x i16> @llvm.arm.neon.vabdu.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2) @@ -179,10 +162,8 @@ define <4 x i32> @vabdQu32(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r1] ; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vabd.u32 q8, q9, q8 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.u32 q0, q9, q8 +; CHECK-NEXT: bx lr %tmp1 = load <4 x i32>, ptr %A %tmp2 = load <4 x i32>, ptr %B %tmp3 = call <4 x i32> @llvm.arm.neon.vabdu.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2) @@ -194,10 +175,8 @@ define <4 x float> @vabdQf32(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r1] ; CHECK-NEXT: vld1.64 {d18, d19}, [r0] -; CHECK-NEXT: vabd.f32 q8, q9, q8 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabd.f32 q0, q9, q8 +; CHECK-NEXT: bx lr %tmp1 = load <4 x float>, ptr %A %tmp2 = load <4 x float>, ptr %B %tmp3 = call <4 x float> @llvm.arm.neon.vabds.v4f32(<4 x float> %tmp1, <4 x float> %tmp2) @@ -229,10 +208,8 @@ define <8 x i16> @vabdls8(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] -; CHECK-NEXT: vabdl.s8 q8, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabdl.s8 q0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <8 x i8>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = call <8 x i8> @llvm.arm.neon.vabds.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2) @@ -245,10 +222,8 @@ define <4 x i32> @vabdls16(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] -; CHECK-NEXT: vabdl.s16 q8, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabdl.s16 q0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <4 x i16>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = call <4 x i16> @llvm.arm.neon.vabds.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2) @@ -261,10 +236,8 @@ define <2 x i64> @vabdls32(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] -; CHECK-NEXT: vabdl.s32 q8, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabdl.s32 q0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <2 x i32>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = call <2 x i32> @llvm.arm.neon.vabds.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2) @@ -277,10 +250,8 @@ define <8 x i16> @vabdlu8(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] -; CHECK-NEXT: vabdl.u8 q8, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabdl.u8 q0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <8 x i8>, ptr %A %tmp2 = load <8 x i8>, ptr %B %tmp3 = call <8 x i8> @llvm.arm.neon.vabdu.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2) @@ -293,10 +264,8 @@ define <4 x i32> @vabdlu16(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] -; CHECK-NEXT: vabdl.u16 q8, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabdl.u16 q0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <4 x i16>, ptr %A %tmp2 = load <4 x i16>, ptr %B %tmp3 = call <4 x i16> @llvm.arm.neon.vabdu.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2) @@ -309,10 +278,8 @@ define <2 x i64> @vabdlu32(ptr %A, ptr %B) nounwind { ; CHECK: @ %bb.0: ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] -; CHECK-NEXT: vabdl.u32 q8, d17, d16 -; CHECK-NEXT: vmov r0, r1, d16 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: mov pc, lr +; CHECK-NEXT: vabdl.u32 q0, d17, d16 +; CHECK-NEXT: bx lr %tmp1 = load <2 x i32>, ptr %A %tmp2 = load <2 x i32>, ptr %B %tmp3 = call <2 x i32> @llvm.arm.neon.vabdu.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2) -- GitLab From 2f0308ed02ea622b8228d271d9aebe4bd4deacdd Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Fri, 7 Jun 2024 09:42:06 -0400 Subject: [PATCH 248/462] [arm64] Add tan intrinsic lowering (#94545) This change is an implementation of https://github.com/llvm/llvm-project/issues/87367's investigation on supporting IEEE math operations as intrinsics. Which was discussed in this RFC: https://discourse.llvm.org/t/rfc-all-the-math-intrinsics/78294 This PR is just for Tan. Now that x86 tan backend landed: https://github.com/llvm/llvm-project/pull/90503 we can add other backends since the shared pieces are in tree now. Changes: - `llvm/include/llvm/Analysis/VecFuncs.def` - vectorization of tan for arm64 backends. - `llvm/lib/Target/AArch64/AArch64FastISel.cpp` - Add tan to the libcall table - `llvm/lib/Target/AArch64/AArch64ISelLowering.cpp` - Add tan expansion for f128, f16, and vector\neon operations - `llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp` define `G_FTAN` as a legal arm64 instruction resolves #94755 --- llvm/include/llvm/Analysis/VecFuncs.def | 14 ++ llvm/lib/Target/AArch64/AArch64FastISel.cpp | 16 +- .../Target/AArch64/AArch64ISelLowering.cpp | 56 +++-- .../AArch64/GISel/AArch64LegalizerInfo.cpp | 5 +- .../AArch64/GlobalISel/arm64-irtranslator.ll | 8 + .../AArch64/GlobalISel/legalize-tan.mir | 227 ++++++++++++++++++ .../GlobalISel/legalizer-info-validation.mir | 5 +- llvm/test/CodeGen/AArch64/f16-instructions.ll | 26 ++ .../AArch64/fast-isel-runtime-libcall.ll | 24 ++ .../test/CodeGen/AArch64/illegal-float-ops.ll | 22 ++ .../AArch64/replace-with-veclib-armpl.ll | 46 +++- .../replace-with-veclib-sleef-scalable.ll | 23 +- .../AArch64/replace-with-veclib-sleef.ll | 22 +- llvm/test/CodeGen/AArch64/vec-libcalls.ll | 32 +++ .../AArch64/veclib-calls-libsystem-darwin.ll | 144 +++++++++++ .../AArch64/veclib-intrinsic-calls.ll | 75 +++++- 16 files changed, 704 insertions(+), 41 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-tan.mir diff --git a/llvm/include/llvm/Analysis/VecFuncs.def b/llvm/include/llvm/Analysis/VecFuncs.def index 402189769c20..ffdf8b8c3bc7 100644 --- a/llvm/include/llvm/Analysis/VecFuncs.def +++ b/llvm/include/llvm/Analysis/VecFuncs.def @@ -92,6 +92,11 @@ TLI_DEFINE_VECFUNC("llvm.sin.f64", "_simd_sin_d2", FIXED(2), "_ZGV_LLVM_N2v") TLI_DEFINE_VECFUNC("sinf", "_simd_sin_f4", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("llvm.sin.f32", "_simd_sin_f4", FIXED(4), "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("tan", "_simd_tan_d2", FIXED(2), "_ZGV_LLVM_N2v") +TLI_DEFINE_VECFUNC("llvm.tan.f64", "_simd_tan_d2", FIXED(2), "_ZGV_LLVM_N2v") +TLI_DEFINE_VECFUNC("tanf", "_simd_tan_f4", FIXED(4), "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("llvm.tan.f32", "_simd_tan_f4", FIXED(4), "_ZGV_LLVM_N4v") + // Floating-Point Arithmetic and Auxiliary Functions TLI_DEFINE_VECFUNC("cbrt", "_simd_cbrt_d2", FIXED(2), "_ZGV_LLVM_N2v") TLI_DEFINE_VECFUNC("cbrtf", "_simd_cbrt_f4", FIXED(4), "_ZGV_LLVM_N4v") @@ -584,6 +589,7 @@ TLI_DEFINE_VECFUNC("sinpi", "_ZGVnN2v_sinpi", FIXED(2), "_ZGV_LLVM_N2v") TLI_DEFINE_VECFUNC("sqrt", "_ZGVnN2v_sqrt", FIXED(2), "_ZGV_LLVM_N2v") TLI_DEFINE_VECFUNC("tan", "_ZGVnN2v_tan", FIXED(2), "_ZGV_LLVM_N2v") +TLI_DEFINE_VECFUNC("llvm.tan.f64", "_ZGVnN2v_tan", FIXED(2), "_ZGV_LLVM_N2v") TLI_DEFINE_VECFUNC("tanh", "_ZGVnN2v_tanh", FIXED(2), "_ZGV_LLVM_N2v") @@ -681,6 +687,7 @@ TLI_DEFINE_VECFUNC("sinpif", "_ZGVnN4v_sinpif", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("sqrtf", "_ZGVnN4v_sqrtf", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("tanf", "_ZGVnN4v_tanf", FIXED(4), "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("llvm.tan.f32", "_ZGVnN4v_tanf", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("tanhf", "_ZGVnN4v_tanhf", FIXED(4), "_ZGV_LLVM_N4v") @@ -828,6 +835,8 @@ TLI_DEFINE_VECFUNC("sqrtf", "_ZGVsMxv_sqrtf", SCALABLE(4), MASKED, "_ZGVsMxv") TLI_DEFINE_VECFUNC("tan", "_ZGVsMxv_tan", SCALABLE(2), MASKED, "_ZGVsMxv") TLI_DEFINE_VECFUNC("tanf", "_ZGVsMxv_tanf", SCALABLE(4), MASKED, "_ZGVsMxv") +TLI_DEFINE_VECFUNC("llvm.tan.f64", "_ZGVsMxv_tan", SCALABLE(2), MASKED, "_ZGVsMxv") +TLI_DEFINE_VECFUNC("llvm.tan.f32", "_ZGVsMxv_tanf", SCALABLE(4), MASKED, "_ZGVsMxv") TLI_DEFINE_VECFUNC("tanh", "_ZGVsMxv_tanh", SCALABLE(2), MASKED, "_ZGVsMxv") TLI_DEFINE_VECFUNC("tanhf", "_ZGVsMxv_tanhf", SCALABLE(4), MASKED, "_ZGVsMxv") @@ -1087,6 +1096,11 @@ TLI_DEFINE_VECFUNC("tanf", "armpl_vtanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("tan", "armpl_svtan_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv") TLI_DEFINE_VECFUNC("tanf", "armpl_svtan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv") +TLI_DEFINE_VECFUNC("llvm.tan.f64", "armpl_vtanq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v") +TLI_DEFINE_VECFUNC("llvm.tan.f32", "armpl_vtanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("llvm.tan.f64", "armpl_svtan_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv") +TLI_DEFINE_VECFUNC("llvm.tan.f32", "armpl_svtan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv") + TLI_DEFINE_VECFUNC("tanh", "armpl_vtanhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v") TLI_DEFINE_VECFUNC("tanhf", "armpl_vtanhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("tanh", "armpl_svtanh_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv") diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp index 62cf6a2c47ac..56fd15f23363 100644 --- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp +++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp @@ -3534,6 +3534,7 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) { } case Intrinsic::sin: case Intrinsic::cos: + case Intrinsic::tan: case Intrinsic::pow: { MVT RetVT; if (!isTypeLegal(II->getType(), RetVT)) @@ -3542,11 +3543,11 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) { if (RetVT != MVT::f32 && RetVT != MVT::f64) return false; - static const RTLIB::Libcall LibCallTable[3][2] = { - { RTLIB::SIN_F32, RTLIB::SIN_F64 }, - { RTLIB::COS_F32, RTLIB::COS_F64 }, - { RTLIB::POW_F32, RTLIB::POW_F64 } - }; + static const RTLIB::Libcall LibCallTable[4][2] = { + {RTLIB::SIN_F32, RTLIB::SIN_F64}, + {RTLIB::COS_F32, RTLIB::COS_F64}, + {RTLIB::TAN_F32, RTLIB::TAN_F64}, + {RTLIB::POW_F32, RTLIB::POW_F64}}; RTLIB::Libcall LC; bool Is64Bit = RetVT == MVT::f64; switch (II->getIntrinsicID()) { @@ -3558,9 +3559,12 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) { case Intrinsic::cos: LC = LibCallTable[1][Is64Bit]; break; - case Intrinsic::pow: + case Intrinsic::tan: LC = LibCallTable[2][Is64Bit]; break; + case Intrinsic::pow: + LC = LibCallTable[3][Is64Bit]; + break; } ArgListTy Args; diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 360a841bdade..48bf648b0052 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -543,6 +543,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::FSINCOS, MVT::f128, Expand); setOperationAction(ISD::FSQRT, MVT::f128, Expand); setOperationAction(ISD::FSUB, MVT::f128, LibCall); + setOperationAction(ISD::FTAN, MVT::f128, Expand); setOperationAction(ISD::FTRUNC, MVT::f128, Expand); setOperationAction(ISD::SETCC, MVT::f128, Custom); setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom); @@ -727,14 +728,14 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Promote); } - for (auto Op : {ISD::FREM, ISD::FPOW, ISD::FPOWI, - ISD::FCOS, ISD::FSIN, ISD::FSINCOS, - ISD::FEXP, ISD::FEXP2, ISD::FEXP10, - ISD::FLOG, ISD::FLOG2, ISD::FLOG10, - ISD::STRICT_FREM, - ISD::STRICT_FPOW, ISD::STRICT_FPOWI, ISD::STRICT_FCOS, - ISD::STRICT_FSIN, ISD::STRICT_FEXP, ISD::STRICT_FEXP2, - ISD::STRICT_FLOG, ISD::STRICT_FLOG2, ISD::STRICT_FLOG10}) { + for (auto Op : {ISD::FREM, ISD::FPOW, ISD::FPOWI, + ISD::FCOS, ISD::FSIN, ISD::FSINCOS, + ISD::FTAN, ISD::FEXP, ISD::FEXP2, + ISD::FEXP10, ISD::FLOG, ISD::FLOG2, + ISD::FLOG10, ISD::STRICT_FREM, ISD::STRICT_FPOW, + ISD::STRICT_FPOWI, ISD::STRICT_FCOS, ISD::STRICT_FSIN, + ISD::STRICT_FEXP, ISD::STRICT_FEXP2, ISD::STRICT_FLOG, + ISD::STRICT_FLOG2, ISD::STRICT_FLOG10}) { setOperationAction(Op, MVT::f16, Promote); setOperationAction(Op, MVT::v4f16, Expand); setOperationAction(Op, MVT::v8f16, Expand); @@ -1171,26 +1172,27 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, if (Subtarget->isNeonAvailable()) { // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to // silliness like this: + // clang-format off for (auto Op : - {ISD::SELECT, ISD::SELECT_CC, - ISD::BR_CC, ISD::FADD, ISD::FSUB, - ISD::FMUL, ISD::FDIV, ISD::FMA, - ISD::FNEG, ISD::FABS, ISD::FCEIL, - ISD::FSQRT, ISD::FFLOOR, ISD::FNEARBYINT, - ISD::FSIN, ISD::FCOS, ISD::FPOW, - ISD::FLOG, ISD::FLOG2, ISD::FLOG10, - ISD::FEXP, ISD::FEXP2, ISD::FEXP10, - ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN, - ISD::FTRUNC, ISD::FMINNUM, ISD::FMAXNUM, - ISD::FMINIMUM, ISD::FMAXIMUM, ISD::STRICT_FADD, - ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV, - ISD::STRICT_FMA, ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, - ISD::STRICT_FSQRT, ISD::STRICT_FRINT, ISD::STRICT_FNEARBYINT, - ISD::STRICT_FROUND, ISD::STRICT_FTRUNC, ISD::STRICT_FROUNDEVEN, - ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, ISD::STRICT_FMINIMUM, - ISD::STRICT_FMAXIMUM}) + {ISD::SELECT, ISD::SELECT_CC, + ISD::BR_CC, ISD::FADD, ISD::FSUB, + ISD::FMUL, ISD::FDIV, ISD::FMA, + ISD::FNEG, ISD::FABS, ISD::FCEIL, + ISD::FSQRT, ISD::FFLOOR, ISD::FNEARBYINT, + ISD::FSIN, ISD::FCOS, ISD::FTAN, + ISD::FPOW, ISD::FLOG, ISD::FLOG2, + ISD::FLOG10, ISD::FEXP, ISD::FEXP2, + ISD::FEXP10, ISD::FRINT, ISD::FROUND, + ISD::FROUNDEVEN, ISD::FTRUNC, ISD::FMINNUM, + ISD::FMAXNUM, ISD::FMINIMUM, ISD::FMAXIMUM, + ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL, + ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FCEIL, + ISD::STRICT_FFLOOR, ISD::STRICT_FSQRT, ISD::STRICT_FRINT, + ISD::STRICT_FNEARBYINT, ISD::STRICT_FROUND, ISD::STRICT_FTRUNC, + ISD::STRICT_FROUNDEVEN, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, + ISD::STRICT_FMINIMUM, ISD::STRICT_FMAXIMUM}) setOperationAction(Op, MVT::v1f64, Expand); - + // clang-format on for (auto Op : {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_ROUND, ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT, ISD::MUL, @@ -1622,6 +1624,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::FCOS, VT, Expand); setOperationAction(ISD::FSIN, VT, Expand); setOperationAction(ISD::FSINCOS, VT, Expand); + setOperationAction(ISD::FTAN, VT, Expand); setOperationAction(ISD::FEXP, VT, Expand); setOperationAction(ISD::FEXP2, VT, Expand); setOperationAction(ISD::FEXP10, VT, Expand); @@ -1803,6 +1806,7 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) { if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) { setOperationAction(ISD::FSIN, VT, Expand); setOperationAction(ISD::FCOS, VT, Expand); + setOperationAction(ISD::FTAN, VT, Expand); setOperationAction(ISD::FPOW, VT, Expand); setOperationAction(ISD::FLOG, VT, Expand); setOperationAction(ISD::FLOG2, VT, Expand); diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 07a0473888ee..42cd43c3afa3 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -267,9 +267,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) .libcallFor({{s64, s128}}) .minScalarOrElt(1, MinFPScalar); - getActionDefinitionsBuilder( - {G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2, G_FLOG10, - G_FEXP, G_FEXP2, G_FEXP10}) + getActionDefinitionsBuilder({G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2, + G_FLOG10, G_FTAN, G_FEXP, G_FEXP2, G_FEXP10}) // We need a call for these, so we always need to scalarize. .scalarize(0) // Regardless of FP16 support, widen 16-bit elements to 32-bits. diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll index a61931b898ae..eb94cc5d0fb6 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll @@ -2313,6 +2313,14 @@ define float @test_sin_f32(float %x) { ret float %y } +declare float @llvm.tan.f32(float) +define float @test_tan_f32(float %x) { + ; CHECK-LABEL: name: test_tan_f32 + ; CHECK: %{{[0-9]+}}:_(s32) = G_FTAN %{{[0-9]+}} + %y = call float @llvm.tan.f32(float %x) + ret float %y +} + declare float @llvm.sqrt.f32(float) define float @test_sqrt_f32(float %x) { ; CHECK-LABEL: name: test_sqrt_f32 diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-tan.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-tan.mir new file mode 100644 index 000000000000..455aae012836 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-tan.mir @@ -0,0 +1,227 @@ +# RUN: llc -verify-machineinstrs -mtriple aarch64--- \ +# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \ +# RUN: | FileCheck %s +... +--- +name: test_v4f16.tan +alignment: 4 +tracksRegLiveness: true +registers: + - { id: 0, class: _ } + - { id: 1, class: _ } +body: | + bb.0: + liveins: $d0 + ; CHECK-LABEL: name: test_v4f16.tan + ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>) + + ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16) + ; CHECK-NEXT: ADJCALLSTACKDOWN + ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32) + ; CHECK-NEXT: BL &tanf + ; CHECK-NEXT: ADJCALLSTACKUP + ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32) + + ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16) + ; CHECK-NEXT: ADJCALLSTACKDOWN + ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32) + ; CHECK-NEXT: BL &tanf + ; CHECK-NEXT: ADJCALLSTACKUP + ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32) + + ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16) + ; CHECK-NEXT: ADJCALLSTACKDOWN + ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32) + ; CHECK-NEXT: BL &tanf + ; CHECK-NEXT: ADJCALLSTACKUP + ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32) + + ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16) + ; CHECK-NEXT: ADJCALLSTACKDOWN + ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32) + ; CHECK-NEXT: BL &tanf + ; CHECK-NEXT: ADJCALLSTACKUP + ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32) + + ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16) + + %0:_(<4 x s16>) = COPY $d0 + %1:_(<4 x s16>) = G_FTAN %0 + $d0 = COPY %1(<4 x s16>) + RET_ReallyLR implicit $d0 + +... +--- +name: test_v8f16.tan +alignment: 4 +tracksRegLiveness: true +registers: + - { id: 0, class: _ } + - { id: 1, class: _ } +body: | + bb.0: + liveins: $q0 + + ; CHECK-LABEL: name: test_v8f16.tan + + ; This is big, so let's just check for the 8 calls to tanf, the the + ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought + ; to be covered by the other tests. + + ; CHECK: G_UNMERGE_VALUES + ; CHECK: BL &tanf + ; CHECK: BL &tanf + ; CHECK: BL &tanf + ; CHECK: BL &tanf + ; CHECK: BL &tanf + ; CHECK: BL &tanf + ; CHECK: BL &tanf + ; CHECK: BL &tanf + ; CHECK: G_BUILD_VECTOR + + %0:_(<8 x s16>) = COPY $q0 + %1:_(<8 x s16>) = G_FTAN %0 + $q0 = COPY %1(<8 x s16>) + RET_ReallyLR implicit $q0 + +... +--- +name: test_v2f32.tan +alignment: 4 +tracksRegLiveness: true +registers: + - { id: 0, class: _ } + - { id: 1, class: _ } +body: | + bb.0: + liveins: $d0 + + ; CHECK-LABEL: name: test_v2f32.tan + ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>) + + ; CHECK-NEXT: ADJCALLSTACKDOWN + ; CHECK-DAG: $s0 = COPY [[V1]](s32) + ; CHECK-DAG: BL &tanf + ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-DAG: ADJCALLSTACKUP + + ; CHECK-DAG: ADJCALLSTACKDOWN + ; CHECK-DAG: $s0 = COPY [[V2]](s32) + ; CHECK-DAG: BL &tanf + ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-DAG: ADJCALLSTACKUP + + ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32) + + %0:_(<2 x s32>) = COPY $d0 + %1:_(<2 x s32>) = G_FTAN %0 + $d0 = COPY %1(<2 x s32>) + RET_ReallyLR implicit $d0 + +... +--- +name: test_v4f32.tan +alignment: 4 +tracksRegLiveness: true +registers: + - { id: 0, class: _ } + - { id: 1, class: _ } +body: | + bb.0: + liveins: $q0 + ; CHECK-LABEL: name: test_v4f32.tan + ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>) + + ; CHECK-NEXT: ADJCALLSTACKDOWN + ; CHECK-DAG: $s0 = COPY [[V1]](s32) + ; CHECK-DAG: BL &tanf + ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-DAG: ADJCALLSTACKUP + + ; CHECK-DAG: ADJCALLSTACKDOWN + ; CHECK-DAG: $s0 = COPY [[V2]](s32) + ; CHECK-DAG: BL &tanf + ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-DAG: ADJCALLSTACKUP + + ; CHECK-DAG: ADJCALLSTACKDOWN + ; CHECK-DAG: $s0 = COPY [[V3]](s32) + ; CHECK-DAG: BL &tanf + ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-DAG: ADJCALLSTACKUP + + ; CHECK-DAG: ADJCALLSTACKDOWN + ; CHECK-DAG: $s0 = COPY [[V4]](s32) + ; CHECK-DAG: BL &tanf + ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-DAG: ADJCALLSTACKUP + + ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32) + + %0:_(<4 x s32>) = COPY $q0 + %1:_(<4 x s32>) = G_FTAN %0 + $q0 = COPY %1(<4 x s32>) + RET_ReallyLR implicit $q0 + +... +--- +name: test_v2f64.tan +alignment: 4 +tracksRegLiveness: true +registers: + - { id: 0, class: _ } + - { id: 1, class: _ } +body: | + bb.0: + liveins: $q0 + + ; CHECK-LABEL: name: test_v2f64.tan + ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>) + + ; CHECK-NEXT: ADJCALLSTACKDOWN + ; CHECK-DAG: $d0 = COPY [[V1]](s64) + ; CHECK-DAG: BL &tan + ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0 + ; CHECK-DAG: ADJCALLSTACKUP + + ; CHECK-DAG: ADJCALLSTACKDOWN + ; CHECK-DAG: $d0 = COPY [[V2]](s64) + ; CHECK-DAG: BL &tan + ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0 + ; CHECK-DAG: ADJCALLSTACKUP + + ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64) + + %0:_(<2 x s64>) = COPY $q0 + %1:_(<2 x s64>) = G_FTAN %0 + $q0 = COPY %1(<2 x s64>) + RET_ReallyLR implicit $q0 + +... +--- +name: test_tan_half +alignment: 4 +tracksRegLiveness: true +registers: + - { id: 0, class: _ } + - { id: 1, class: _ } +body: | + bb.0: + liveins: $h0 + ; CHECK-LABEL: name: test_tan_half + ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16) + ; CHECK-NEXT: ADJCALLSTACKDOWN + ; CHECK-NEXT: $s0 = COPY [[REG1]](s32) + ; CHECK-NEXT: BL &tanf + ; CHECK-NEXT: ADJCALLSTACKUP + ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32) + + %0:_(s16) = COPY $h0 + %1:_(s16) = G_FTAN %0 + $h0 = COPY %1(s16) + RET_ReallyLR implicit $h0 diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index d71111b57efe..04c9a08c5003 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -675,8 +675,9 @@ # DEBUG-NEXT: .. the first uncovered type index: 1, OK # DEBUG-NEXT: .. the first uncovered imm index: 0, OK # DEBUG-NEXT: G_FTAN (opcode {{[0-9]+}}): 1 type index, 0 imm indices -# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined -# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}} +# DEBUG-NEXT: .. the first uncovered type index: 1, OK +# DEBUG-NEXT: .. the first uncovered imm index: 0, OK # DEBUG-NEXT: G_FSQRT (opcode {{[0-9]+}}): 1 type index, 0 imm indices # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}} # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected diff --git a/llvm/test/CodeGen/AArch64/f16-instructions.ll b/llvm/test/CodeGen/AArch64/f16-instructions.ll index fa362ae798aa..998f7dd38c3f 100644 --- a/llvm/test/CodeGen/AArch64/f16-instructions.ll +++ b/llvm/test/CodeGen/AArch64/f16-instructions.ll @@ -759,6 +759,7 @@ declare half @llvm.sqrt.f16(half %a) #0 declare half @llvm.powi.f16.i32(half %a, i32 %b) #0 declare half @llvm.sin.f16(half %a) #0 declare half @llvm.cos.f16(half %a) #0 +declare half @llvm.tan.f16(half %a) #0 declare half @llvm.pow.f16(half %a, half %b) #0 declare half @llvm.exp.f16(half %a) #0 declare half @llvm.exp2.f16(half %a) #0 @@ -870,6 +871,31 @@ define half @test_cos(half %a) #0 { ret half %r } +; FALLBACK-NOT: remark:{{.*}}test_tan +; FALLBACK-FP16-NOT: remark:{{.*}}test_tan + +; CHECK-COMMON-LABEL: test_tan: +; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]! +; CHECK-COMMON-NEXT: mov x29, sp +; CHECK-COMMON-NEXT: fcvt s0, h0 +; CHECK-COMMON-NEXT: bl {{_?}}tanf +; CHECK-COMMON-NEXT: fcvt h0, s0 +; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16 +; CHECK-COMMON-NEXT: ret + +; GISEL-LABEL: test_tan: +; GISEL-NEXT: stp x29, x30, [sp, #-16]! +; GISEL-NEXT: mov x29, sp +; GISEL-NEXT: fcvt s0, h0 +; GISEL-NEXT: bl {{_?}}tanf +; GISEL-NEXT: fcvt h0, s0 +; GISEL-NEXT: ldp x29, x30, [sp], #16 +; GISEL-NEXT: ret +define half @test_tan(half %a) #0 { + %r = call half @llvm.tan.f16(half %a) + ret half %r +} + ; CHECK-COMMON-LABEL: test_pow: ; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]! ; CHECK-COMMON-NEXT: mov x29, sp diff --git a/llvm/test/CodeGen/AArch64/fast-isel-runtime-libcall.ll b/llvm/test/CodeGen/AArch64/fast-isel-runtime-libcall.ll index 16bab1565e7f..7ff4dfca3f4c 100644 --- a/llvm/test/CodeGen/AArch64/fast-isel-runtime-libcall.ll +++ b/llvm/test/CodeGen/AArch64/fast-isel-runtime-libcall.ll @@ -67,6 +67,28 @@ define double @cos_f64(double %a) { ret double %1 } +define float @tan_f32(float %a) { +; SMALL-LABEL: tan_f32 +; SMALL: bl _tanf +; LARGE-LABEL: tan_f32 +; LARGE: adrp [[REG:x[0-9]+]], _tanf@GOTPAGE +; LARGE: ldr [[REG]], [[[REG]], _tanf@GOTPAGEOFF] +; LARGE-NEXT: blr [[REG]] + %1 = call float @llvm.tan.f32(float %a) + ret float %1 +} + +define double @tan_f64(double %a) { +; SMALL-LABEL: tan_f64 +; SMALL: bl _tan +; LARGE-LABEL: tan_f64 +; LARGE: adrp [[REG:x[0-9]+]], _tan@GOTPAGE +; LARGE: ldr [[REG]], [[[REG]], _tan@GOTPAGEOFF] +; LARGE-NEXT: blr [[REG]] + %1 = call double @llvm.tan.f64(double %a) + ret double %1 +} + define float @pow_f32(float %a, float %b) { ; SMALL-LABEL: pow_f32 ; SMALL: bl _powf @@ -92,5 +114,7 @@ declare float @llvm.sin.f32(float) declare double @llvm.sin.f64(double) declare float @llvm.cos.f32(float) declare double @llvm.cos.f64(double) +declare float @llvm.tan.f32(float) +declare double @llvm.tan.f64(double) declare float @llvm.pow.f32(float, float) declare double @llvm.pow.f64(double, double) diff --git a/llvm/test/CodeGen/AArch64/illegal-float-ops.ll b/llvm/test/CodeGen/AArch64/illegal-float-ops.ll index 20435e10d89f..3281a9876779 100644 --- a/llvm/test/CodeGen/AArch64/illegal-float-ops.ll +++ b/llvm/test/CodeGen/AArch64/illegal-float-ops.ll @@ -159,6 +159,28 @@ define void @test_sin(float %float, double %double, fp128 %fp128) { } +declare float @llvm.tan.f32(float) +declare double @llvm.tan.f64(double) +declare fp128 @llvm.tan.f128(fp128) + +define void @test_tan(float %float, double %double, fp128 %fp128) { +; CHECK-LABEL: test_tan: + + %tanfloat = call float @llvm.tan.f32(float %float) + store float %tanfloat, ptr @varfloat +; CHECK: bl tanf + + %tandouble = call double @llvm.tan.f64(double %double) + store double %tandouble, ptr @vardouble +; CHECK: bl tan + + %tanfp128 = call fp128 @llvm.tan.f128(fp128 %fp128) + store fp128 %tanfp128, ptr @varfp128 +; CHECK: bl tanl + ret void + +} + declare float @llvm.pow.f32(float, float) declare double @llvm.pow.f64(double, double) declare fp128 @llvm.pow.f128(fp128, fp128) diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll index 758df0493cc5..ce0c5572577d 100644 --- a/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll +++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll @@ -15,7 +15,7 @@ declare @llvm.cos.nxv2f64() declare @llvm.cos.nxv4f32() ;. -; CHECK: @llvm.compiler.used = appending global [36 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_svcos_f64_x, ptr @armpl_svcos_f32_x, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_svexp_f64_x, ptr @armpl_svexp_f32_x, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_svexp10_f64_x, ptr @armpl_svexp10_f32_x, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_svexp2_f64_x, ptr @armpl_svexp2_f32_x, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_svlog_f64_x, ptr @armpl_svlog_f32_x, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32, ptr @armpl_svlog10_f64_x, ptr @armpl_svlog10_f32_x, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_svlog2_f64_x, ptr @armpl_svlog2_f32_x, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_svsin_f64_x, ptr @armpl_svsin_f32_x, ptr @armpl_vfmodq_f64, ptr @armpl_vfmodq_f32, ptr @armpl_svfmod_f64_x, ptr @armpl_svfmod_f32_x], section "llvm.metadata" +; CHECK: @llvm.compiler.used = appending global [40 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_svcos_f64_x, ptr @armpl_svcos_f32_x, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_svexp_f64_x, ptr @armpl_svexp_f32_x, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_svexp10_f64_x, ptr @armpl_svexp10_f32_x, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_svexp2_f64_x, ptr @armpl_svexp2_f32_x, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_svlog_f64_x, ptr @armpl_svlog_f32_x, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32, ptr @armpl_svlog10_f64_x, ptr @armpl_svlog10_f32_x, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_svlog2_f64_x, ptr @armpl_svlog2_f32_x, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_svsin_f64_x, ptr @armpl_svsin_f32_x, ptr @armpl_vtanq_f64, ptr @armpl_vtanq_f32, ptr @armpl_svtan_f64_x, ptr @armpl_svtan_f32_x, ptr @armpl_vfmodq_f64, ptr @armpl_vfmodq_f32, ptr @armpl_svfmod_f64_x, ptr @armpl_svfmod_f32_x], section "llvm.metadata" ;. define <2 x double> @llvm_cos_f64(<2 x double> %in) { ; CHECK-LABEL: define <2 x double> @llvm_cos_f64 @@ -424,6 +424,50 @@ define @llvm_sin_vscale_f32( %in) #0 { ret %1 } +declare <2 x double> @llvm.tan.v2f64(<2 x double>) +declare <4 x float> @llvm.tan.v4f32(<4 x float>) +declare @llvm.tan.nxv2f64() +declare @llvm.tan.nxv4f32() + +define <2 x double> @llvm_tan_f64(<2 x double> %in) { +; CHECK-LABEL: define <2 x double> @llvm_tan_f64 +; CHECK-SAME: (<2 x double> [[IN:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = call fast <2 x double> @armpl_vtanq_f64(<2 x double> [[IN]]) +; CHECK-NEXT: ret <2 x double> [[TMP1]] +; + %1 = call fast <2 x double> @llvm.tan.v2f64(<2 x double> %in) + ret <2 x double> %1 +} + +define <4 x float> @llvm_tan_f32(<4 x float> %in) { +; CHECK-LABEL: define <4 x float> @llvm_tan_f32 +; CHECK-SAME: (<4 x float> [[IN:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @armpl_vtanq_f32(<4 x float> [[IN]]) +; CHECK-NEXT: ret <4 x float> [[TMP1]] +; + %1 = call fast <4 x float> @llvm.tan.v4f32(<4 x float> %in) + ret <4 x float> %1 +} + +define @llvm_tan_vscale_f64( %in) #0 { +; CHECK-LABEL: define @llvm_tan_vscale_f64 +; CHECK-SAME: ( [[IN:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[TMP1:%.*]] = call fast @armpl_svtan_f64_x( [[IN]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) +; CHECK-NEXT: ret [[TMP1]] +; + %1 = call fast @llvm.tan.nxv2f64( %in) + ret %1 +} + +define @llvm_tan_vscale_f32( %in) #0 { +; CHECK-LABEL: define @llvm_tan_vscale_f32 +; CHECK-SAME: ( [[IN:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[TMP1:%.*]] = call fast @armpl_svtan_f32_x( [[IN]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) +; CHECK-NEXT: ret [[TMP1]] +; + %1 = call fast @llvm.tan.nxv4f32( %in) + ret %1 +} define <2 x double> @frem_f64(<2 x double> %in) { ; CHECK-LABEL: define <2 x double> @frem_f64 diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll index 9e214655e413..c58a9bc8342d 100644 --- a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll +++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll @@ -4,7 +4,7 @@ target triple = "aarch64-unknown-linux-gnu" ;. -; CHECK: @llvm.compiler.used = appending global [18 x ptr] [ptr @_ZGVsMxv_cos, ptr @_ZGVsMxv_cosf, ptr @_ZGVsMxv_exp, ptr @_ZGVsMxv_expf, ptr @_ZGVsMxv_exp10, ptr @_ZGVsMxv_exp10f, ptr @_ZGVsMxv_exp2, ptr @_ZGVsMxv_exp2f, ptr @_ZGVsMxv_log, ptr @_ZGVsMxv_logf, ptr @_ZGVsMxv_log10, ptr @_ZGVsMxv_log10f, ptr @_ZGVsMxv_log2, ptr @_ZGVsMxv_log2f, ptr @_ZGVsMxv_sin, ptr @_ZGVsMxv_sinf, ptr @_ZGVsMxvv_fmod, ptr @_ZGVsMxvv_fmodf], section "llvm.metadata" +; CHECK: @llvm.compiler.used = appending global [20 x ptr] [ptr @_ZGVsMxv_cos, ptr @_ZGVsMxv_cosf, ptr @_ZGVsMxv_exp, ptr @_ZGVsMxv_expf, ptr @_ZGVsMxv_exp10, ptr @_ZGVsMxv_exp10f, ptr @_ZGVsMxv_exp2, ptr @_ZGVsMxv_exp2f, ptr @_ZGVsMxv_log, ptr @_ZGVsMxv_logf, ptr @_ZGVsMxv_log10, ptr @_ZGVsMxv_log10f, ptr @_ZGVsMxv_log2, ptr @_ZGVsMxv_log2f, ptr @_ZGVsMxv_sin, ptr @_ZGVsMxv_sinf, ptr @_ZGVsMxv_tan, ptr @_ZGVsMxv_tanf, ptr @_ZGVsMxvv_fmod, ptr @_ZGVsMxvv_fmodf], section "llvm.metadata" ;. define @llvm_ceil_vscale_f64( %in) { ; CHECK-LABEL: @llvm_ceil_vscale_f64( @@ -366,6 +366,25 @@ define @llvm_sqrt_vscale_f32( %in) { ret %1 } +define @llvm_tan_vscale_f64( %in) { +; CHECK-LABEL: @llvm_tan_vscale_f64( +; CHECK-NEXT: [[TMP1:%.*]] = call fast @_ZGVsMxv_tan( [[IN:%.*]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) +; CHECK-NEXT: ret [[TMP1]] +; + %1 = call fast @llvm.tan.nxv2f64( %in) + ret %1 +} + +define @llvm_tan_vscale_f32( %in) { +; CHECK-LABEL: @llvm_tan_vscale_f32( +; CHECK-NEXT: [[TMP1:%.*]] = call fast @_ZGVsMxv_tanf( [[IN:%.*]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) +; CHECK-NEXT: ret [[TMP1]] +; + %1 = call fast @llvm.tan.nxv4f32( %in) + ret %1 +} + + define @llvm_trunc_vscale_f64( %in) { ; CHECK-LABEL: @llvm_trunc_vscale_f64( ; CHECK-NEXT: [[TMP1:%.*]] = call fast @llvm.trunc.nxv2f64( [[IN:%.*]]) @@ -442,6 +461,8 @@ declare @llvm.sin.nxv2f64() declare @llvm.sin.nxv4f32() declare @llvm.sqrt.nxv2f64() declare @llvm.sqrt.nxv4f32() +declare @llvm.tan.nxv2f64() +declare @llvm.tan.nxv4f32() declare @llvm.trunc.nxv2f64() declare @llvm.trunc.nxv4f32() ;. diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll index f408df570fdc..1df2caed4244 100644 --- a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll +++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll @@ -4,7 +4,7 @@ target triple = "aarch64-unknown-linux-gnu" ;. -; CHECK: @llvm.compiler.used = appending global [18 x ptr] [ptr @_ZGVnN2v_cos, ptr @_ZGVnN4v_cosf, ptr @_ZGVnN2v_exp, ptr @_ZGVnN4v_expf, ptr @_ZGVnN2v_exp10, ptr @_ZGVnN4v_exp10f, ptr @_ZGVnN2v_exp2, ptr @_ZGVnN4v_exp2f, ptr @_ZGVnN2v_log, ptr @_ZGVnN4v_logf, ptr @_ZGVnN2v_log10, ptr @_ZGVnN4v_log10f, ptr @_ZGVnN2v_log2, ptr @_ZGVnN4v_log2f, ptr @_ZGVnN2v_sin, ptr @_ZGVnN4v_sinf, ptr @_ZGVnN2vv_fmod, ptr @_ZGVnN4vv_fmodf], section "llvm.metadata" +; CHECK: @llvm.compiler.used = appending global [20 x ptr] [ptr @_ZGVnN2v_cos, ptr @_ZGVnN4v_cosf, ptr @_ZGVnN2v_exp, ptr @_ZGVnN4v_expf, ptr @_ZGVnN2v_exp10, ptr @_ZGVnN4v_exp10f, ptr @_ZGVnN2v_exp2, ptr @_ZGVnN4v_exp2f, ptr @_ZGVnN2v_log, ptr @_ZGVnN4v_logf, ptr @_ZGVnN2v_log10, ptr @_ZGVnN4v_log10f, ptr @_ZGVnN2v_log2, ptr @_ZGVnN4v_log2f, ptr @_ZGVnN2v_sin, ptr @_ZGVnN4v_sinf, ptr @_ZGVnN2v_tan, ptr @_ZGVnN4v_tanf, ptr @_ZGVnN2vv_fmod, ptr @_ZGVnN4vv_fmodf], section "llvm.metadata" ;. define <2 x double> @llvm_ceil_f64(<2 x double> %in) { ; CHECK-LABEL: @llvm_ceil_f64( @@ -366,6 +366,24 @@ define <4 x float> @llvm_sqrt_f32(<4 x float> %in) { ret <4 x float> %1 } +define <2 x double> @llvm_tan_f64(<2 x double> %in) { +; CHECK-LABEL: @llvm_tan_f64( +; CHECK-NEXT: [[TMP1:%.*]] = call fast <2 x double> @_ZGVnN2v_tan(<2 x double> [[IN:%.*]]) +; CHECK-NEXT: ret <2 x double> [[TMP1]] +; + %1 = call fast <2 x double> @llvm.tan.v2f64(<2 x double> %in) + ret <2 x double> %1 +} + +define <4 x float> @llvm_tan_f32(<4 x float> %in) { +; CHECK-LABEL: @llvm_tan_f32( +; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @_ZGVnN4v_tanf(<4 x float> [[IN:%.*]]) +; CHECK-NEXT: ret <4 x float> [[TMP1]] +; + %1 = call fast <4 x float> @llvm.tan.v4f32(<4 x float> %in) + ret <4 x float> %1 +} + define <2 x double> @llvm_trunc_f64(<2 x double> %in) { ; CHECK-LABEL: @llvm_trunc_f64( ; CHECK-NEXT: [[TMP1:%.*]] = call fast <2 x double> @llvm.trunc.v2f64(<2 x double> [[IN:%.*]]) @@ -442,6 +460,8 @@ declare <2 x double> @llvm.sin.v2f64(<2 x double>) declare <4 x float> @llvm.sin.v4f32(<4 x float>) declare <2 x double> @llvm.sqrt.v2f64(<2 x double>) declare <4 x float> @llvm.sqrt.v4f32(<4 x float>) +declare <2 x double> @llvm.tan.v2f64(<2 x double>) +declare <4 x float> @llvm.tan.v4f32(<4 x float>) declare <2 x double> @llvm.trunc.v2f64(<2 x double>) declare <4 x float> @llvm.trunc.v4f32(<4 x float>) ;. diff --git a/llvm/test/CodeGen/AArch64/vec-libcalls.ll b/llvm/test/CodeGen/AArch64/vec-libcalls.ll index aa2cca40a92c..9bbac5d69c28 100644 --- a/llvm/test/CodeGen/AArch64/vec-libcalls.ll +++ b/llvm/test/CodeGen/AArch64/vec-libcalls.ll @@ -20,6 +20,7 @@ declare <3 x double> @llvm.sin.v3f64(<3 x double>) declare <3 x float> @llvm.fabs.v3f32(<3 x float>) declare <3 x float> @llvm.ceil.v3f32(<3 x float>) declare <3 x float> @llvm.cos.v3f32(<3 x float>) +declare <3 x float> @llvm.tan.v3f32(<3 x float>) declare <3 x float> @llvm.exp.v3f32(<3 x float>) declare <3 x float> @llvm.exp2.v3f32(<3 x float>) declare <3 x float> @llvm.floor.v3f32(<3 x float>) @@ -297,6 +298,37 @@ define <3 x float> @cos_v3f32(<3 x float> %x) nounwind { ret <3 x float> %r } +define <3 x float> @tan_v3f32(<3 x float> %x) nounwind { +; CHECK-LABEL: tan_v3f32: +; CHECK: // %bb.0: +; CHECK-NEXT: sub sp, sp, #48 +; CHECK-NEXT: str q0, [sp, #16] // 16-byte Folded Spill +; CHECK-NEXT: mov s0, v0.s[1] +; CHECK-NEXT: str x30, [sp, #32] // 8-byte Folded Spill +; CHECK-NEXT: bl tanf +; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0 +; CHECK-NEXT: str q0, [sp] // 16-byte Folded Spill +; CHECK-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload +; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0 +; CHECK-NEXT: bl tanf +; CHECK-NEXT: ldr q1, [sp] // 16-byte Folded Reload +; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0 +; CHECK-NEXT: mov v0.s[1], v1.s[0] +; CHECK-NEXT: str q0, [sp] // 16-byte Folded Spill +; CHECK-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload +; CHECK-NEXT: mov s0, v0.s[2] +; CHECK-NEXT: bl tanf +; CHECK-NEXT: ldr q1, [sp] // 16-byte Folded Reload +; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0 +; CHECK-NEXT: ldr x30, [sp, #32] // 8-byte Folded Reload +; CHECK-NEXT: mov v1.s[2], v0.s[0] +; CHECK-NEXT: mov v0.16b, v1.16b +; CHECK-NEXT: add sp, sp, #48 +; CHECK-NEXT: ret + %r = call <3 x float> @llvm.tan.v3f32(<3 x float> %x) + ret <3 x float> %r +} + define <3 x float> @exp_v3f32(<3 x float> %x) nounwind { ; CHECK-LABEL: exp_v3f32: ; CHECK: // %bb.0: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll index 2017797288c2..89a513515b20 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-calls-libsystem-darwin.ll @@ -291,6 +291,150 @@ for.end: ret void } +declare float @sinf(float) nounwind readnone +define void @sinf_v4f32(i64 %n, ptr noalias %y, ptr noalias %x) { +; CHECK-LABEL: @sinf_v4f32( +; CHECK: call <4 x float> @_simd_sin_f4( +; CHECK: ret void + +entry: + br label %for.body + +for.body: + %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ] + %gep.y = getelementptr inbounds float, ptr %y, i64 %iv + %lv = load float, ptr %gep.y, align 4 + %call = tail call float @sinf(float %lv) + %gep.x = getelementptr inbounds float, ptr %x, i64 %iv + store float %call, ptr %gep.x, align 4 + %iv.next = add i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + +declare double @sin(double) nounwind readnone +define void @sin_v2f64(i64 %n, ptr noalias %y, ptr noalias %x) { +; CHECK-LABEL: @sin_v2f64( +; CHECK: call <2 x double> @_simd_sin_d2( +; CHECK: ret void + +entry: + br label %for.body + +for.body: + %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ] + %gep.y = getelementptr inbounds double, ptr %y, i64 %iv + %lv = load double, ptr %gep.y, align 4 + %call = tail call double @sin(double %lv) + %gep.x = getelementptr inbounds double, ptr %x, i64 %iv + store double %call, ptr %gep.x, align 4 + %iv.next = add i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + +declare float @tanf(float) nounwind readnone +define void @tanf_v4f32(i64 %n, ptr noalias %y, ptr noalias %x) { +; CHECK-LABEL: @tanf_v4f32( +; CHECK: call <4 x float> @_simd_tan_f4( +; CHECK: ret void + +entry: + br label %for.body + +for.body: + %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ] + %gep.y = getelementptr inbounds float, ptr %y, i64 %iv + %lv = load float, ptr %gep.y, align 4 + %call = tail call float @tanf(float %lv) + %gep.x = getelementptr inbounds float, ptr %x, i64 %iv + store float %call, ptr %gep.x, align 4 + %iv.next = add i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + +declare double @tan(double) nounwind readnone +define void @tan_v2f64(i64 %n, ptr noalias %y, ptr noalias %x) { +; CHECK-LABEL: @tan_v2f64( +; CHECK: call <2 x double> @_simd_tan_d2( +; CHECK: ret void + +entry: + br label %for.body + +for.body: + %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ] + %gep.y = getelementptr inbounds double, ptr %y, i64 %iv + %lv = load double, ptr %gep.y, align 4 + %call = tail call double @tan(double %lv) + %gep.x = getelementptr inbounds double, ptr %x, i64 %iv + store double %call, ptr %gep.x, align 4 + %iv.next = add i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + +declare float @llvm.tan.f32(float) nounwind readnone +define void @tan_v4f32_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) { +; CHECK-LABEL: @tan_v4f32_intrinsic( +; CHECK: call <4 x float> @_simd_tan_f4(<4 x float> +; CHECK: ret void + +entry: + br label %for.body + +for.body: + %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ] + %gep.y = getelementptr inbounds float, ptr %y, i64 %iv + %lv = load float, ptr %gep.y, align 4 + %call = tail call float @llvm.tan.f32(float %lv) + %gep.x = getelementptr inbounds float, ptr %x, i64 %iv + store float %call, ptr %gep.x, align 4 + %iv.next = add i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + +declare double @llvm.tan.f64(double) nounwind readnone +define void @tan_v2f64_intrinsic(i64 %n, ptr noalias %y, ptr noalias %x) { +; CHECK-LABEL: @tan_v2f64_intrinsic( +; CHECK: call <2 x double> @_simd_tan_d2(<2 x double> +; CHECK: ret void + +entry: + br label %for.body + +for.body: + %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ] + %gep.y = getelementptr inbounds double, ptr %y, i64 %iv + %lv = load double, ptr %gep.y, align 4 + %call = tail call double @llvm.tan.f64(double %lv) + %gep.x = getelementptr inbounds double, ptr %x, i64 %iv + store double %call, ptr %gep.x, align 4 + %iv.next = add i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} + declare float @cbrtf(float) nounwind readnone define void @cbrtf_v4f32(i64 %n, ptr noalias %y, ptr noalias %x) { ; CHECK-LABEL: @cbrtf_v4f32( diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll index 2a552077a42b..f0ae0093b264 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll @@ -1,4 +1,4 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call.*(cos|exp|log|sin|pow|ceil|copysign|fabs|floor|fma|m..num|nearbyint|rint|round|sqrt|trunc)" --version 2 +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call.*(cos|exp|log|sin|pow|ceil|copysign|fabs|floor|fma|m..num|nearbyint|rint|round|sqrt|tan|trunc)" --version 2 ; RUN: opt -mattr=+neon -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=SLEEF-NEON ; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s | FileCheck %s --check-prefix=SLEEF-SVE @@ -1472,6 +1472,79 @@ define void @sqrt_f32(ptr noalias %in.ptr, ptr %out.ptr) { ret void } +declare double @llvm.tan.f64(double) +declare float @llvm.tan.f32(float) + +define void @tan_f64(ptr noalias %in.ptr, ptr %out.ptr) { +; SLEEF-NEON-LABEL: define void @tan_f64 +; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] { +; SLEEF-NEON: [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_tan(<2 x double> [[WIDE_LOAD:%.*]]) +; +; SLEEF-SVE-LABEL: define void @tan_f64 +; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] { +; SLEEF-SVE: [[TMP15:%.*]] = call @_ZGVsMxv_tan( [[WIDE_MASKED_LOAD:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +; ARMPL-NEON-LABEL: define void @tan_f64 +; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] { +; ARMPL-NEON: [[TMP3:%.*]] = call <2 x double> @armpl_vtanq_f64(<2 x double> [[WIDE_LOAD:%.*]]) +; +; ARMPL-SVE-LABEL: define void @tan_f64 +; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] { +; ARMPL-SVE: [[TMP15:%.*]] = call @armpl_svtan_f64_x( [[WIDE_MASKED_LOAD:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; + entry: + br label %for.body + + for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %in.gep = getelementptr inbounds double, ptr %in.ptr, i64 %iv + %in = load double, ptr %in.gep, align 8 + %call = tail call double @llvm.tan.f64(double %in) + %out.gep = getelementptr inbounds double, ptr %out.ptr, i64 %iv + store double %call, ptr %out.gep, align 8 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body + + for.end: + ret void +} + +define void @tan_f32(ptr noalias %in.ptr, ptr %out.ptr) { +; SLEEF-NEON-LABEL: define void @tan_f32 +; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] { +; SLEEF-NEON: [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_tanf(<4 x float> [[WIDE_LOAD:%.*]]) +; +; SLEEF-SVE-LABEL: define void @tan_f32 +; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] { +; SLEEF-SVE: [[TMP15:%.*]] = call @_ZGVsMxv_tanf( [[WIDE_MASKED_LOAD:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +; ARMPL-NEON-LABEL: define void @tan_f32 +; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] { +; ARMPL-NEON: [[TMP3:%.*]] = call <4 x float> @armpl_vtanq_f32(<4 x float> [[WIDE_LOAD:%.*]]) +; +; ARMPL-SVE-LABEL: define void @tan_f32 +; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] { +; ARMPL-SVE: [[TMP15:%.*]] = call @armpl_svtan_f32_x( [[WIDE_MASKED_LOAD:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; + entry: + br label %for.body + + for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %in.gep = getelementptr inbounds float, ptr %in.ptr, i64 %iv + %in = load float, ptr %in.gep, align 8 + %call = tail call float @llvm.tan.f32(float %in) + %out.gep = getelementptr inbounds float, ptr %out.ptr, i64 %iv + store float %call, ptr %out.gep, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %for.end, label %for.body + + for.end: + ret void +} + declare double @llvm.trunc.f64(double) declare float @llvm.trunc.f32(float) -- GitLab From c5fcc2ea55372060760b0ba46d36d03ed39825d5 Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 7 Jun 2024 14:42:22 +0100 Subject: [PATCH 249/462] [AArch64] Add addp from shuffles tests. NFC --- llvm/test/CodeGen/AArch64/addp-shuffle.ll | 224 ++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 llvm/test/CodeGen/AArch64/addp-shuffle.ll diff --git a/llvm/test/CodeGen/AArch64/addp-shuffle.ll b/llvm/test/CodeGen/AArch64/addp-shuffle.ll new file mode 100644 index 000000000000..c15a84c7b3a2 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/addp-shuffle.ll @@ -0,0 +1,224 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16 +; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-FP16 + +define <4 x i32> @deinterleave_shuffle_v8i32(<8 x i32> %a) { +; CHECK-LABEL: deinterleave_shuffle_v8i32: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 v2.4s, v0.4s, v1.4s +; CHECK-NEXT: uzp2 v0.4s, v0.4s, v1.4s +; CHECK-NEXT: add v0.4s, v2.4s, v0.4s +; CHECK-NEXT: ret + %r0 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> + %r1 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> + %o = add <4 x i32> %r0, %r1 + ret <4 x i32> %o +} + +define <4 x i32> @deinterleave_shuffle_v8i32_c(<8 x i32> %a) { +; CHECK-LABEL: deinterleave_shuffle_v8i32_c: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 v2.4s, v0.4s, v1.4s +; CHECK-NEXT: uzp2 v0.4s, v0.4s, v1.4s +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: ret + %r0 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> + %r1 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> + %o = add <4 x i32> %r1, %r0 + ret <4 x i32> %o +} + +define <2 x i32> @deinterleave_shuffle_v4i32(<4 x i32> %a) { +; CHECK-LABEL: deinterleave_shuffle_v4i32: +; CHECK: // %bb.0: +; CHECK-NEXT: ext v1.16b, v0.16b, v0.16b, #8 +; CHECK-NEXT: zip1 v2.2s, v0.2s, v1.2s +; CHECK-NEXT: zip2 v0.2s, v0.2s, v1.2s +; CHECK-NEXT: add v0.2s, v2.2s, v0.2s +; CHECK-NEXT: ret + %r0 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> + %r1 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> + %o = add <2 x i32> %r0, %r1 + ret <2 x i32> %o +} + +define <8 x i16> @deinterleave_shuffle_v16i16(<16 x i16> %a) { +; CHECK-LABEL: deinterleave_shuffle_v16i16: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 v2.8h, v0.8h, v1.8h +; CHECK-NEXT: uzp2 v0.8h, v0.8h, v1.8h +; CHECK-NEXT: add v0.8h, v2.8h, v0.8h +; CHECK-NEXT: ret + %r0 = shufflevector <16 x i16> %a, <16 x i16> poison, <8 x i32> + %r1 = shufflevector <16 x i16> %a, <16 x i16> poison, <8 x i32> + %o = add <8 x i16> %r0, %r1 + ret <8 x i16> %o +} + +define <16 x i8> @deinterleave_shuffle_v32i8(<32 x i8> %a) { +; CHECK-LABEL: deinterleave_shuffle_v32i8: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 v2.16b, v0.16b, v1.16b +; CHECK-NEXT: uzp2 v0.16b, v0.16b, v1.16b +; CHECK-NEXT: add v0.16b, v2.16b, v0.16b +; CHECK-NEXT: ret + %r0 = shufflevector <32 x i8> %a, <32 x i8> poison, <16 x i32> + %r1 = shufflevector <32 x i8> %a, <32 x i8> poison, <16 x i32> + %o = add <16 x i8> %r0, %r1 + ret <16 x i8> %o +} + +define <4 x i64> @deinterleave_shuffle_v8i64(<8 x i64> %a) { +; CHECK-LABEL: deinterleave_shuffle_v8i64: +; CHECK: // %bb.0: +; CHECK-NEXT: zip1 v4.2d, v2.2d, v3.2d +; CHECK-NEXT: zip1 v5.2d, v0.2d, v1.2d +; CHECK-NEXT: zip2 v2.2d, v2.2d, v3.2d +; CHECK-NEXT: zip2 v0.2d, v0.2d, v1.2d +; CHECK-NEXT: add v1.2d, v4.2d, v2.2d +; CHECK-NEXT: add v0.2d, v5.2d, v0.2d +; CHECK-NEXT: ret + %r0 = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> + %r1 = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> + %o = add <4 x i64> %r0, %r1 + ret <4 x i64> %o +} + +define <4 x float> @deinterleave_shuffle_v8f32(<8 x float> %a) { +; CHECK-LABEL: deinterleave_shuffle_v8f32: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 v2.4s, v0.4s, v1.4s +; CHECK-NEXT: uzp2 v0.4s, v0.4s, v1.4s +; CHECK-NEXT: fadd v0.4s, v2.4s, v0.4s +; CHECK-NEXT: ret + %r0 = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> + %r1 = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> + %o = fadd <4 x float> %r0, %r1 + ret <4 x float> %o +} + +define <4 x float> @deinterleave_shuffle_v8f32_c(<8 x float> %a) { +; CHECK-LABEL: deinterleave_shuffle_v8f32_c: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 v2.4s, v0.4s, v1.4s +; CHECK-NEXT: uzp2 v0.4s, v0.4s, v1.4s +; CHECK-NEXT: fadd v0.4s, v0.4s, v2.4s +; CHECK-NEXT: ret + %r0 = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> + %r1 = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> + %o = fadd <4 x float> %r1, %r0 + ret <4 x float> %o +} + +define <8 x half> @deinterleave_shuffle_v16f16(<16 x half> %a) { +; CHECK-NOFP16-LABEL: deinterleave_shuffle_v16f16: +; CHECK-NOFP16: // %bb.0: +; CHECK-NOFP16-NEXT: uzp1 v2.8h, v0.8h, v1.8h +; CHECK-NOFP16-NEXT: uzp2 v0.8h, v0.8h, v1.8h +; CHECK-NOFP16-NEXT: fcvtl v1.4s, v0.4h +; CHECK-NOFP16-NEXT: fcvtl v3.4s, v2.4h +; CHECK-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h +; CHECK-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h +; CHECK-NOFP16-NEXT: fadd v1.4s, v3.4s, v1.4s +; CHECK-NOFP16-NEXT: fadd v2.4s, v2.4s, v0.4s +; CHECK-NOFP16-NEXT: fcvtn v0.4h, v1.4s +; CHECK-NOFP16-NEXT: fcvtn2 v0.8h, v2.4s +; CHECK-NOFP16-NEXT: ret +; +; CHECK-FP16-LABEL: deinterleave_shuffle_v16f16: +; CHECK-FP16: // %bb.0: +; CHECK-FP16-NEXT: uzp1 v2.8h, v0.8h, v1.8h +; CHECK-FP16-NEXT: uzp2 v0.8h, v0.8h, v1.8h +; CHECK-FP16-NEXT: fadd v0.8h, v2.8h, v0.8h +; CHECK-FP16-NEXT: ret + %r0 = shufflevector <16 x half> %a, <16 x half> poison, <8 x i32> + %r1 = shufflevector <16 x half> %a, <16 x half> poison, <8 x i32> + %o = fadd <8 x half> %r0, %r1 + ret <8 x half> %o +} + +define <4 x double> @deinterleave_shuffle_v8f64(<8 x double> %a) { +; CHECK-LABEL: deinterleave_shuffle_v8f64: +; CHECK: // %bb.0: +; CHECK-NEXT: zip1 v4.2d, v2.2d, v3.2d +; CHECK-NEXT: zip1 v5.2d, v0.2d, v1.2d +; CHECK-NEXT: zip2 v2.2d, v2.2d, v3.2d +; CHECK-NEXT: zip2 v0.2d, v0.2d, v1.2d +; CHECK-NEXT: fadd v1.2d, v4.2d, v2.2d +; CHECK-NEXT: fadd v0.2d, v5.2d, v0.2d +; CHECK-NEXT: ret + %r0 = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> + %r1 = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> + %o = fadd <4 x double> %r0, %r1 + ret <4 x double> %o +} + +define <4 x i32> @udot(<4 x i32> %z, <16 x i8> %a, <16 x i8> %b) { +; CHECK-LABEL: udot: +; CHECK: // %bb.0: +; CHECK-NEXT: ushll v3.8h, v1.8b, #0 +; CHECK-NEXT: ushll v4.8h, v2.8b, #0 +; CHECK-NEXT: ushll2 v1.8h, v1.16b, #0 +; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: umull2 v5.4s, v3.8h, v4.8h +; CHECK-NEXT: umull v3.4s, v3.4h, v4.4h +; CHECK-NEXT: umull2 v4.4s, v1.8h, v2.8h +; CHECK-NEXT: umull v1.4s, v1.4h, v2.4h +; CHECK-NEXT: uzp1 v2.4s, v3.4s, v5.4s +; CHECK-NEXT: uzp2 v3.4s, v3.4s, v5.4s +; CHECK-NEXT: uzp1 v6.4s, v1.4s, v4.4s +; CHECK-NEXT: uzp2 v1.4s, v1.4s, v4.4s +; CHECK-NEXT: add v2.4s, v2.4s, v3.4s +; CHECK-NEXT: add v1.4s, v6.4s, v1.4s +; CHECK-NEXT: uzp1 v3.4s, v2.4s, v1.4s +; CHECK-NEXT: uzp2 v1.4s, v2.4s, v1.4s +; CHECK-NEXT: add v1.4s, v3.4s, v1.4s +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-NEXT: ret + %za = zext <16 x i8> %a to <16 x i32> + %zb = zext <16 x i8> %b to <16 x i32> + %m = mul <16 x i32> %za, %zb + %r0 = shufflevector <16 x i32> %m, <16 x i32> poison, <8 x i32> + %r1 = shufflevector <16 x i32> %m, <16 x i32> poison, <8 x i32> + %m2 = add <8 x i32> %r0, %r1 + %s0 = shufflevector <8 x i32> %m2, <8 x i32> poison, <4 x i32> + %s1 = shufflevector <8 x i32> %m2, <8 x i32> poison, <4 x i32> + %o = add <4 x i32> %s0, %s1 + %n = add <4 x i32> %z, %o + ret <4 x i32> %n +} + +define <4 x i32> @sdot(<4 x i32> %z, <16 x i8> %a, <16 x i8> %b) { +; CHECK-LABEL: sdot: +; CHECK: // %bb.0: +; CHECK-NEXT: sshll v3.8h, v1.8b, #0 +; CHECK-NEXT: sshll v4.8h, v2.8b, #0 +; CHECK-NEXT: sshll2 v1.8h, v1.16b, #0 +; CHECK-NEXT: sshll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: smull2 v5.4s, v3.8h, v4.8h +; CHECK-NEXT: smull v3.4s, v3.4h, v4.4h +; CHECK-NEXT: smull2 v4.4s, v1.8h, v2.8h +; CHECK-NEXT: smull v1.4s, v1.4h, v2.4h +; CHECK-NEXT: uzp1 v2.4s, v3.4s, v5.4s +; CHECK-NEXT: uzp2 v3.4s, v3.4s, v5.4s +; CHECK-NEXT: uzp1 v6.4s, v1.4s, v4.4s +; CHECK-NEXT: uzp2 v1.4s, v1.4s, v4.4s +; CHECK-NEXT: add v2.4s, v2.4s, v3.4s +; CHECK-NEXT: add v1.4s, v6.4s, v1.4s +; CHECK-NEXT: uzp1 v3.4s, v2.4s, v1.4s +; CHECK-NEXT: uzp2 v1.4s, v2.4s, v1.4s +; CHECK-NEXT: add v1.4s, v3.4s, v1.4s +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-NEXT: ret + %za = sext <16 x i8> %a to <16 x i32> + %zb = sext <16 x i8> %b to <16 x i32> + %m = mul <16 x i32> %za, %zb + %r0 = shufflevector <16 x i32> %m, <16 x i32> poison, <8 x i32> + %r1 = shufflevector <16 x i32> %m, <16 x i32> poison, <8 x i32> + %m2 = add <8 x i32> %r0, %r1 + %s0 = shufflevector <8 x i32> %m2, <8 x i32> poison, <4 x i32> + %s1 = shufflevector <8 x i32> %m2, <8 x i32> poison, <4 x i32> + %o = add <4 x i32> %s0, %s1 + %n = add <4 x i32> %z, %o + ret <4 x i32> %n +} -- GitLab From 2981f3a284302bb12b292bcf09e7e09ae2eb696a Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 7 Jun 2024 08:45:35 -0500 Subject: [PATCH 250/462] [Clang] Add timeout for GPU detection utilities (#94751) Summary: The utilities `nvptx-arch` and `amdgpu-arch` are used to support `--offload-arch=native` among other utilities in clang. However, these rely on the GPU drivers to query the features. In certain cases these drivers can become locked up, which will lead to indefinate hangs on any compiler jobs running in the meantime. This patch adds a ten second timeout period for these utilities before it kills the job and errors out. --- clang/include/clang/Driver/ToolChain.h | 3 ++- clang/lib/Driver/ToolChain.cpp | 8 ++++---- clang/lib/Driver/ToolChains/AMDGPU.cpp | 2 +- clang/lib/Driver/ToolChains/Cuda.cpp | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h index a4f9cad98aa8..9789cfacafd7 100644 --- a/clang/include/clang/Driver/ToolChain.h +++ b/clang/include/clang/Driver/ToolChain.h @@ -205,7 +205,8 @@ protected: /// Executes the given \p Executable and returns the stdout. llvm::Expected> - executeToolChainProgram(StringRef Executable) const; + executeToolChainProgram(StringRef Executable, + unsigned SecondsToWait = 0) const; void setTripleEnvironment(llvm::Triple::EnvironmentType Env); diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp index 0e86bc07e0ea..40ab2e91125d 100644 --- a/clang/lib/Driver/ToolChain.cpp +++ b/clang/lib/Driver/ToolChain.cpp @@ -104,7 +104,8 @@ ToolChain::ToolChain(const Driver &D, const llvm::Triple &T, } llvm::Expected> -ToolChain::executeToolChainProgram(StringRef Executable) const { +ToolChain::executeToolChainProgram(StringRef Executable, + unsigned SecondsToWait) const { llvm::SmallString<64> OutputFile; llvm::sys::fs::createTemporaryFile("toolchain-program", "txt", OutputFile); llvm::FileRemover OutputRemover(OutputFile.c_str()); @@ -115,9 +116,8 @@ ToolChain::executeToolChainProgram(StringRef Executable) const { }; std::string ErrorMessage; - if (llvm::sys::ExecuteAndWait(Executable, {}, {}, Redirects, - /* SecondsToWait */ 0, - /*MemoryLimit*/ 0, &ErrorMessage)) + if (llvm::sys::ExecuteAndWait(Executable, {}, {}, Redirects, SecondsToWait, + /*MemoryLimit=*/0, &ErrorMessage)) return llvm::createStringError(std::error_code(), Executable + ": " + ErrorMessage); diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp index 9ffea57b005d..11a98a0ec314 100644 --- a/clang/lib/Driver/ToolChains/AMDGPU.cpp +++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp @@ -877,7 +877,7 @@ AMDGPUToolChain::getSystemGPUArchs(const ArgList &Args) const { else Program = GetProgramPath("amdgpu-arch"); - auto StdoutOrErr = executeToolChainProgram(Program); + auto StdoutOrErr = executeToolChainProgram(Program, /*SecondsToWait=*/10); if (!StdoutOrErr) return StdoutOrErr.takeError(); diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp index bbc8be91fd70..2dfc7457b0ac 100644 --- a/clang/lib/Driver/ToolChains/Cuda.cpp +++ b/clang/lib/Driver/ToolChains/Cuda.cpp @@ -826,7 +826,7 @@ NVPTXToolChain::getSystemGPUArchs(const ArgList &Args) const { else Program = GetProgramPath("nvptx-arch"); - auto StdoutOrErr = executeToolChainProgram(Program); + auto StdoutOrErr = executeToolChainProgram(Program, /*SecondsToWait=*/10); if (!StdoutOrErr) return StdoutOrErr.takeError(); -- GitLab From 2afea7296812b7e12c6ec683e6858bd4cbe8dd8d Mon Sep 17 00:00:00 2001 From: Liao Chunyu Date: Fri, 7 Jun 2024 21:45:49 +0800 Subject: [PATCH 251/462] [RISCV] Codegen support for XCVmem extension (#76916) All post-Increment load/store, register-register load/store spec: https://github.com/openhwgroup/cv32e40p/blob/master/docs/source/instruction_set_extensions.rst Contributors: @CharKeaney, @jeremybennett, @lewis-revill, @NandniJamnadas, @PaoloS02, @serkm, @simonpcook, @xingmingjie, @realqhc --- .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 2 +- llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp | 74 +++++ llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h | 2 + llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 30 ++ llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td | 45 ++- llvm/test/CodeGen/RISCV/xcvmem.ll | 295 ++++++++++++++++++ 6 files changed, 446 insertions(+), 2 deletions(-) create mode 100644 llvm/test/CodeGen/RISCV/xcvmem.ll diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index d92998ced91e..5906a2cdb3bf 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -1245,7 +1245,7 @@ public: } void addRegRegOperands(MCInst &Inst, unsigned N) const { - assert(N == 1 && "Invalid number of operands!"); + assert(N == 2 && "Invalid number of operands!"); Inst.addOperand(MCOperand::createReg(RegReg.Reg1)); Inst.addOperand(MCOperand::createReg(RegReg.Reg2)); } diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index 251401db3e28..8c1f8dca4e10 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -1527,6 +1527,67 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { case ISD::LOAD: { if (tryIndexedLoad(Node)) return; + + if (Subtarget->hasVendorXCVmem()) { + // We match post-incrementing load here + LoadSDNode *Load = cast(Node); + if (Load->getAddressingMode() != ISD::POST_INC) + break; + + SDValue Chain = Node->getOperand(0); + SDValue Base = Node->getOperand(1); + SDValue Offset = Node->getOperand(2); + + bool Simm12 = false; + bool SignExtend = Load->getExtensionType() == ISD::SEXTLOAD; + + if (auto ConstantOffset = dyn_cast(Offset)) { + int ConstantVal = ConstantOffset->getSExtValue(); + Simm12 = isInt<12>(ConstantVal); + if (Simm12) + Offset = CurDAG->getTargetConstant(ConstantVal, SDLoc(Offset), + Offset.getValueType()); + } + + unsigned Opcode = 0; + switch (Load->getMemoryVT().getSimpleVT().SimpleTy) { + case MVT::i8: + if (Simm12 && SignExtend) + Opcode = RISCV::CV_LB_ri_inc; + else if (Simm12 && !SignExtend) + Opcode = RISCV::CV_LBU_ri_inc; + else if (!Simm12 && SignExtend) + Opcode = RISCV::CV_LB_rr_inc; + else + Opcode = RISCV::CV_LBU_rr_inc; + break; + case MVT::i16: + if (Simm12 && SignExtend) + Opcode = RISCV::CV_LH_ri_inc; + else if (Simm12 && !SignExtend) + Opcode = RISCV::CV_LHU_ri_inc; + else if (!Simm12 && SignExtend) + Opcode = RISCV::CV_LH_rr_inc; + else + Opcode = RISCV::CV_LHU_rr_inc; + break; + case MVT::i32: + if (Simm12) + Opcode = RISCV::CV_LW_ri_inc; + else + Opcode = RISCV::CV_LW_rr_inc; + break; + default: + break; + } + if (!Opcode) + break; + + ReplaceNode(Node, CurDAG->getMachineNode(Opcode, DL, XLenVT, XLenVT, + Chain.getSimpleValueType(), Base, + Offset, Chain)); + return; + } break; } case ISD::INTRINSIC_WO_CHAIN: { @@ -2669,6 +2730,19 @@ bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base, return true; } +bool RISCVDAGToDAGISel::SelectAddrRegReg(SDValue Addr, SDValue &Base, + SDValue &Offset) { + if (Addr.getOpcode() != ISD::ADD) + return false; + + if (isa(Addr.getOperand(1))) + return false; + + Base = Addr.getOperand(1); + Offset = Addr.getOperand(0); + return true; +} + bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth, SDValue &ShAmt) { ShAmt = N; diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h index 5d70245c0c55..1b3b00eeccce 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h @@ -78,6 +78,8 @@ public: return false; } + bool SelectAddrRegReg(SDValue Addr, SDValue &Base, SDValue &Offset); + bool tryShrinkShlLogicImm(SDNode *Node); bool trySignedBitfieldExtract(SDNode *Node); bool tryIndexedLoad(SDNode *Node); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 4051279fdbf8..1eae1dc968db 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -1451,6 +1451,16 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, } } + if (Subtarget.hasVendorXCVmem()) { + setIndexedLoadAction(ISD::POST_INC, MVT::i8, Legal); + setIndexedLoadAction(ISD::POST_INC, MVT::i16, Legal); + setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal); + + setIndexedStoreAction(ISD::POST_INC, MVT::i8, Legal); + setIndexedStoreAction(ISD::POST_INC, MVT::i16, Legal); + setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal); + } + // Function alignments. const Align FunctionAlignment(Subtarget.hasStdExtCOrZca() ? 2 : 4); setMinFunctionAlignment(FunctionAlignment); @@ -20933,6 +20943,26 @@ bool RISCVTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const { + if (Subtarget.hasVendorXCVmem()) { + if (Op->getOpcode() != ISD::ADD) + return false; + + if (LSBaseSDNode *LS = dyn_cast(N)) + Base = LS->getBasePtr(); + else + return false; + + if (Base == Op->getOperand(0)) + Offset = Op->getOperand(1); + else if (Base == Op->getOperand(1)) + Offset = Op->getOperand(0); + else + return false; + + AM = ISD::POST_INC; + return true; + } + EVT VT; SDValue Ptr; if (LoadSDNode *LD = dyn_cast(N)) { diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td index 6dae8ca8f7a8..f0d6913a9d3f 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td @@ -512,11 +512,13 @@ def CVrrAsmOperand : AsmOperandClass { let DiagnosticType = "InvalidRegReg"; } -def CVrr : Operand { +def CVrr : Operand, + ComplexPattern { let ParserMatchClass = CVrrAsmOperand; let EncoderMethod = "getRegReg"; let DecoderMethod = "decodeRegReg"; let PrintMethod = "printRegReg"; + let MIOperandInfo = (ops GPR:$base, GPR:$offset); } class CVLoad_ri_inc funct3, string opcodestr> @@ -659,6 +661,47 @@ let Predicates = [HasVendorXCVelw, IsRV32], hasSideEffects = 0, def CV_ELW : CVLoad_ri<0b011, "cv.elw">; } +//===----------------------------------------------------------------------===// +// Patterns for load & store operations +//===----------------------------------------------------------------------===// +class CVLdrrPat + : Pat<(XLenVT (LoadOp CVrr:$regreg)), + (Inst CVrr:$regreg)>; + +class CVStriPat + : Pat<(StoreOp (XLenVT GPR:$rs2), GPR:$rs1, simm12:$imm12), + (Inst GPR:$rs2, GPR:$rs1, simm12:$imm12)>; + +class CVStrriPat + : Pat<(StoreOp (XLenVT GPR:$rs2), GPR:$rs1, GPR:$rs3), + (Inst GPR:$rs2, GPR:$rs1, GPR:$rs3)>; + +class CVStrrPat + : Pat<(StoreOp (XLenVT GPR:$rs2), CVrr:$regreg), + (Inst GPR:$rs2, CVrr:$regreg)>; + +let Predicates = [HasVendorXCVmem, IsRV32], AddedComplexity = 1 in { + def : CVLdrrPat; + def : CVLdrrPat; + def : CVLdrrPat; + def : CVLdrrPat; + def : CVLdrrPat; + def : CVLdrrPat; + def : CVLdrrPat; + + def : CVStriPat; + def : CVStriPat; + def : CVStriPat; + + def : CVStrriPat; + def : CVStrriPat; + def : CVStrriPat; + + def : CVStrrPat; + def : CVStrrPat; + def : CVStrrPat; +} + def cv_tuimm2 : TImmLeaf(Imm);}]>; def cv_tuimm5 : TImmLeaf(Imm);}]>; def cv_uimm10 : ImmLeaf(Imm);}]>; diff --git a/llvm/test/CodeGen/RISCV/xcvmem.ll b/llvm/test/CodeGen/RISCV/xcvmem.ll new file mode 100644 index 000000000000..037e49b9b0df --- /dev/null +++ b/llvm/test/CodeGen/RISCV/xcvmem.ll @@ -0,0 +1,295 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -O3 -mtriple=riscv32 -mattr=+xcvmem -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK + +define <2 x i32> @lb_ri_inc(i8* %a) { +; CHECK-LABEL: lb_ri_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.lb a1, (a0), 42 +; CHECK-NEXT: ret + %1 = load i8, i8* %a + %2 = sext i8 %1 to i32 + %3 = getelementptr i8, i8* %a, i32 42 + %4 = ptrtoint i8* %3 to i32 + %5 = insertelement <2 x i32> undef, i32 %4, i32 0 + %6 = insertelement <2 x i32> %5, i32 %2, i32 1 + ret <2 x i32> %6 +} + +define <2 x i32> @lb_rr_inc(i8* %a, i32 %b) { +; CHECK-LABEL: lb_rr_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.lb a1, (a0), a1 +; CHECK-NEXT: ret + %1 = load i8, i8* %a + %2 = sext i8 %1 to i32 + %3 = getelementptr i8, i8* %a, i32 %b + %4 = ptrtoint i8* %3 to i32 + %5 = insertelement <2 x i32> undef, i32 %4, i32 0 + %6 = insertelement <2 x i32> %5, i32 %2, i32 1 + ret <2 x i32> %6 +} + +define i32 @lb_rr(i8* %a, i32 %b) { +; CHECK-LABEL: lb_rr: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.lb a0, a1(a0) +; CHECK-NEXT: ret + %1 = getelementptr i8, i8* %a, i32 %b + %2 = load i8, i8* %1 + %3 = sext i8 %2 to i32 + ret i32 %3 +} + +define <2 x i32> @lbu_ri_inc(i8* %a) { +; CHECK-LABEL: lbu_ri_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.lbu a1, (a0), 42 +; CHECK-NEXT: ret + %1 = load i8, i8* %a + %2 = zext i8 %1 to i32 + %3 = getelementptr i8, i8* %a, i32 42 + %4 = ptrtoint i8* %3 to i32 + %5 = insertelement <2 x i32> undef, i32 %4, i32 0 + %6 = insertelement <2 x i32> %5, i32 %2, i32 1 + ret <2 x i32> %6 +} + +define <2 x i32> @lbu_rr_inc(i8* %a, i32 %b) { +; CHECK-LABEL: lbu_rr_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.lbu a1, (a0), a1 +; CHECK-NEXT: ret + %1 = load i8, i8* %a + %2 = zext i8 %1 to i32 + %3 = getelementptr i8, i8* %a, i32 %b + %4 = ptrtoint i8* %3 to i32 + %5 = insertelement <2 x i32> undef, i32 %4, i32 0 + %6 = insertelement <2 x i32> %5, i32 %2, i32 1 + ret <2 x i32> %6 +} + +define i32 @lbu_rr(i8* %a, i32 %b) { +; CHECK-LABEL: lbu_rr: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.lbu a0, a1(a0) +; CHECK-NEXT: ret + %1 = getelementptr i8, i8* %a, i32 %b + %2 = load i8, i8* %1 + %3 = zext i8 %2 to i32 + ret i32 %3 +} + +define <2 x i32> @lh_ri_inc(i16* %a) { +; CHECK-LABEL: lh_ri_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.lh a1, (a0), 84 +; CHECK-NEXT: ret + %1 = load i16, i16* %a + %2 = sext i16 %1 to i32 + %3 = getelementptr i16, i16* %a, i32 42 + %4 = ptrtoint i16* %3 to i32 + %5 = insertelement <2 x i32> undef, i32 %4, i32 0 + %6 = insertelement <2 x i32> %5, i32 %2, i32 1 + ret <2 x i32> %6 +} + +define <2 x i32> @lh_rr_inc(i16* %a, i32 %b) { +; CHECK-LABEL: lh_rr_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a1, a1, 1 +; CHECK-NEXT: cv.lh a1, (a0), a1 +; CHECK-NEXT: ret + %1 = load i16, i16* %a + %2 = sext i16 %1 to i32 + %3 = getelementptr i16, i16* %a, i32 %b + %4 = ptrtoint i16* %3 to i32 + %5 = insertelement <2 x i32> undef, i32 %4, i32 0 + %6 = insertelement <2 x i32> %5, i32 %2, i32 1 + ret <2 x i32> %6 +} + +define i32 @lh_rr(i16* %a, i32 %b) { +; CHECK-LABEL: lh_rr: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a1, a1, 1 +; CHECK-NEXT: cv.lh a0, a1(a0) +; CHECK-NEXT: ret + %1 = getelementptr i16, i16* %a, i32 %b + %2 = load i16, i16* %1 + %3 = sext i16 %2 to i32 + ret i32 %3 +} + +define <2 x i32> @lhu_ri_inc(i16* %a) { +; CHECK-LABEL: lhu_ri_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.lhu a1, (a0), 84 +; CHECK-NEXT: ret + %1 = load i16, i16* %a + %2 = zext i16 %1 to i32 + %3 = getelementptr i16, i16* %a, i32 42 + %4 = ptrtoint i16* %3 to i32 + %5 = insertelement <2 x i32> undef, i32 %4, i32 0 + %6 = insertelement <2 x i32> %5, i32 %2, i32 1 + ret <2 x i32> %6 +} + +define <2 x i32> @lhu_rr_inc(i16* %a, i32 %b) { +; CHECK-LABEL: lhu_rr_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a1, a1, 1 +; CHECK-NEXT: cv.lhu a1, (a0), a1 +; CHECK-NEXT: ret + %1 = load i16, i16* %a + %2 = zext i16 %1 to i32 + %3 = getelementptr i16, i16* %a, i32 %b + %4 = ptrtoint i16* %3 to i32 + %5 = insertelement <2 x i32> undef, i32 %4, i32 0 + %6 = insertelement <2 x i32> %5, i32 %2, i32 1 + ret <2 x i32> %6 +} + +define i32 @lhu_rr(i16* %a, i32 %b) { +; CHECK-LABEL: lhu_rr: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a1, a1, 1 +; CHECK-NEXT: cv.lhu a0, a1(a0) +; CHECK-NEXT: ret + %1 = getelementptr i16, i16* %a, i32 %b + %2 = load i16, i16* %1 + %3 = zext i16 %2 to i32 + ret i32 %3 +} + +define <2 x i32> @lw_ri_inc(i32* %a) { +; CHECK-LABEL: lw_ri_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.lw a1, (a0), 168 +; CHECK-NEXT: ret + %1 = load i32, i32* %a + %2 = getelementptr i32, i32* %a, i32 42 + %3 = ptrtoint i32* %2 to i32 + %4 = insertelement <2 x i32> undef, i32 %3, i32 0 + %5 = insertelement <2 x i32> %4, i32 %1, i32 1 + ret <2 x i32> %5 +} + +define <2 x i32> @lw_rr_inc(i32* %a, i32 %b) { +; CHECK-LABEL: lw_rr_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a1, a1, 2 +; CHECK-NEXT: cv.lw a1, (a0), a1 +; CHECK-NEXT: ret + %1 = load i32, i32* %a + %2 = getelementptr i32, i32* %a, i32 %b + %3 = ptrtoint i32* %2 to i32 + %4 = insertelement <2 x i32> undef, i32 %3, i32 0 + %5 = insertelement <2 x i32> %4, i32 %1, i32 1 + ret <2 x i32> %5 +} + +define i32 @lw_rr(i32* %a, i32 %b) { +; CHECK-LABEL: lw_rr: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a1, a1, 2 +; CHECK-NEXT: cv.lw a0, a1(a0) +; CHECK-NEXT: ret + %1 = getelementptr i32, i32* %a, i32 %b + %2 = load i32, i32* %1 + ret i32 %2 +} + +define i8* @sb_ri_inc(i8* %a, i8 %b) { +; CHECK-LABEL: sb_ri_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.sb a1, (a0), 42 +; CHECK-NEXT: ret + store i8 %b, i8* %a + %1 = getelementptr i8, i8* %a, i32 42 + ret i8* %1 +} + +define i8* @sb_rr_inc(i8* %a, i8 %b, i32 %c) { +; CHECK-LABEL: sb_rr_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.sb a1, (a0), a2 +; CHECK-NEXT: ret + store i8 %b, i8* %a + %1 = getelementptr i8, i8* %a, i32 %c + ret i8* %1 +} + +define void @sb_rr(i8* %a, i8 %b, i32 %c) { +; CHECK-LABEL: sb_rr: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.sb a1, a2(a0) +; CHECK-NEXT: ret + %1 = getelementptr i8, i8* %a, i32 %c + store i8 %b, i8* %1 + ret void +} + +define i16* @sh_ri_inc(i16* %a, i16 %b) { +; CHECK-LABEL: sh_ri_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.sh a1, (a0), 84 +; CHECK-NEXT: ret + store i16 %b, i16* %a + %1 = getelementptr i16, i16* %a, i32 42 + ret i16* %1 +} + +define i16* @sh_rr_inc(i16* %a, i16 %b, i32 %c) { +; CHECK-LABEL: sh_rr_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a2, a2, 1 +; CHECK-NEXT: cv.sh a1, (a0), a2 +; CHECK-NEXT: ret + store i16 %b, i16* %a + %1 = getelementptr i16, i16* %a, i32 %c + ret i16* %1 +} + +define void @sh_rr(i16* %a, i16 %b, i32 %c) { +; CHECK-LABEL: sh_rr: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a2, a2, 1 +; CHECK-NEXT: cv.sh a1, a2(a0) +; CHECK-NEXT: ret + %1 = getelementptr i16, i16* %a, i32 %c + store i16 %b, i16* %1 + ret void +} + +define i32* @sw_ri_inc(i32* %a, i32 %b) { +; CHECK-LABEL: sw_ri_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: cv.sw a1, (a0), 168 +; CHECK-NEXT: ret + store i32 %b, i32* %a + %1 = getelementptr i32, i32* %a, i32 42 + ret i32* %1 +} + +define i32* @sw_rr_inc(i32* %a, i32 %b, i32 %c) { +; CHECK-LABEL: sw_rr_inc: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a2, a2, 2 +; CHECK-NEXT: cv.sw a1, (a0), a2 +; CHECK-NEXT: ret + store i32 %b, i32* %a + %1 = getelementptr i32, i32* %a, i32 %c + ret i32* %1 +} + +define void @sw_rr(i32* %a, i32 %b, i32 %c) { +; CHECK-LABEL: sw_rr: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a2, a2, 2 +; CHECK-NEXT: cv.sw a1, a2(a0) +; CHECK-NEXT: ret + %1 = getelementptr i32, i32* %a, i32 %c + store i32 %b, i32* %1 + ret void +} -- GitLab From 3b16630c26505060a876f578e4b2ba701c780e9a Mon Sep 17 00:00:00 2001 From: Xuan Zhang <144393379+xuanzh-meta@users.noreply.github.com> Date: Fri, 7 Jun 2024 09:50:13 -0400 Subject: [PATCH 252/462] [MachineOutliner] Sort by Benefit to Cost Ratio (#90264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR depends on https://github.com/llvm/llvm-project/pull/90260 We changed the order in which functions are outlined in Machine Outliner. The formula for priority is found via a black-box Bayesian optimization toolbox. Using this formula for sorting consistently reduces the uncompressed size of large real-world mobile apps. We also ran a few benchmarks using LLVM test suites, and showed that sorting by priority consistently reduces the text segment size. |run (CTMark/)   |baseline (1)|priority (2)|diff (1 -> 2)| |----------------|------------|------------|-------------| |lencod          |349624      |349264      |-0.1030%     | |SPASS           |219672      |219480      |-0.0874%     | |kc              |271956      |251200      |-7.6321%     | |sqlite3         |223920      |223708      |-0.0947%     | |7zip-benchmark  |405364      |402624      |-0.6759%     | |bullet          |139820      |139500      |-0.2289%     | |consumer-typeset|295684      |290196      |-1.8560%     | |pairlocalalign  |72236       |72092       |-0.1993%     | |tramp3d-v4      |189572      |189292      |-0.1477%     | This is part of an enhanced version of machine outliner -- see [RFC](https://discourse.llvm.org/t/rfc-enhanced-machine-outliner-part-1-fulllto-part-2-thinlto-nolto-to-come/78732). --- llvm/lib/CodeGen/MachineOutliner.cpp | 6 +- .../machine-outliner-sort-per-priority.ll | 96 ++++++++ .../machine-outliner-sort-per-priority.mir | 206 ++++++++++++++++++ .../CodeGen/ARM/machine-outliner-calls.mir | 80 +++---- .../CodeGen/ARM/machine-outliner-default.mir | 33 ++- .../ARM/machine-outliner-stack-fixup-arm.mir | 56 ++--- .../machine-outliner-stack-fixup-thumb.mir | 74 +++---- 7 files changed, 400 insertions(+), 151 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.mir diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp index 626e577a30bf..9e2e6316dc6d 100644 --- a/llvm/lib/CodeGen/MachineOutliner.cpp +++ b/llvm/lib/CodeGen/MachineOutliner.cpp @@ -828,10 +828,12 @@ bool MachineOutliner::outline(Module &M, << "\n"); bool OutlinedSomething = false; - // Sort by benefit. The most beneficial functions should be outlined first. + // Sort by priority where priority := getNotOutlinedCost / getOutliningCost. + // The function with highest priority should be outlined first. stable_sort(FunctionList, [](const OutlinedFunction &LHS, const OutlinedFunction &RHS) { - return LHS.getBenefit() > RHS.getBenefit(); + return LHS.getNotOutlinedCost() * RHS.getOutliningCost() > + RHS.getNotOutlinedCost() * LHS.getOutliningCost(); }); // Walk over each function, outlining them as we go along. Functions are diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll b/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll new file mode 100644 index 000000000000..09b02531d2a7 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll @@ -0,0 +1,96 @@ +; This tests the order in which functions are outlined in MachineOutliner +; There are TWO key OutlinedFunction in FunctionList +; +; ===================== First One ===================== +; ``` +; mov w0, #1 +; mov w1, #2 +; mov w2, #3 +; mov w3, #4 +; mov w4, #5 +; ``` +; It has: +; - `SequenceSize=20` and `OccurrenceCount=6` +; - each Candidate has `CallOverhead=12` and `FrameOverhead=4` +; - `NotOutlinedCost=20*6=120` and `OutliningCost=12*6+20+4=96` +; - `Benefit=120-96=24` and `Priority=120/96=1.25` +; +; ===================== Second One ===================== +; ``` +; mov w6, #6 +; mov w7, #7 +; b +; ``` +; It has: +; - `SequenceSize=12` and `OccurrenceCount=4` +; - each Candidate has `CallOverhead=4` and `FrameOverhead=0` +; - `NotOutlinedCost=12*4=48` and `OutliningCost=4*4+12+0=28` +; - `Benefit=48-28=20` and `Priority=48/28=1.71` +; +; Note that the first one has higher benefit, but lower priority. +; Hence, when outlining per priority, the second one will be outlined first. + +; RUN: llc %s -enable-machine-outliner=always -filetype=obj -o %t +; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-SORT-BY-PRIORITY + +; RUN: llc %s -enable-machine-outliner=always -outliner-benefit-threshold=22 -filetype=obj -o %t +; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-THRESHOLD + + +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" +target triple = "arm64-apple-macosx14.0.0" + +declare i32 @_Z3fooiiii(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) + +define i32 @_Z2f1v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 11, i32 noundef 6, i32 noundef 7) + ret i32 %1 +} + +define i32 @_Z2f2v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 12, i32 noundef 6, i32 noundef 7) + ret i32 %1 +} + +define i32 @_Z2f3v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 13, i32 noundef 6, i32 noundef 7) + ret i32 %1 +} + +define i32 @_Z2f4v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 14, i32 noundef 6, i32 noundef 7) + ret i32 %1 +} + +define i32 @_Z2f5v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 15, i32 noundef 8, i32 noundef 9) + ret i32 %1 +} + +define i32 @_Z2f6v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 16, i32 noundef 9, i32 noundef 8) + ret i32 %1 +} + +; CHECK-SORT-BY-PRIORITY: <_OUTLINED_FUNCTION_0>: +; CHECK-SORT-BY-PRIORITY-NEXT: mov w6, #0x6 +; CHECK-SORT-BY-PRIORITY-NEXT: mov w7, #0x7 +; CHECK-SORT-BY-PRIORITY-NEXT: b + +; CHECK-SORT-BY-PRIORITY: <_OUTLINED_FUNCTION_1>: +; CHECK-SORT-BY-PRIORITY-NEXT: mov w0, #0x1 +; CHECK-SORT-BY-PRIORITY-NEXT: mov w1, #0x2 +; CHECK-SORT-BY-PRIORITY-NEXT: mov w2, #0x3 +; CHECK-SORT-BY-PRIORITY-NEXT: mov w3, #0x4 +; CHECK-SORT-BY-PRIORITY-NEXT: mov w4, #0x5 +; CHECK-SORT-BY-PRIORITY-NEXT: ret + +; CHECK-THRESHOLD: <_OUTLINED_FUNCTION_0>: +; CHECK-THRESHOLD-NEXT: mov w0, #0x1 +; CHECK-THRESHOLD-NEXT: mov w1, #0x2 +; CHECK-THRESHOLD-NEXT: mov w2, #0x3 +; CHECK-THRESHOLD-NEXT: mov w3, #0x4 +; CHECK-THRESHOLD-NEXT: mov w4, #0x5 +; CHECK-THRESHOLD-NEXT: ret + +; CHECK-THRESHOLD-NOT: <_OUTLINED_FUNCTION_1>: diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.mir b/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.mir new file mode 100644 index 000000000000..444590657a20 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.mir @@ -0,0 +1,206 @@ +# The content of this test is modfied upon the output obtained from running +# `bin/llc -O2 -stop-before=machine-outliner /llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll -o -` +# RUN: llc -mtriple=aarch64 -run-pass=machine-outliner -verify-machineinstrs %s -o - | FileCheck %s + +--- | + declare i32 @foo() + + define void @f1() #0 { ret void } + define void @f2() #0 { ret void } + define void @f3() #0 { ret void } + define void @f4() #0 { ret void } + define void @f5() #0 { ret void } + define void @f6() #0 { ret void } + + attributes #0 = { minsize } +... +--- +# CHECK-LABEL: name: f1 +# CHECK-LABEL: bb.0: +# CHECK-NEXT: liveins: $lr +# CHECK-NEXT: {{ $}} +# CHECK-NEXT: $x5 = ORRXrs $xzr, $lr, 0 +# CHECK-NEXT: BL @OUTLINED_FUNCTION_1 +# CHECK-NEXT: $lr = ORRXrs $xzr, $x5, 0 +# CHECK-NEXT: $w5 = MOVZWi 11, 0 +# CHECK-NEXT: TCRETURNdi @OUTLINED_FUNCTION_0 +name: f1 +tracksRegLiveness: true +frameInfo: + isCalleeSavedInfoValid: true +machineFunctionInfo: + hasRedZone: false +body: | + bb.0: + $w0 = MOVZWi 1, 0 + $w1 = MOVZWi 2, 0 + $w2 = MOVZWi 3, 0 + $w3 = MOVZWi 4, 0 + $w4 = MOVZWi 5, 0 + $w5 = MOVZWi 11, 0 + $w6 = MOVZWi 6, 0 + $w7 = MOVZWi 7, 0 + TCRETURNdi @foo, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit killed $w4, implicit killed $w5, implicit killed $w6, implicit killed $w7 + +... +--- +# CHECK-LABEL: name: f2 +# CHECK-LABEL: bb.0: +# CHECK-NEXT: liveins: $lr +# CHECK-NEXT: {{ $}} +# CHECK-NEXT: $x5 = ORRXrs $xzr, $lr, 0 +# CHECK-NEXT: BL @OUTLINED_FUNCTION_1 +# CHECK-NEXT: $lr = ORRXrs $xzr, $x5, 0 +# CHECK-NEXT: $w5 = MOVZWi 12, 0 +# CHECK-NEXT: TCRETURNdi @OUTLINED_FUNCTION_0 +name: f2 +tracksRegLiveness: true +frameInfo: + isCalleeSavedInfoValid: true +machineFunctionInfo: + hasRedZone: false +body: | + bb.0: + $w0 = MOVZWi 1, 0 + $w1 = MOVZWi 2, 0 + $w2 = MOVZWi 3, 0 + $w3 = MOVZWi 4, 0 + $w4 = MOVZWi 5, 0 + $w5 = MOVZWi 12, 0 + $w6 = MOVZWi 6, 0 + $w7 = MOVZWi 7, 0 + TCRETURNdi @foo, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit killed $w4, implicit killed $w5, implicit killed $w6, implicit killed $w7 + +... +--- +# CHECK-LABEL: name: f3 +# CHECK-LABEL: bb.0: +# CHECK-NEXT: liveins: $lr +# CHECK-NEXT: {{ $}} +# CHECK-NEXT: $x5 = ORRXrs $xzr, $lr, 0 +# CHECK-NEXT: BL @OUTLINED_FUNCTION_1 +# CHECK-NEXT: $lr = ORRXrs $xzr, $x5, 0 +# CHECK-NEXT: $w5 = MOVZWi 13, 0 +# CHECK-NEXT: TCRETURNdi @OUTLINED_FUNCTION_0 +name: f3 +tracksRegLiveness: true +frameInfo: + isCalleeSavedInfoValid: true +machineFunctionInfo: + hasRedZone: false +body: | + bb.0: + $w0 = MOVZWi 1, 0 + $w1 = MOVZWi 2, 0 + $w2 = MOVZWi 3, 0 + $w3 = MOVZWi 4, 0 + $w4 = MOVZWi 5, 0 + $w5 = MOVZWi 13, 0 + $w6 = MOVZWi 6, 0 + $w7 = MOVZWi 7, 0 + TCRETURNdi @foo, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit killed $w4, implicit killed $w5, implicit killed $w6, implicit killed $w7 + +... +--- +# CHECK-LABEL: name: f4 +# CHECK-LABEL: bb.0: +# CHECK-NEXT: liveins: $lr +# CHECK-NEXT: {{ $}} +# CHECK-NEXT: $x5 = ORRXrs $xzr, $lr, 0 +# CHECK-NEXT: BL @OUTLINED_FUNCTION_1 +# CHECK-NEXT: $lr = ORRXrs $xzr, $x5, 0 +# CHECK-NEXT: $w5 = MOVZWi 14, 0 +# CHECK-NEXT: TCRETURNdi @OUTLINED_FUNCTION_0 +name: f4 +tracksRegLiveness: true +frameInfo: + isCalleeSavedInfoValid: true +machineFunctionInfo: + hasRedZone: false +body: | + bb.0: + $w0 = MOVZWi 1, 0 + $w1 = MOVZWi 2, 0 + $w2 = MOVZWi 3, 0 + $w3 = MOVZWi 4, 0 + $w4 = MOVZWi 5, 0 + $w5 = MOVZWi 14, 0 + $w6 = MOVZWi 6, 0 + $w7 = MOVZWi 7, 0 + TCRETURNdi @foo, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit killed $w4, implicit killed $w5, implicit killed $w6, implicit killed $w7 + +... +--- +# CHECK-LABEL: name: f5 +# CHECK-LABEL: bb.0: +# CHECK-NEXT: liveins: $lr +# CHECK-NEXT: {{ $}} +# CHECK-NEXT: $x5 = ORRXrs $xzr, $lr, 0 +# CHECK-NEXT: BL @OUTLINED_FUNCTION_1 +# CHECK-NEXT: $lr = ORRXrs $xzr, $x5, 0 +# CHECK-NOT: @OUTLINED_FUNCTION_0 +name: f5 +tracksRegLiveness: true +frameInfo: + isCalleeSavedInfoValid: true +machineFunctionInfo: + hasRedZone: false +body: | + bb.0: + $w0 = MOVZWi 1, 0 + $w1 = MOVZWi 2, 0 + $w2 = MOVZWi 3, 0 + $w3 = MOVZWi 4, 0 + $w4 = MOVZWi 5, 0 + $w5 = MOVZWi 15, 0 + $w6 = MOVZWi 8, 0 + $w7 = MOVZWi 9, 0 + TCRETURNdi @foo, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit killed $w4, implicit killed $w5, implicit killed $w6, implicit killed $w7 + +... +--- +# CHECK-LABEL: name: f6 +# CHECK-LABEL: bb.0: +# CHECK-NEXT: liveins: $lr +# CHECK-NEXT: {{ $}} +# CHECK-NEXT: $x5 = ORRXrs $xzr, $lr, 0 +# CHECK-NEXT: BL @OUTLINED_FUNCTION_1 +# CHECK-NEXT: $lr = ORRXrs $xzr, $x5, 0 +# CHECK-NOT: @OUTLINED_FUNCTION_0 +name: f6 +tracksRegLiveness: true +frameInfo: + isCalleeSavedInfoValid: true +machineFunctionInfo: + hasRedZone: false +body: | + bb.0: + $w0 = MOVZWi 1, 0 + $w1 = MOVZWi 2, 0 + $w2 = MOVZWi 3, 0 + $w3 = MOVZWi 4, 0 + $w4 = MOVZWi 5, 0 + $w5 = MOVZWi 16, 0 + $w6 = MOVZWi 9, 0 + $w7 = MOVZWi 8, 0 + TCRETURNdi @foo, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit killed $w4, implicit killed $w5, implicit killed $w6, implicit killed $w7 + +... + +# CHECK-LABEL: name: OUTLINED_FUNCTION_0 +# CHECK-LABEL: bb.0: +# CHECK-NEXT: liveins: $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $lr, $d8, $d9, $d10, $d11, $d12, $d13, $d14, $d15, $w0, $w1, $w2, $w3, $w4, $w5 +# CHECK-NEXT: {{ $}} +# CHECK-NEXT: $w6 = MOVZWi 6, 0 +# CHECK-NEXT: $w7 = MOVZWi 7, 0 + +# CHECK-LABEL: name: OUTLINED_FUNCTION_1 +# CHECK-LABEL: bb.0: +# CHECK-NEXT: liveins: $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $lr, $d8, $d9, $d10, $d11, $d12, $d13, $d14, $d15 +# CHECK-NEXT: {{ $}} +# CHECK-NEXT: $w0 = MOVZWi 1, 0 +# CHECK-NEXT: $w1 = MOVZWi 2, 0 +# CHECK-NEXT: $w2 = MOVZWi 3, 0 +# CHECK-NEXT: $w3 = MOVZWi 4, 0 +# CHECK-NEXT: $w4 = MOVZWi 5, 0 +# CHECK-NEXT: RET $lr diff --git a/llvm/test/CodeGen/ARM/machine-outliner-calls.mir b/llvm/test/CodeGen/ARM/machine-outliner-calls.mir index a92c9dd28be5..7634ecd6e863 100644 --- a/llvm/test/CodeGen/ARM/machine-outliner-calls.mir +++ b/llvm/test/CodeGen/ARM/machine-outliner-calls.mir @@ -26,15 +26,15 @@ body: | ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8 ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -4 ; CHECK: frame-setup CFI_INSTRUCTION offset $r4, -8 - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_2 ; CHECK: bb.1: - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_2 ; CHECK: bb.2: - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_2 ; CHECK: bb.3: - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_2 ; CHECK: bb.4: - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_2 ; CHECK: bb.5: ; CHECK: $sp = frame-destroy LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $lr ; CHECK: BX_RET 14 /* CC::al */, $noreg @@ -139,13 +139,13 @@ body: | ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8 ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -4 ; CHECK: frame-setup CFI_INSTRUCTION offset $r4, -8 - ; CHECK: BL @OUTLINED_FUNCTION_1 + ; CHECK: BL @OUTLINED_FUNCTION_0 ; CHECK: bb.1: - ; CHECK: BL @OUTLINED_FUNCTION_1 + ; CHECK: BL @OUTLINED_FUNCTION_0 ; CHECK: bb.2: - ; CHECK: BL @OUTLINED_FUNCTION_1 + ; CHECK: BL @OUTLINED_FUNCTION_0 ; CHECK: bb.3: - ; CHECK: BL @OUTLINED_FUNCTION_1 + ; CHECK: BL @OUTLINED_FUNCTION_0 ; CHECK: bb.4: ; CHECK: $sp = frame-destroy LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $lr ; CHECK: BX_RET 14 /* CC::al */, $noreg @@ -245,19 +245,19 @@ body: | ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -4 ; CHECK: frame-setup CFI_INSTRUCTION offset $r4, -8 ; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp - ; CHECK: BL @OUTLINED_FUNCTION_2 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: bb.1: ; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp - ; CHECK: BL @OUTLINED_FUNCTION_2 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: bb.2: ; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp - ; CHECK: BL @OUTLINED_FUNCTION_2 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: bb.3: ; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp - ; CHECK: BL @OUTLINED_FUNCTION_2 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: bb.4: ; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp - ; CHECK: BL @OUTLINED_FUNCTION_2 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: bb.5: ; CHECK: $sp = frame-destroy LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $lr ; CHECK: BX_RET 14 /* CC::al */, $noreg @@ -307,38 +307,17 @@ body: | bb.0: BX_RET 14, $noreg - ; CHECK-LABEL: name: OUTLINED_FUNCTION_0 ; CHECK: bb.0: - ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr - ; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -8 - ; CHECK: BL @bar, implicit-def dead $lr, implicit $sp - ; CHECK: $r0 = MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r1 = MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r2 = MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r3 = MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r4 = MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg - ; CHECK: MOVPCLR 14 /* CC::al */, $noreg - - ; CHECK-LABEL: name: OUTLINED_FUNCTION_1 - ; CHECK: bb.0: - ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr - ; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -8 - ; CHECK: BL @bar, implicit-def dead $lr, implicit $sp + ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 ; CHECK: $r0 = MOVi 2, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r1 = MOVi 2, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r2 = MOVi 2, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r3 = MOVi 2, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r4 = MOVi 2, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg ; CHECK: TAILJMPd @bar, implicit $sp - ; CHECK-LABEL: name: OUTLINED_FUNCTION_2 + ; CHECK-LABEL: name: OUTLINED_FUNCTION_1 ; CHECK: bb.0: ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 ; CHECK: $r0 = MOVi 3, 14 /* CC::al */, $noreg, $noreg @@ -348,31 +327,28 @@ body: | ; CHECK: $r4 = MOVi 3, 14 /* CC::al */, $noreg, $noreg ; CHECK: MOVPCLR 14 /* CC::al */, $noreg + ; CHECK-LABEL: name: OUTLINED_FUNCTION_2 + ; CHECK: bb.0: + ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 + ; CHECK: $r0 = MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r1 = MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r2 = MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r3 = MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r4 = MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: MOVPCLR 14 /* CC::al */, $noreg + ; CHECK-LABEL: name: OUTLINED_FUNCTION_3 ; CHECK: bb.0: - ; CHECK: liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr - ; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -8 - ; CHECK: tBL 14 /* CC::al */, $noreg, @bar, implicit-def dead $lr, implicit $sp + ; CHECK: liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 ; CHECK: $r0 = t2MOVi 2, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r1 = t2MOVi 2, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r2 = t2MOVi 2, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg ; CHECK: tTAILJMPdND @bar, 14 /* CC::al */, $noreg, implicit $sp ; CHECK-LABEL: name: OUTLINED_FUNCTION_4 ; CHECK: bb.0: - ; CHECK: liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr - ; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -8 - ; CHECK: tBL 14 /* CC::al */, $noreg, @bar, implicit-def dead $lr, implicit $sp + ; CHECK: liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 ; CHECK: $r0 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r1 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r2 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg ; CHECK: tBX_RET 14 /* CC::al */, $noreg - - - diff --git a/llvm/test/CodeGen/ARM/machine-outliner-default.mir b/llvm/test/CodeGen/ARM/machine-outliner-default.mir index 6d0218dbfe63..de2b8f559697 100644 --- a/llvm/test/CodeGen/ARM/machine-outliner-default.mir +++ b/llvm/test/CodeGen/ARM/machine-outliner-default.mir @@ -19,17 +19,17 @@ body: | ; CHECK: bb.0: ; CHECK: liveins: $lr ; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg ; CHECK: bb.1: ; CHECK: liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11 ; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg ; CHECK: bb.2: ; CHECK: liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11 ; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg ; CHECK: bb.3: ; CHECK: liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11 @@ -73,17 +73,17 @@ body: | ; CHECK: bb.0: ; CHECK: liveins: $lr ; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_1 + ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0 ; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg ; CHECK: bb.1: ; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11 ; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_1 + ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0 ; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg ; CHECK: bb.2: ; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11 ; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_1 + ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0 ; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg ; CHECK: bb.3: ; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11 @@ -114,6 +114,15 @@ body: | ; CHECK-LABEL: name: OUTLINED_FUNCTION_0 ; CHECK: bb.0: + ; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11 + ; CHECK: $r0 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r1 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r2 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r3 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: tBX_RET 14 /* CC::al */, $noreg + + ; CHECK-LABEL: name: OUTLINED_FUNCTION_1 + ; CHECK: bb.0: ; CHECK: liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11 ; CHECK: $r0 = MOVi 1, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r1 = MOVi 1, 14 /* CC::al */, $noreg, $noreg @@ -122,15 +131,3 @@ body: | ; CHECK: $r4 = MOVi 1, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r5 = MOVi 1, 14 /* CC::al */, $noreg, $noreg ; CHECK: MOVPCLR 14 /* CC::al */, $noreg - - ; CHECK-LABEL: name: OUTLINED_FUNCTION_1 - ; CHECK: bb.0: - ; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11 - ; CHECK: $r0 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r1 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r2 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r3 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: tBX_RET 14 /* CC::al */, $noreg - - - diff --git a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir index ae5caa5b7c06..e71edc8ceb3f 100644 --- a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir +++ b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir @@ -18,6 +18,7 @@ body: | liveins: $r0 ; CHECK-LABEL: name: CheckAddrMode_i12 ; CHECK: $r1 = MOVr killed $r0, 14 /* CC::al */, $noreg, $noreg + ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp ; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I12:[0-9]+]] ; CHECK-NEXT: $r6 = LDRi12 $sp, 4088, 14 /* CC::al */, $noreg $r1 = MOVr killed $r0, 14, $noreg, $noreg @@ -47,6 +48,7 @@ body: | liveins: $r1 ; CHECK-LABEL: name: CheckAddrMode3 ; CHECK: $r0 = MOVr killed $r1, 14 /* CC::al */, $noreg, $noreg + ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp ; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I3:[0-9]+]] ; CHECK-NEXT: $r6 = LDRSH $sp, $noreg, 248, 14 /* CC::al */, $noreg $r0 = MOVr killed $r1, 14, $noreg, $noreg @@ -76,6 +78,7 @@ body: | liveins: $r2 ; CHECK-LABEL: name: CheckAddrMode5 ; CHECK: $r0 = MOVr killed $r2, 14 /* CC::al */, $noreg, $noreg + ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp ; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I5:[0-9]+]] ; CHECK-NEXT: $d5 = VLDRD $sp, 254, 14 /* CC::al */, $noreg $r0 = MOVr killed $r2, 14, $noreg, $noreg @@ -110,6 +113,7 @@ body: | liveins: $r3 ; CHECK-LABEL: name: CheckAddrMode5FP16 ; CHECK: $r0 = MOVr killed $r3, 14 /* CC::al */, $noreg, $noreg + ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp ; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I5FP16:[0-9]+]] ; CHECK-NEXT: $s6 = VLDRH $sp, 252, 14, $noreg $r0 = MOVr killed $r3, 14, $noreg, $noreg @@ -146,41 +150,29 @@ body: | BX_RET 14, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[I5]] - ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: $d0 = VLDRD $sp, 2, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $d1 = VLDRD $sp, 10, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $d4 = VLDRD $sp, 255, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg + ;CHECK: liveins: $r10, $r9, $r8, $r7, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 + ;CHECK: $d0 = VLDRD $sp, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $d1 = VLDRD $sp, 8, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $d4 = VLDRD $sp, 253, 14 /* CC::al */, $noreg + ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[I5FP16]] - ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: $s1 = VLDRH $sp, 4, 14, $noreg - ;CHECK-NEXT: $s2 = VLDRH $sp, 12, 14, $noreg - ;CHECK-NEXT: $s5 = VLDRH $sp, 244, 14, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg + ;CHECK: liveins: $r10, $r9, $r8, $r7, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 + ;CHECK: $s1 = VLDRH $sp, 0, 14, $noreg + ;CHECK-NEXT: $s2 = VLDRH $sp, 8, 14, $noreg + ;CHECK-NEXT: $s5 = VLDRH $sp, 240, 14, $noreg + ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[I12]] - ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: $r1 = LDRi12 $sp, 8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r2 = LDRi12 $sp, 16, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r5 = LDRi12 $sp, 4094, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg + ;CHECK: liveins: $r10, $r9, $r8, $r7, $d8, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9 + ;CHECK: $r1 = LDRi12 $sp, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r2 = LDRi12 $sp, 8, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r5 = LDRi12 $sp, 4086, 14 /* CC::al */, $noreg + ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[I3]] - ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: $r1 = LDRSH $sp, $noreg, 8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r2 = LDRSH $sp, $noreg, 16, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r5 = LDRSH $sp, $noreg, 255, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg + ;CHECK: liveins: $r10, $r9, $r8, $r7, $d8, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9 + ;CHECK: $r1 = LDRSH $sp, $noreg, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r2 = LDRSH $sp, $noreg, 8, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r5 = LDRSH $sp, $noreg, 247, 14 /* CC::al */, $noreg + ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg diff --git a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir index 561844484244..f2cdaebdfa8b 100644 --- a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir +++ b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir @@ -19,7 +19,7 @@ body: | bb.0: liveins: $r1 ;CHECK-LABEL: name: CheckAddrModeT2_i12 - ;CHECK: $r0 = tMOVr killed $r1, 14 /* CC::al */, $noreg + ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]] ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[I12:[0-9]+]] ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4088, 14 /* CC::al */, $noreg $r0 = tMOVr killed $r1, 14, $noreg @@ -49,7 +49,7 @@ body: | bb.0: liveins: $r1 ;CHECK-LABEL: name: CheckAddrModeT2_i8 - ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg + ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]] ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[I8:[0-9]+]] ;CHECK-NEXT: t2STRHT $r0, $sp, 248, 14 /* CC::al */, $noreg $r0 = tMOVr $r1, 14, $noreg @@ -79,7 +79,7 @@ body: | bb.0: liveins: $r1 ;CHECK-LABEL: name: CheckAddrModeT2_i8s4 - ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg + ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]] ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[I8S4:[0-9]+]] ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 1020, 14 /* CC::al */, $noreg $r0 = tMOVr $r1, 14, $noreg @@ -109,7 +109,7 @@ body: | bb.0: liveins: $r1 ;CHECK-LABEL: name: CheckAddrModeT2_ldrex - ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg + ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]] ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[LDREX:[0-9]+]] ;CHECK-NEXT: $r1 = t2LDREX $sp, 254, 14 /* CC::al */, $noreg $r0 = tMOVr $r1, 14, $noreg @@ -144,8 +144,10 @@ body: | bb.0: liveins: $r0, $r1 ;CHECK-LABEL: name: CheckAddrModeT1_s - ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg - ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[T1_S:[0-9]+]] + ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]] + ;CHECK-NEXT: tSTRspi $r0, $sp, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tSTRspi $r0, $sp, 4, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tSTRspi $r0, $sp, 253, 14 /* CC::al */, $noreg ;CHECK-NEXT: tSTRspi $r0, $sp, 254, 14 /* CC::al */, $noreg $r0 = tMOVr $r1, 14, $noreg tBL 14, $noreg, @foo, implicit-def dead $lr, implicit $sp @@ -181,51 +183,29 @@ body: | BX_RET 14, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[LDREX]] - ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: $r1 = t2LDREX $sp, 2, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r1 = t2LDREX $sp, 10, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r1 = t2LDREX $sp, 255, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg + ;CHECK: $r1 = t2LDREX $sp, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r1 = t2LDREX $sp, 8, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r1 = t2LDREX $sp, 253, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[I8]] - ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: t2STRHT $r0, $sp, 8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: t2STRHT $r0, $sp, 12, 14 /* CC::al */, $noreg - ;CHECK-NEXT: t2STRHT $r0, $sp, 255, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg + ;CHECK: t2STRHT $r0, $sp, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: t2STRHT $r0, $sp, 4, 14 /* CC::al */, $noreg + ;CHECK-NEXT: t2STRHT $r0, $sp, 247, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[I8S4]] - ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp + ;CHECK: t2STRDi8 $r0, $r1, $sp, 0, 14 /* CC::al */, $noreg ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 16, 14 /* CC::al */, $noreg - ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 1020, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg + ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 1012, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $nore ;CHECK: name: OUTLINED_FUNCTION_[[I12]] - ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 12, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4094, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg - - ;CHECK: name: OUTLINED_FUNCTION_[[T1_S]] - ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: tSTRspi $r0, $sp, 2, 14 /* CC::al */, $noreg - ;CHECK-NEXT: tSTRspi $r0, $sp, 6, 14 /* CC::al */, $noreg - ;CHECK-NEXT: tSTRspi $r0, $sp, 255, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg + ;CHECK: $r0 = t2LDRi12 $sp, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4086, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $noreg + + ;CHECK: name: OUTLINED_FUNCTION_[[SHARED]] + ;CHECK: $r0 = tMOVr killed $r1, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tTAILJMPdND @foo, 14 /* CC::al */, $noreg, implicit $sp -- GitLab From eb33e462ba536735b3d8449a81009a253f0f43bc Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Fri, 7 Jun 2024 07:04:17 -0700 Subject: [PATCH 253/462] [memprof] Clean up IndexedMemProfReader (NFC) (#94710) Parameter "Version" is confusing in deserializeV012 and deserializeV3 because we also have member variable "Version". Fortunately, parameter "Version" and member variable "Version" always have the same value because IndexedMemProfReader::deserialize initializes the member variable and passes it to deserializeV012 and deserializeV3. This patch removes the parameter. --- llvm/include/llvm/ProfileData/InstrProfReader.h | 5 ++--- llvm/lib/ProfileData/InstrProfReader.cpp | 12 +++++------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 1c3d4c795f63..34dba870d8da 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -665,9 +665,8 @@ private: const unsigned char *CallStackBase = nullptr; Error deserializeV012(const unsigned char *Start, const unsigned char *Ptr, - uint64_t FirstWord, memprof::IndexedVersion Version); - Error deserializeV3(const unsigned char *Start, const unsigned char *Ptr, - memprof::IndexedVersion Version); + uint64_t FirstWord); + Error deserializeV3(const unsigned char *Start, const unsigned char *Ptr); public: IndexedMemProfReader() = default; diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 8298d3d37295..7758363d9c95 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -1204,8 +1204,7 @@ IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version, Error IndexedMemProfReader::deserializeV012(const unsigned char *Start, const unsigned char *Ptr, - uint64_t FirstWord, - memprof::IndexedVersion Version) { + uint64_t FirstWord) { // The value returned from RecordTableGenerator.Emit. const uint64_t RecordTableOffset = Version == memprof::Version0 @@ -1259,8 +1258,7 @@ Error IndexedMemProfReader::deserializeV012(const unsigned char *Start, } Error IndexedMemProfReader::deserializeV3(const unsigned char *Start, - const unsigned char *Ptr, - memprof::IndexedVersion Version) { + const unsigned char *Ptr) { // The offset in the stream right before invoking // CallStackTableGenerator.Emit. const uint64_t CallStackPayloadOffset = @@ -1285,7 +1283,7 @@ Error IndexedMemProfReader::deserializeV3(const unsigned char *Start, MemProfRecordTable.reset(MemProfRecordHashTable::Create( /*Buckets=*/Start + RecordTableOffset, /*Payload=*/Start + RecordPayloadOffset, - /*Base=*/Start, memprof::RecordLookupTrait(Version, Schema))); + /*Base=*/Start, memprof::RecordLookupTrait(memprof::Version3, Schema))); return Error::success(); } @@ -1323,11 +1321,11 @@ Error IndexedMemProfReader::deserialize(const unsigned char *Start, case memprof::Version0: case memprof::Version1: case memprof::Version2: - if (Error E = deserializeV012(Start, Ptr, FirstWord, Version)) + if (Error E = deserializeV012(Start, Ptr, FirstWord)) return E; break; case memprof::Version3: - if (Error E = deserializeV3(Start, Ptr, Version)) + if (Error E = deserializeV3(Start, Ptr)) return E; break; } -- GitLab From 55bdb36e39670d07aaaf04a24783a2008e8e60dd Mon Sep 17 00:00:00 2001 From: jeanPerier Date: Fri, 7 Jun 2024 16:09:56 +0200 Subject: [PATCH 254/462] [flang] lower SIZE and SIZEOF for assumed-ranks (#94684) --- .../flang/Optimizer/Builder/BoxValue.h | 12 +- flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 19 ++- .../Lower/HLFIR/assumed-rank-inquiries-2.f90 | 108 ++++++++++++++++++ 3 files changed, 128 insertions(+), 11 deletions(-) create mode 100644 flang/test/Lower/HLFIR/assumed-rank-inquiries-2.f90 diff --git a/flang/include/flang/Optimizer/Builder/BoxValue.h b/flang/include/flang/Optimizer/Builder/BoxValue.h index 5c7e89dbc08f..9fdaa0b19717 100644 --- a/flang/include/flang/Optimizer/Builder/BoxValue.h +++ b/flang/include/flang/Optimizer/Builder/BoxValue.h @@ -433,7 +433,8 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &, const ExtendedValue &); /// substituted. ExtendedValue substBase(const ExtendedValue &exv, mlir::Value base); -/// Is the extended value `exv` an array? +/// Is the extended value `exv` an array? Note that this returns true for +/// assumed-ranks that could actually be scalars at runtime. bool isArray(const ExtendedValue &exv); /// Get the type parameters for `exv`. @@ -527,6 +528,15 @@ public: [](const auto &box) -> bool { return false; }); } + bool hasAssumedRank() const { + return match( + [](const fir::BoxValue &box) -> bool { return box.hasAssumedRank(); }, + [](const fir::MutableBoxValue &box) -> bool { + return box.hasAssumedRank(); + }, + [](const auto &box) -> bool { return false; }); + } + /// LLVM style debugging of extended values LLVM_DUMP_METHOD void dump() const { llvm::errs() << *this << '\n'; } diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp index d3f6fa16ac80..861b26de0637 100644 --- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -6121,9 +6121,6 @@ IntrinsicLibrary::genSize(mlir::Type resultType, // Note that the value of the KIND argument is already reflected in the // resultType assert(args.size() == 3); - if (const auto *boxValue = args[0].getBoxOf()) - if (boxValue->hasAssumedRank()) - TODO(loc, "intrinsic: size with assumed rank argument"); // Get the ARRAY argument mlir::Value array = builder.createBox(loc, args[0]); @@ -6137,13 +6134,15 @@ IntrinsicLibrary::genSize(mlir::Type resultType, // Get the DIM argument. mlir::Value dim = fir::getBase(args[1]); - if (std::optional cstDim = fir::getIntIfConstant(dim)) { - // If it is a compile time constant, skip the runtime call. - return builder.createConvert(loc, resultType, - fir::factory::readExtent(builder, loc, - fir::BoxValue{array}, - cstDim.value() - 1)); - } + if (!args[0].hasAssumedRank()) + if (std::optional cstDim = fir::getIntIfConstant(dim)) { + // If both DIM and the rank are compile time constants, skip the runtime + // call. + return builder.createConvert( + loc, resultType, + fir::factory::readExtent(builder, loc, fir::BoxValue{array}, + cstDim.value() - 1)); + } if (!fir::isa_ref_type(dim.getType())) return builder.createConvert( loc, resultType, fir::runtime::genSizeDim(builder, loc, array, dim)); diff --git a/flang/test/Lower/HLFIR/assumed-rank-inquiries-2.f90 b/flang/test/Lower/HLFIR/assumed-rank-inquiries-2.f90 new file mode 100644 index 000000000000..353e944a8ac1 --- /dev/null +++ b/flang/test/Lower/HLFIR/assumed-rank-inquiries-2.f90 @@ -0,0 +1,108 @@ +! Test lowering of SIZE/SIZEOF inquiry intrinsics with assumed-ranks +! arguments. +! RUN: bbc -emit-hlfir -o - %s -allow-assumed-rank | FileCheck %s + + +subroutine test_size_1(x) + real :: x(..) + call takes_integer(size(x)) +end subroutine + +subroutine test_size_2(x) + real :: x(..) + call takes_integer(size(x, 2)) +end subroutine + +subroutine test_size_3(x, d) + real :: x(..) + integer, optional :: d + call takes_integer(size(x, d)) +end subroutine + +subroutine test_size_4(x) + real, allocatable :: x(..) + call takes_integer(size(x)) +end subroutine + + +! CHECK-LABEL: func.func @_QPtest_size_1( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFtest_size_1Ex"} : (!fir.box>, !fir.dscope) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_5:.*]] = fir.convert %[[VAL_2]]#1 : (!fir.box>) -> !fir.box +! CHECK: %[[VAL_7:.*]] = fir.call @_FortranASize(%[[VAL_5]] +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i64) -> i32 +! CHECK: %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (i32) -> (!fir.ref, !fir.ref, i1) +! CHECK: fir.call @_QPtakes_integer(%[[VAL_9]]#1) fastmath : (!fir.ref) -> () +! CHECK: hlfir.end_associate %[[VAL_9]]#1, %[[VAL_9]]#2 : !fir.ref, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_size_2( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 +! CHECK: %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {uniq_name = "_QFtest_size_2Ex"} : (!fir.box>, !fir.dscope) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_4:.*]] = arith.constant 2 : i32 +! CHECK: fir.store %[[VAL_4]] to %[[VAL_1]] : !fir.ref +! CHECK: %[[VAL_5:.*]] = fir.convert %[[VAL_1]] : (!fir.ref) -> i64 +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i64 +! CHECK: %[[VAL_7:.*]] = arith.cmpi eq, %[[VAL_5]], %[[VAL_6]] : i64 +! CHECK: %[[VAL_8:.*]] = fir.if %[[VAL_7]] -> (i32) { +! CHECK: %[[VAL_11:.*]] = fir.convert %[[VAL_3]]#1 : (!fir.box>) -> !fir.box +! CHECK: %[[VAL_13:.*]] = fir.call @_FortranASize(%[[VAL_11]] +! CHECK: %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (i64) -> i32 +! CHECK: fir.result %[[VAL_14]] : i32 +! CHECK: } else { +! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_1]] : !fir.ref +! CHECK: %[[VAL_18:.*]] = fir.convert %[[VAL_3]]#1 : (!fir.box>) -> !fir.box +! CHECK: %[[VAL_20:.*]] = fir.call @_FortranASizeDim(%[[VAL_18]] +! CHECK: %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (i64) -> i32 +! CHECK: fir.result %[[VAL_21]] : i32 +! CHECK: } +! CHECK: %[[VAL_22:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (i32) -> (!fir.ref, !fir.ref, i1) +! CHECK: fir.call @_QPtakes_integer(%[[VAL_22]]#1) fastmath : (!fir.ref) -> () +! CHECK: hlfir.end_associate %[[VAL_22]]#1, %[[VAL_22]]#2 : !fir.ref, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_size_3( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "x"}, +! CHECK-SAME: %[[VAL_1:.*]]: !fir.ref {fir.bindc_name = "d", fir.optional}) { +! CHECK: %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_2]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_size_3Ed"} : (!fir.ref, !fir.dscope) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_2]] {uniq_name = "_QFtest_size_3Ex"} : (!fir.box>, !fir.dscope) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_5:.*]] = fir.convert %[[VAL_3]]#1 : (!fir.ref) -> i64 +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i64 +! CHECK: %[[VAL_7:.*]] = arith.cmpi eq, %[[VAL_5]], %[[VAL_6]] : i64 +! CHECK: %[[VAL_8:.*]] = fir.if %[[VAL_7]] -> (i32) { +! CHECK: %[[VAL_11:.*]] = fir.convert %[[VAL_4]]#1 : (!fir.box>) -> !fir.box +! CHECK: %[[VAL_13:.*]] = fir.call @_FortranASize(%[[VAL_11]], +! CHECK: %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (i64) -> i32 +! CHECK: fir.result %[[VAL_14]] : i32 +! CHECK: } else { +! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref +! CHECK: %[[VAL_18:.*]] = fir.convert %[[VAL_4]]#1 : (!fir.box>) -> !fir.box +! CHECK: %[[VAL_20:.*]] = fir.call @_FortranASizeDim(%[[VAL_18]] +! CHECK: %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (i64) -> i32 +! CHECK: fir.result %[[VAL_21]] : i32 +! CHECK: } +! CHECK: %[[VAL_22:.*]]:3 = hlfir.associate %[[VAL_8]] {adapt.valuebyref} : (i32) -> (!fir.ref, !fir.ref, i1) +! CHECK: fir.call @_QPtakes_integer(%[[VAL_22]]#1) fastmath : (!fir.ref) -> () +! CHECK: hlfir.end_associate %[[VAL_22]]#1, %[[VAL_22]]#2 : !fir.ref, i1 +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest_size_4( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_size_4Ex"} : (!fir.ref>>>, !fir.dscope) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]]#1 : !fir.ref>>> +! CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_3]] : (!fir.box>>) -> !fir.box +! CHECK: %[[VAL_8:.*]] = fir.call @_FortranASize(%[[VAL_6]] +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i64) -> i32 +! CHECK: %[[VAL_10:.*]]:3 = hlfir.associate %[[VAL_9]] {adapt.valuebyref} : (i32) -> (!fir.ref, !fir.ref, i1) +! CHECK: fir.call @_QPtakes_integer(%[[VAL_10]]#1) fastmath : (!fir.ref) -> () +! CHECK: hlfir.end_associate %[[VAL_10]]#1, %[[VAL_10]]#2 : !fir.ref, i1 +! CHECK: return +! CHECK: } -- GitLab From c348e265bd1284f770e66639633199fefd8015ec Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Fri, 7 Jun 2024 07:19:36 -0700 Subject: [PATCH 255/462] [memprof] Use CallStackRadixTreeBuilder in the V3 format (#94708) This patch integrates CallStackRadixTreeBuilder into the V3 format, reducing the profile size to about 27% of the V2 profile size. - Serialization: writeMemProfCallStackArray just needs to write out the radix tree array prepared by CallStackRadixTreeBuilder. Mappings from CallStackIds to LinearCallStackIds are moved by new function CallStackRadixTreeBuilder::takeCallStackPos. - Deserialization: Deserializing a call stack is the same as deserializing an array encoded in the obvious manner -- the length followed by the payload, except that we need to follow a pointer to the parent to take advantage of common prefixes once in a while. This patch teaches LinearCallStackIdConverter to how to handle those pointers. --- llvm/include/llvm/ProfileData/MemProf.h | 18 +++++++++++++----- llvm/lib/ProfileData/InstrProfWriter.cpp | 22 +++++++--------------- llvm/unittests/ProfileData/MemProfTest.cpp | 8 ++++---- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h index 6b0fa6cd6541..0e6245acb77e 100644 --- a/llvm/include/llvm/ProfileData/MemProf.h +++ b/llvm/include/llvm/ProfileData/MemProf.h @@ -900,9 +900,18 @@ struct LinearCallStackIdConverter { Frames.reserve(NumFrames); for (; NumFrames; --NumFrames) { LinearFrameId Elem = - support::endian::readNext( - Ptr); + support::endian::read(Ptr); + // Follow a pointer to the parent, if any. See comments below on + // CallStackRadixTreeBuilder for the description of the radix tree format. + if (static_cast>(Elem) < 0) { + Ptr += (-Elem) * sizeof(LinearFrameId); + Elem = + support::endian::read(Ptr); + } + // We shouldn't encounter another pointer. + assert(static_cast>(Elem) >= 0); Frames.push_back(FrameIdToFrame(Elem)); + Ptr += sizeof(LinearFrameId); } return Frames; @@ -1024,9 +1033,8 @@ public: const std::vector &getRadixArray() const { return RadixArray; } - const llvm::DenseMap & - getCallStackPos() const { - return CallStackPos; + llvm::DenseMap takeCallStackPos() { + return std::move(CallStackPos); } }; diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index e58e6b8acfc8..a73f72a534f1 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -547,19 +547,11 @@ writeMemProfCallStackArray( llvm::DenseMap MemProfCallStackIndexes; - MemProfCallStackIndexes.reserve(MemProfCallStackData.size()); - uint64_t CallStackBase = OS.tell(); - for (const auto &[CSId, CallStack] : MemProfCallStackData) { - memprof::LinearCallStackId CallStackIndex = - (OS.tell() - CallStackBase) / sizeof(memprof::LinearCallStackId); - MemProfCallStackIndexes.insert({CSId, CallStackIndex}); - const llvm::SmallVector CS = CallStack; - OS.write32(CS.size()); - for (const auto F : CS) { - assert(MemProfFrameIndexes.contains(F)); - OS.write32(MemProfFrameIndexes[F]); - } - } + memprof::CallStackRadixTreeBuilder Builder; + Builder.build(std::move(MemProfCallStackData), MemProfFrameIndexes); + for (auto I : Builder.getRadixArray()) + OS.write32(I); + MemProfCallStackIndexes = Builder.takeCallStackPos(); // Release the memory of this vector as it is no longer needed. MemProfCallStackData.clear(); @@ -695,8 +687,8 @@ static Error writeMemProfV2(ProfOStream &OS, // uint64_t Schema entry 1 // .... // uint64_t Schema entry N - 1 -// OnDiskChainedHashTable MemProfFrameData -// OnDiskChainedHashTable MemProfCallStackData +// Frames serialized one after another +// Call stacks encoded as a radix tree // OnDiskChainedHashTable MemProfRecordData static Error writeMemProfV3(ProfOStream &OS, memprof::IndexedMemProfData &MemProfData, diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp index e120bc3e3594..26421200e1a1 100644 --- a/llvm/unittests/ProfileData/MemProfTest.cpp +++ b/llvm/unittests/ProfileData/MemProfTest.cpp @@ -670,7 +670,7 @@ TEST(MemProf, RadixTreeBuilderEmpty) { llvm::memprof::CallStackRadixTreeBuilder Builder; Builder.build(std::move(MemProfCallStackData), MemProfFrameIndexes); ASSERT_THAT(Builder.getRadixArray(), testing::IsEmpty()); - const auto &Mappings = Builder.getCallStackPos(); + const auto Mappings = Builder.takeCallStackPos(); ASSERT_THAT(Mappings, testing::IsEmpty()); } @@ -689,7 +689,7 @@ TEST(MemProf, RadixTreeBuilderOne) { 2U, // MemProfFrameIndexes[12] 1U // MemProfFrameIndexes[11] })); - const auto &Mappings = Builder.getCallStackPos(); + const auto Mappings = Builder.takeCallStackPos(); ASSERT_THAT(Mappings, SizeIs(1)); EXPECT_THAT(Mappings, testing::Contains(testing::Pair( llvm::memprof::hashCallStack(CS1), 0U))); @@ -715,7 +715,7 @@ TEST(MemProf, RadixTreeBuilderTwo) { 2U, // MemProfFrameIndexes[12] 1U // MemProfFrameIndexes[11] })); - const auto &Mappings = Builder.getCallStackPos(); + const auto Mappings = Builder.takeCallStackPos(); ASSERT_THAT(Mappings, SizeIs(2)); EXPECT_THAT(Mappings, testing::Contains(testing::Pair( llvm::memprof::hashCallStack(CS1), 0U))); @@ -758,7 +758,7 @@ TEST(MemProf, RadixTreeBuilderSuccessiveJumps) { 2U, // MemProfFrameIndexes[12] 1U // MemProfFrameIndexes[11] })); - const auto &Mappings = Builder.getCallStackPos(); + const auto Mappings = Builder.takeCallStackPos(); ASSERT_THAT(Mappings, SizeIs(4)); EXPECT_THAT(Mappings, testing::Contains(testing::Pair( llvm::memprof::hashCallStack(CS1), 0U))); -- GitLab From 7d69095fd50b7ef35282f83a9263bb8027ad521b Mon Sep 17 00:00:00 2001 From: Mubashar Ahmad Date: Fri, 7 Jun 2024 15:38:50 +0100 Subject: [PATCH 256/462] [mlir][vector] Remove Emulated Sub-directory (#94742) The "Emulated" sub-directories under "ArmSVE" and "ArmSME" have been removed. Associated tests have been moved up a directory and now include the "REQUIRES" constraint for the arm-emulator. --- .../Dialect/Vector/CPU/ArmSME/Emulated/lit.local.cfg | 5 ----- .../Vector/CPU/ArmSME/{Emulated => }/test-setArmSVLBits.mlir | 2 ++ .../Dialect/Vector/CPU/ArmSVE/Emulated/lit.local.cfg | 5 ----- .../ArmSVE/{Emulated => }/test-scalable-deinterleave.mlir | 2 ++ .../Vector/CPU/ArmSVE/{Emulated => }/test-setArmVLBits.mlir | 2 ++ mlir/test/lit.cfg.py | 3 +++ 6 files changed, 9 insertions(+), 10 deletions(-) delete mode 100644 mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/lit.local.cfg rename mlir/test/Integration/Dialect/Vector/CPU/ArmSME/{Emulated => }/test-setArmSVLBits.mlir (98%) delete mode 100644 mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/lit.local.cfg rename mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/{Emulated => }/test-scalable-deinterleave.mlir (97%) rename mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/{Emulated => }/test-setArmVLBits.mlir (97%) diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/lit.local.cfg b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/lit.local.cfg deleted file mode 100644 index 0d8ad605f598..000000000000 --- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/lit.local.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# The tests in this folder assume full control of the hardware features, such as -# the vector length, so must be run under an emulator. - -if not config.arm_emulator_executable: - config.unsupported = True diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/test-setArmSVLBits.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir similarity index 98% rename from mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/test-setArmSVLBits.mlir rename to mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir index 0648e771b889..4c02bcb9d192 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/test-setArmSVLBits.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir @@ -1,3 +1,5 @@ +// REQUIRES: arm-emulator + // DEFINE: %{entry_point} = main // DEFINE: %{compile} = mlir-opt %s \ // DEFINE: --pass-pipeline="builtin.module(func.func(convert-arm-sme-to-llvm),test-lower-to-llvm)" diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/lit.local.cfg b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/lit.local.cfg deleted file mode 100644 index 0d8ad605f598..000000000000 --- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/lit.local.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# The tests in this folder assume full control of the hardware features, such as -# the vector length, so must be run under an emulator. - -if not config.arm_emulator_executable: - config.unsupported = True diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/test-scalable-deinterleave.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir similarity index 97% rename from mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/test-scalable-deinterleave.mlir rename to mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir index e6c561437132..3ef41b0bf5cf 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/test-scalable-deinterleave.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir @@ -1,3 +1,5 @@ +// REQUIRES: arm-emulator + // DEFINE: %{entry_point} = entry // DEFINE: %{compile} = mlir-opt %s -test-lower-to-llvm // DEFINE: %{run} = %mcr_aarch64_cmd -march=aarch64 -mattr=+sve \ diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/test-setArmVLBits.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir similarity index 97% rename from mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/test-setArmVLBits.mlir rename to mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir index aa8d0e4d5104..6e25d9d16b8b 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/Emulated/test-setArmVLBits.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir @@ -1,3 +1,5 @@ +// REQUIRES: arm-emulator + // DEFINE: %{entry_point} = main // DEFINE: %{compile} = mlir-opt %s -test-lower-to-llvm // DEFINE: %{run} = %mcr_aarch64_cmd -march=aarch64 -mattr=+sve \ diff --git a/mlir/test/lit.cfg.py b/mlir/test/lit.cfg.py index 9ed3a2efcb8f..9838a02136d1 100644 --- a/mlir/test/lit.cfg.py +++ b/mlir/test/lit.cfg.py @@ -250,3 +250,6 @@ if config.run_nvptx_tests: if config.run_rocm_tests: config.available_features.add("host-supports-amdgpu") + +if config.arm_emulator_executable: + config.available_features.add("arm-emulator") -- GitLab From d099d6c76b3216b30b30de22cccf155fcc9e1c51 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Fri, 7 Jun 2024 10:42:27 -0400 Subject: [PATCH 257/462] [gn] port 33a6ce18373ff (check-clang obj2yaml dep) --- llvm/utils/gn/secondary/clang/test/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn index 2575dbf3cd0d..1ec94a419f56 100644 --- a/llvm/utils/gn/secondary/clang/test/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn @@ -201,6 +201,7 @@ group("test") { "//llvm/tools/llvm-readobj:symlinks", "//llvm/tools/llvm-readtapi:symlinks", "//llvm/tools/llvm-symbolizer:symlinks", + "//llvm/tools/obj2yaml", "//llvm/tools/opt", "//llvm/tools/yaml2obj", "//llvm/utils/FileCheck", -- GitLab From fc95645e37f244c2fc155f1ee51047f90329e8c1 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Fri, 7 Jun 2024 10:47:51 -0400 Subject: [PATCH 258/462] [gn] port cb7690af09b95 (ntdll dep) --- llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn | 5 ++++- llvm/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn index 0d4725716191..ee5905ae5a7d 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn @@ -190,7 +190,10 @@ static_library("Support") { if (current_os == "linux" || current_os == "android") { libs += [ "dl" ] } else if (current_os == "win") { - libs = [ "ws2_32.lib" ] + libs = [ + "ntdll.lib", + "ws2_32.lib", + ] # Delay load shell32.dll if possible to speed up process startup. libs += [ "delayimp.lib" ] diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn index 711e4e3b4315..ddf0e55b4f00 100644 --- a/llvm/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn @@ -46,7 +46,7 @@ write_cmake_config("BuildVariables.inc") { if (host_os == "win") { # uuid.lib for FOLDERID_Profile in lib/Support/Windows/Path.inc. # advapi32.lib for CryptAcquireContextW in lib/Support/Windows/Path.inc. - system_libs = "psapi.lib shell32.lib ole32.lib uuid.lib advapi32.lib ws2_32.lib" + system_libs = "psapi.lib shell32.lib ole32.lib uuid.lib advapi32.lib ws2_32.lib ntdll.lib" } else { system_libs += "-lm" if (host_os == "linux") { -- GitLab From b25b1db8199d86cb3645e92200cda8d5d30922d0 Mon Sep 17 00:00:00 2001 From: c8ef Date: Fri, 7 Jun 2024 23:01:22 +0800 Subject: [PATCH 259/462] [KnownBits] Remove `hasConflict()` assertions (#94568) Allow KnownBits to represent "always poison" values via conflict. close: #94436 --- llvm/include/llvm/Support/KnownBits.h | 11 +-- llvm/lib/Analysis/ValueTracking.cpp | 3 - .../lib/CodeGen/GlobalISel/GISelKnownBits.cpp | 1 - .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 1 - .../CodeGen/SelectionDAG/TargetLowering.cpp | 22 +----- llvm/lib/IR/ConstantRange.cpp | 4 +- llvm/lib/Support/KnownBits.cpp | 33 ++------ llvm/lib/Target/X86/X86ISelLowering.cpp | 5 -- .../InstCombineSimplifyDemanded.cpp | 19 +---- llvm/unittests/IR/DemandedBitsTest.cpp | 6 +- llvm/unittests/Support/KnownBitsTest.cpp | 77 +++++++++++-------- llvm/unittests/Support/KnownBitsTest.h | 3 - 12 files changed, 61 insertions(+), 124 deletions(-) diff --git a/llvm/include/llvm/Support/KnownBits.h b/llvm/include/llvm/Support/KnownBits.h index ba4a5f01036c..c206bd77d588 100644 --- a/llvm/include/llvm/Support/KnownBits.h +++ b/llvm/include/llvm/Support/KnownBits.h @@ -48,7 +48,6 @@ public: /// Returns true if we know the value of all bits. bool isConstant() const { - assert(!hasConflict() && "KnownBits conflict!"); return Zero.popcount() + One.popcount() == getBitWidth(); } @@ -74,16 +73,10 @@ public: } /// Returns true if value is all zero. - bool isZero() const { - assert(!hasConflict() && "KnownBits conflict!"); - return Zero.isAllOnes(); - } + bool isZero() const { return Zero.isAllOnes(); } /// Returns true if value is all one bits. - bool isAllOnes() const { - assert(!hasConflict() && "KnownBits conflict!"); - return One.isAllOnes(); - } + bool isAllOnes() const { return One.isAllOnes(); } /// Make all bits known to be zero and discard any previous information. void setAllZero() { diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 782c28c94483..6a806d596efe 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -1158,7 +1158,6 @@ static void computeKnownBitsFromOperator(const Operator *I, Known.makeNonNegative(); } - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); break; } @@ -2055,8 +2054,6 @@ void computeKnownBits(const Value *V, const APInt &DemandedElts, // Check whether we can determine known bits from context such as assumes. computeKnownBitsFromContext(V, Known, Depth, Q); - - assert((Known.Zero & Known.One) == 0 && "Bits known to be one AND zero?"); } /// Try to detect a recurrence that the value of the induction variable is diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp index 32d607cfd71a..2988fa45b2e3 100644 --- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp +++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp @@ -607,7 +607,6 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known, } } - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); LLVM_DEBUG(dumpResult(MI, Known, Depth)); // Update the cache. diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index e176cf2cc2a6..ddb8a0ee8179 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -4212,7 +4212,6 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, break; } - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); return Known; } diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index e1c1a6b09b11..c8a5e1525760 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -1436,11 +1436,9 @@ bool TargetLowering::SimplifyDemandedBits( if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts, Known2, TLO, Depth + 1)) return true; - assert(!Known2.hasConflict() && "Bits known to be one AND zero?"); // If all of the demanded bits are known one on one side, return the other. // These bits cannot contribute to the result of the 'and'. @@ -1488,7 +1486,7 @@ bool TargetLowering::SimplifyDemandedBits( } return true; } - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); + if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts, Known2, TLO, Depth + 1)) { if (Flags.hasDisjoint()) { @@ -1497,7 +1495,6 @@ bool TargetLowering::SimplifyDemandedBits( } return true; } - assert(!Known2.hasConflict() && "Bits known to be one AND zero?"); // If all of the demanded bits are known zero on one side, return the other. // These bits cannot contribute to the result of the 'or'. @@ -1563,11 +1560,9 @@ bool TargetLowering::SimplifyDemandedBits( if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO, Depth + 1)) return true; - assert(!Known2.hasConflict() && "Bits known to be one AND zero?"); // If all of the demanded bits are known zero on one side, return the other. // These bits cannot contribute to the result of the 'xor'. @@ -1663,8 +1658,6 @@ bool TargetLowering::SimplifyDemandedBits( if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, DemandedElts, Known2, TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); - assert(!Known2.hasConflict() && "Bits known to be one AND zero?"); // If the operands are constants, see if we can simplify them. if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO)) @@ -1680,8 +1673,6 @@ bool TargetLowering::SimplifyDemandedBits( if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, DemandedElts, Known2, TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); - assert(!Known2.hasConflict() && "Bits known to be one AND zero?"); // Only known if known in both the LHS and RHS. Known = Known.intersectWith(Known2); @@ -1693,8 +1684,6 @@ bool TargetLowering::SimplifyDemandedBits( if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, DemandedElts, Known2, TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); - assert(!Known2.hasConflict() && "Bits known to be one AND zero?"); // If the operands are constants, see if we can simplify them. if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO)) @@ -1819,7 +1808,6 @@ bool TargetLowering::SimplifyDemandedBits( } return true; } - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); Known.Zero <<= ShAmt; Known.One <<= ShAmt; // low bits known zero. @@ -1993,7 +1981,6 @@ bool TargetLowering::SimplifyDemandedBits( if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); Known.Zero.lshrInPlace(ShAmt); Known.One.lshrInPlace(ShAmt); // High bits known zero. @@ -2090,7 +2077,6 @@ bool TargetLowering::SimplifyDemandedBits( if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); Known.Zero.lshrInPlace(ShAmt); Known.One.lshrInPlace(ShAmt); @@ -2385,7 +2371,6 @@ bool TargetLowering::SimplifyDemandedBits( if (SimplifyDemandedBits(Op0, InputDemandedBits, DemandedElts, Known, TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); // If the sign bit of the input is known set or clear, then we know the // top bits of the result. @@ -2458,7 +2443,6 @@ bool TargetLowering::SimplifyDemandedBits( } return true; } - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); assert(Known.getBitWidth() == InBits && "Src width has changed?"); Known = Known.zext(BitWidth); @@ -2508,7 +2492,6 @@ bool TargetLowering::SimplifyDemandedBits( if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); assert(Known.getBitWidth() == InBits && "Src width has changed?"); // If the sign bit is known one, the top bits match. @@ -2554,7 +2537,6 @@ bool TargetLowering::SimplifyDemandedBits( if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); assert(Known.getBitWidth() == InBits && "Src width has changed?"); Known = Known.anyext(BitWidth); @@ -2620,7 +2602,6 @@ bool TargetLowering::SimplifyDemandedBits( break; } - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); break; } case ISD::AssertZext: { @@ -2631,7 +2612,6 @@ bool TargetLowering::SimplifyDemandedBits( if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known, TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); Known.Zero |= ~InMask; Known.One &= (~Known.Zero); diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp index c3bde48b982c..08041c96ffe5 100644 --- a/llvm/lib/IR/ConstantRange.cpp +++ b/llvm/lib/IR/ConstantRange.cpp @@ -58,8 +58,8 @@ ConstantRange::ConstantRange(APInt L, APInt U) ConstantRange ConstantRange::fromKnownBits(const KnownBits &Known, bool IsSigned) { - assert(!Known.hasConflict() && "Expected valid KnownBits"); - + if (Known.hasConflict()) + return getEmpty(Known.getBitWidth()); if (Known.isUnknown()) return getFull(Known.getBitWidth()); diff --git a/llvm/lib/Support/KnownBits.cpp b/llvm/lib/Support/KnownBits.cpp index d6012a8eea8a..e3ad1468874c 100644 --- a/llvm/lib/Support/KnownBits.cpp +++ b/llvm/lib/Support/KnownBits.cpp @@ -18,11 +18,8 @@ using namespace llvm; -static KnownBits computeForAddCarry( - const KnownBits &LHS, const KnownBits &RHS, - bool CarryZero, bool CarryOne) { - assert(!(CarryZero && CarryOne) && - "Carry can't be zero and one at the same time"); +static KnownBits computeForAddCarry(const KnownBits &LHS, const KnownBits &RHS, + bool CarryZero, bool CarryOne) { APInt PossibleSumZero = LHS.getMaxValue() + RHS.getMaxValue() + !CarryZero; APInt PossibleSumOne = LHS.getMinValue() + RHS.getMinValue() + CarryOne; @@ -37,9 +34,6 @@ static KnownBits computeForAddCarry( APInt CarryKnownUnion = std::move(CarryKnownZero) | CarryKnownOne; APInt Known = std::move(LHSKnownUnion) & RHSKnownUnion & CarryKnownUnion; - assert((PossibleSumZero & Known) == (PossibleSumOne & Known) && - "known bits of sum differ"); - // Compute known bits of the result. KnownBits KnownOut; KnownOut.Zero = ~std::move(PossibleSumZero) & Known; @@ -608,14 +602,12 @@ KnownBits KnownBits::abs(bool IntMinIsPoison) const { } } - assert(!KnownAbs.hasConflict() && "Bad Output"); return KnownAbs; } static KnownBits computeForSatAddSub(bool Add, bool Signed, const KnownBits &LHS, const KnownBits &RHS) { - assert(!LHS.hasConflict() && !RHS.hasConflict() && "Bad inputs"); // We don't see NSW even for sadd/ssub as we want to check if the result has // signed overflow. KnownBits Res = @@ -715,7 +707,6 @@ static KnownBits computeForSatAddSub(bool Add, bool Signed, // We know whether or not we overflowed. if (!(*Overflow)) { // No overflow. - assert(!Res.hasConflict() && "Bad Output"); return Res; } @@ -737,7 +728,6 @@ static KnownBits computeForSatAddSub(bool Add, bool Signed, Res.One = C; Res.Zero = ~C; - assert(!Res.hasConflict() && "Bad Output"); return Res; } @@ -757,7 +747,6 @@ static KnownBits computeForSatAddSub(bool Add, bool Signed, Res.One.clearAllBits(); } - assert(!Res.hasConflict() && "Bad Output"); return Res; } @@ -808,8 +797,7 @@ KnownBits KnownBits::avgCeilU(const KnownBits &LHS, const KnownBits &RHS) { KnownBits KnownBits::mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply) { unsigned BitWidth = LHS.getBitWidth(); - assert(BitWidth == RHS.getBitWidth() && !LHS.hasConflict() && - !RHS.hasConflict() && "Operand mismatch"); + assert(BitWidth == RHS.getBitWidth() && "Operand mismatch"); assert((!NoUndefSelfMultiply || LHS == RHS) && "Self multiplication knownbits mismatch"); @@ -905,8 +893,7 @@ KnownBits KnownBits::mul(const KnownBits &LHS, const KnownBits &RHS, KnownBits KnownBits::mulhs(const KnownBits &LHS, const KnownBits &RHS) { unsigned BitWidth = LHS.getBitWidth(); - assert(BitWidth == RHS.getBitWidth() && !LHS.hasConflict() && - !RHS.hasConflict() && "Operand mismatch"); + assert(BitWidth == RHS.getBitWidth() && "Operand mismatch"); KnownBits WideLHS = LHS.sext(2 * BitWidth); KnownBits WideRHS = RHS.sext(2 * BitWidth); return mul(WideLHS, WideRHS).extractBits(BitWidth, BitWidth); @@ -914,8 +901,7 @@ KnownBits KnownBits::mulhs(const KnownBits &LHS, const KnownBits &RHS) { KnownBits KnownBits::mulhu(const KnownBits &LHS, const KnownBits &RHS) { unsigned BitWidth = LHS.getBitWidth(); - assert(BitWidth == RHS.getBitWidth() && !LHS.hasConflict() && - !RHS.hasConflict() && "Operand mismatch"); + assert(BitWidth == RHS.getBitWidth() && "Operand mismatch"); KnownBits WideLHS = LHS.zext(2 * BitWidth); KnownBits WideRHS = RHS.zext(2 * BitWidth); return mul(WideLHS, WideRHS).extractBits(BitWidth, BitWidth); @@ -964,7 +950,6 @@ KnownBits KnownBits::sdiv(const KnownBits &LHS, const KnownBits &RHS, return udiv(LHS, RHS, Exact); unsigned BitWidth = LHS.getBitWidth(); - assert(!LHS.hasConflict() && !RHS.hasConflict() && "Bad inputs"); KnownBits Known(BitWidth); if (LHS.isZero() || RHS.isZero()) { @@ -1011,15 +996,12 @@ KnownBits KnownBits::sdiv(const KnownBits &LHS, const KnownBits &RHS, } Known = divComputeLowBit(Known, LHS, RHS, Exact); - - assert(!Known.hasConflict() && "Bad Output"); return Known; } KnownBits KnownBits::udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact) { unsigned BitWidth = LHS.getBitWidth(); - assert(!LHS.hasConflict() && !RHS.hasConflict()); KnownBits Known(BitWidth); if (LHS.isZero() || RHS.isZero()) { @@ -1041,7 +1023,6 @@ KnownBits KnownBits::udiv(const KnownBits &LHS, const KnownBits &RHS, Known.Zero.setHighBits(LeadZ); Known = divComputeLowBit(Known, LHS, RHS, Exact); - assert(!Known.hasConflict() && "Bad Output"); return Known; } @@ -1059,8 +1040,6 @@ KnownBits KnownBits::remGetLowBits(const KnownBits &LHS, const KnownBits &RHS) { } KnownBits KnownBits::urem(const KnownBits &LHS, const KnownBits &RHS) { - assert(!LHS.hasConflict() && !RHS.hasConflict()); - KnownBits Known = remGetLowBits(LHS, RHS); if (RHS.isConstant() && RHS.getConstant().isPowerOf2()) { // NB: Low bits set in `remGetLowBits`. @@ -1078,8 +1057,6 @@ KnownBits KnownBits::urem(const KnownBits &LHS, const KnownBits &RHS) { } KnownBits KnownBits::srem(const KnownBits &LHS, const KnownBits &RHS) { - assert(!LHS.hasConflict() && !RHS.hasConflict()); - KnownBits Known = remGetLowBits(LHS, RHS); if (RHS.isConstant() && RHS.getConstant().isPowerOf2()) { // NB: Low bits are set in `remGetLowBits`. diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 3fbab3af32bb..2aec14e93d08 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -42452,12 +42452,10 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode( if (SimplifyDemandedBits(Op1, OriginalDemandedBits, OriginalDemandedElts, Known, TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); if (SimplifyDemandedBits(Op0, ~Known.Zero & OriginalDemandedBits, OriginalDemandedElts, Known2, TLO, Depth + 1)) return true; - assert(!Known2.hasConflict() && "Bits known to be one AND zero?"); // If the RHS is a constant, see if we can simplify it. if (ShrinkDemandedConstant(Op, ~Known2.One & OriginalDemandedBits, @@ -42508,7 +42506,6 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode( TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); Known.Zero <<= ShAmt; Known.One <<= ShAmt; @@ -42527,7 +42524,6 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode( OriginalDemandedElts, Known, TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); Known.Zero.lshrInPlace(ShAmt); Known.One.lshrInPlace(ShAmt); @@ -42568,7 +42564,6 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode( TLO, Depth + 1)) return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); Known.Zero.lshrInPlace(ShAmt); Known.One.lshrInPlace(ShAmt); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index 6739b8745d74..b3eebee0d0d8 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -209,8 +209,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, SimplifyDemandedBits(I, 0, DemandedMask & ~RHSKnown.Zero, LHSKnown, Depth + 1)) return I; - assert(!RHSKnown.hasConflict() && "Bits known to be one AND zero?"); - assert(!LHSKnown.hasConflict() && "Bits known to be one AND zero?"); Known = analyzeKnownBitsFromAndXorOr(cast(I), LHSKnown, RHSKnown, Depth, SQ.getWithInstruction(CxtI)); @@ -242,8 +240,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, I->dropPoisonGeneratingFlags(); return I; } - assert(!RHSKnown.hasConflict() && "Bits known to be one AND zero?"); - assert(!LHSKnown.hasConflict() && "Bits known to be one AND zero?"); Known = analyzeKnownBitsFromAndXorOr(cast(I), LHSKnown, RHSKnown, Depth, SQ.getWithInstruction(CxtI)); @@ -291,9 +287,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, return Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, Xor); } - assert(!RHSKnown.hasConflict() && "Bits known to be one AND zero?"); - assert(!LHSKnown.hasConflict() && "Bits known to be one AND zero?"); - Known = analyzeKnownBitsFromAndXorOr(cast(I), LHSKnown, RHSKnown, Depth, SQ.getWithInstruction(CxtI)); @@ -375,8 +368,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, if (SimplifyDemandedBits(I, 2, DemandedMask, RHSKnown, Depth + 1) || SimplifyDemandedBits(I, 1, DemandedMask, LHSKnown, Depth + 1)) return I; - assert(!RHSKnown.hasConflict() && "Bits known to be one AND zero?"); - assert(!LHSKnown.hasConflict() && "Bits known to be one AND zero?"); // If the operands are constants, see if we can simplify them. // This is similar to ShrinkDemandedConstant, but for a select we want to @@ -455,7 +446,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, InputKnown.makeNonNegative(); Known = InputKnown.zextOrTrunc(BitWidth); - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); break; } case Instruction::SExt: { @@ -481,12 +471,11 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, CastInst *NewCast = new ZExtInst(I->getOperand(0), VTy); NewCast->takeName(I); return InsertNewInstWith(NewCast, I->getIterator()); - } + } // If the sign bit of the input is known set or clear, then we know the // top bits of the result. Known = InputKnown.sext(BitWidth); - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); break; } case Instruction::Add: { @@ -696,7 +685,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, if (SimplifyDemandedBits(I, 0, DemandedMaskIn, Known, Depth + 1)) return I; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); Known = KnownBits::shl(Known, KnownBits::makeConstant(APInt(BitWidth, ShiftAmt)), @@ -772,7 +760,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, I->dropPoisonGeneratingFlags(); return I; } - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); Known.Zero.lshrInPlace(ShiftAmt); Known.One.lshrInPlace(ShiftAmt); if (ShiftAmt) @@ -819,7 +806,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, return I; } - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); // Compute the new bits that are at the top now plus sign bits. APInt HighBits(APInt::getHighBitsSet( BitWidth, std::min(SignBits + ShiftAmt - 1, BitWidth))); @@ -900,7 +886,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, if (LHSKnown.isNegative() && LowBits.intersects(LHSKnown.One)) Known.One |= ~LowBits; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); break; } } @@ -978,8 +963,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, // TODO: Should be 1-extend RHSKnown = RHSKnown.anyextOrTrunc(BitWidth); - assert(!RHSKnown.hasConflict() && "Bits known to be one AND zero?"); - assert(!LHSKnown.hasConflict() && "Bits known to be one AND zero?"); Known = LHSKnown & RHSKnown; KnownBitsComputed = true; diff --git a/llvm/unittests/IR/DemandedBitsTest.cpp b/llvm/unittests/IR/DemandedBitsTest.cpp index 4d15e8189961..b52173412a99 100644 --- a/llvm/unittests/IR/DemandedBitsTest.cpp +++ b/llvm/unittests/IR/DemandedBitsTest.cpp @@ -38,8 +38,10 @@ static void TestBinOpExhaustive(Fn1 PropagateFn, Fn2 EvalFn) { APInt AB1R = PropagateFn(0, AOut, Known1Redacted, Known2Redacted); APInt AB2R = PropagateFn(1, AOut, Known1Redacted, Known2Redacted); - EXPECT_EQ(AB1, AB1R); - EXPECT_EQ(AB2, AB2R); + if (!Known1Redacted.hasConflict() && !Known2Redacted.hasConflict()) { + EXPECT_EQ(AB1, AB1R); + EXPECT_EQ(AB2, AB2R); + } } ForeachNumInKnownBits(Known1, [&](APInt Value1) { ForeachNumInKnownBits(Known2, [&](APInt Value2) { diff --git a/llvm/unittests/Support/KnownBitsTest.cpp b/llvm/unittests/Support/KnownBitsTest.cpp index 824cf7501fd4..51e780c42af2 100644 --- a/llvm/unittests/Support/KnownBitsTest.cpp +++ b/llvm/unittests/Support/KnownBitsTest.cpp @@ -68,16 +68,16 @@ static void testUnaryOpExhaustive(StringRef Name, UnaryBitsFn BitsFn, } }); - EXPECT_TRUE(!Computed.hasConflict()); - EXPECT_TRUE(checkResult(Name, Exact, Computed, Known, CheckOptimality)); + if (!Exact.hasConflict()) { + EXPECT_TRUE(checkResult(Name, Exact, Computed, Known, CheckOptimality)); + } }); } } static void testBinaryOpExhaustive(StringRef Name, BinaryBitsFn BitsFn, BinaryIntFn IntFn, - bool CheckOptimality = true, - bool RefinePoisonToZero = false) { + bool CheckOptimality = true) { for (unsigned Bits : {1, 4}) { ForeachKnownBits(Bits, [&](const KnownBits &Known1) { ForeachKnownBits(Bits, [&](const KnownBits &Known2) { @@ -95,13 +95,9 @@ static void testBinaryOpExhaustive(StringRef Name, BinaryBitsFn BitsFn, }); }); - EXPECT_TRUE(!Computed.hasConflict()); - EXPECT_TRUE(checkResult(Name, Exact, Computed, {Known1, Known2}, - CheckOptimality)); - // In some cases we choose to return zero if the result is always - // poison. - if (RefinePoisonToZero && Exact.hasConflict()) { - EXPECT_TRUE(Computed.isZero()); + if (!Exact.hasConflict()) { + EXPECT_TRUE(checkResult(Name, Exact, Computed, {Known1, Known2}, + CheckOptimality)); } }); }); @@ -135,7 +131,9 @@ TEST(KnownBitsTest, AddCarryExhaustive) { KnownBits Computed = KnownBits::computeForAddCarry(Known1, Known2, KnownCarry); - EXPECT_EQ(Exact, Computed); + if (!Exact.hasConflict()) { + EXPECT_EQ(Exact, Computed); + } }); }); }); @@ -246,7 +244,9 @@ TEST(KnownBitsTest, SubBorrowExhaustive) { KnownBits Computed = KnownBits::computeForSubBorrow(Known1, Known2, KnownBorrow); - EXPECT_EQ(Exact, Computed); + if (!Exact.hasConflict()) { + EXPECT_EQ(Exact, Computed); + } }); }); }); @@ -394,7 +394,7 @@ TEST(KnownBitsTest, BinaryExhaustive) { return std::nullopt; return N1.shl(N2); }, - /*CheckOptimality=*/true, /* RefinePoisonToZero */ true); + /*CheckOptimality=*/true); testBinaryOpExhaustive( "ushl_ov", [](const KnownBits &Known1, const KnownBits &Known2) { @@ -407,7 +407,7 @@ TEST(KnownBitsTest, BinaryExhaustive) { return std::nullopt; return Res; }, - /*CheckOptimality=*/true, /* RefinePoisonToZero */ true); + /*CheckOptimality=*/true); testBinaryOpExhaustive( "shl nsw", [](const KnownBits &Known1, const KnownBits &Known2) { @@ -420,7 +420,7 @@ TEST(KnownBitsTest, BinaryExhaustive) { return std::nullopt; return Res; }, - /*CheckOptimality=*/true, /* RefinePoisonToZero */ true); + /*CheckOptimality=*/true); testBinaryOpExhaustive( "shl nuw", [](const KnownBits &Known1, const KnownBits &Known2) { @@ -434,7 +434,7 @@ TEST(KnownBitsTest, BinaryExhaustive) { return std::nullopt; return Res; }, - /*CheckOptimality=*/true, /* RefinePoisonToZero */ true); + /*CheckOptimality=*/true); testBinaryOpExhaustive( "lshr", @@ -446,7 +446,7 @@ TEST(KnownBitsTest, BinaryExhaustive) { return std::nullopt; return N1.lshr(N2); }, - /*CheckOptimality=*/true, /* RefinePoisonToZero */ true); + /*CheckOptimality=*/true); testBinaryOpExhaustive( "lshr exact", [](const KnownBits &Known1, const KnownBits &Known2) { @@ -460,7 +460,7 @@ TEST(KnownBitsTest, BinaryExhaustive) { return std::nullopt; return N1.lshr(N2); }, - /*CheckOptimality=*/true, /* RefinePoisonToZero */ true); + /*CheckOptimality=*/true); testBinaryOpExhaustive( "ashr", [](const KnownBits &Known1, const KnownBits &Known2) { @@ -471,7 +471,7 @@ TEST(KnownBitsTest, BinaryExhaustive) { return std::nullopt; return N1.ashr(N2); }, - /*CheckOptimality=*/true, /* RefinePoisonToZero */ true); + /*CheckOptimality=*/true); testBinaryOpExhaustive( "ashr exact", [](const KnownBits &Known1, const KnownBits &Known2) { @@ -485,7 +485,7 @@ TEST(KnownBitsTest, BinaryExhaustive) { return std::nullopt; return N1.ashr(N2); }, - /*CheckOptimality=*/true, /* RefinePoisonToZero */ true); + /*CheckOptimality=*/true); testBinaryOpExhaustive( "mul", [](const KnownBits &Known1, const KnownBits &Known2) { @@ -608,6 +608,9 @@ TEST(KnownBitsTest, ICmpExhaustive) { std::optional KnownSLT = KnownBits::slt(Known1, Known2); std::optional KnownSLE = KnownBits::sle(Known1, Known2); + if (Known1.hasConflict() || Known2.hasConflict()) + return; + EXPECT_EQ(AllEQ || NoneEQ, KnownEQ.has_value()); EXPECT_EQ(AllNE || NoneNE, KnownNE.has_value()); EXPECT_EQ(AllUGT || NoneUGT, KnownUGT.has_value()); @@ -653,8 +656,10 @@ TEST(KnownBitsTest, GetMinMaxVal) { Min = APIntOps::umin(Min, N); Max = APIntOps::umax(Max, N); }); - EXPECT_EQ(Min, Known.getMinValue()); - EXPECT_EQ(Max, Known.getMaxValue()); + if (!Known.hasConflict()) { + EXPECT_EQ(Min, Known.getMinValue()); + EXPECT_EQ(Max, Known.getMaxValue()); + } }); } @@ -667,8 +672,10 @@ TEST(KnownBitsTest, GetSignedMinMaxVal) { Min = APIntOps::smin(Min, N); Max = APIntOps::smax(Max, N); }); - EXPECT_EQ(Min, Known.getSignedMinValue()); - EXPECT_EQ(Max, Known.getSignedMaxValue()); + if (!Known.hasConflict()) { + EXPECT_EQ(Min, Known.getSignedMinValue()); + EXPECT_EQ(Max, Known.getSignedMaxValue()); + } }); } @@ -679,7 +686,9 @@ TEST(KnownBitsTest, CountMaxActiveBits) { ForeachNumInKnownBits(Known, [&](const APInt &N) { Expected = std::max(Expected, N.getActiveBits()); }); - EXPECT_EQ(Expected, Known.countMaxActiveBits()); + if (!Known.hasConflict()) { + EXPECT_EQ(Expected, Known.countMaxActiveBits()); + } }); } @@ -690,7 +699,9 @@ TEST(KnownBitsTest, CountMaxSignificantBits) { ForeachNumInKnownBits(Known, [&](const APInt &N) { Expected = std::max(Expected, N.getSignificantBits()); }); - EXPECT_EQ(Expected, Known.countMaxSignificantBits()); + if (!Known.hasConflict()) { + EXPECT_EQ(Expected, Known.countMaxSignificantBits()); + } }); } @@ -736,8 +747,10 @@ TEST(KnownBitsTest, SExtInReg) { CommonZero &= ~Ext; }); KnownBits KnownSExtInReg = Known.sextInReg(FromBits); - EXPECT_EQ(CommonOne, KnownSExtInReg.One); - EXPECT_EQ(CommonZero, KnownSExtInReg.Zero); + if (!Known.hasConflict()) { + EXPECT_EQ(CommonOne, KnownSExtInReg.One); + EXPECT_EQ(CommonZero, KnownSExtInReg.Zero); + } }); } } @@ -752,8 +765,10 @@ TEST(KnownBitsTest, CommonBitsSet) { HasCommonBitsSet |= N1.intersects(N2); }); }); - EXPECT_EQ(!HasCommonBitsSet, - KnownBits::haveNoCommonBitsSet(Known1, Known2)); + if (!Known1.hasConflict() && !Known2.hasConflict()) { + EXPECT_EQ(!HasCommonBitsSet, + KnownBits::haveNoCommonBitsSet(Known1, Known2)); + } }); }); } diff --git a/llvm/unittests/Support/KnownBitsTest.h b/llvm/unittests/Support/KnownBitsTest.h index bc291898814b..556da2b9ecda 100644 --- a/llvm/unittests/Support/KnownBitsTest.h +++ b/llvm/unittests/Support/KnownBitsTest.h @@ -26,9 +26,6 @@ template void ForeachKnownBits(unsigned Bits, FnTy Fn) { for (unsigned One = 0; One < Max; ++One) { Known.Zero = Zero; Known.One = One; - if (Known.hasConflict()) - continue; - Fn(Known); } } -- GitLab From 790992dd4018f55479a2c53a79088cb37710c85b Mon Sep 17 00:00:00 2001 From: Jake Egan Date: Fri, 7 Jun 2024 11:06:42 -0400 Subject: [PATCH 260/462] [libc++][test][AIX] Only XFAIL atomic tests for before clang 19 (#94646) These tests pass on 64-bit. They were fixed by 5fdd094837c6 on 32-bit. So XFAIL only for 32-bit before clang 19. --- .../atomics.types.float/fetch_add.pass.cpp | 5 ++++- .../atomics.types.float/fetch_sub.pass.cpp | 5 ++++- .../atomics.types.float/operator.minus_equals.pass.cpp | 5 ++++- .../atomics.types.float/operator.plus_equals.pass.cpp | 5 ++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp index 7350c1ddf0e9..40a475ec38b7 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp @@ -6,9 +6,12 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// UNSUPPORTED: LIBCXX-AIX-FIXME // XFAIL: !has-64-bit-atomics +// Older versions of clang have a bug with atomic builtins affecting double and long double. +// Fixed by 5fdd0948. +// XFAIL: target=powerpc-ibm-{{.*}} && (clang-17 || clang-18) + // https://github.com/llvm/llvm-project/issues/72893 // XFAIL: target={{x86_64-.*}} && tsan diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp index 84dcde5f2784..9e798a2a519f 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp @@ -6,9 +6,12 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// UNSUPPORTED: LIBCXX-AIX-FIXME // XFAIL: !has-64-bit-atomics +// Older versions of clang have a bug with atomic builtins affecting double and long double. +// Fixed by 5fdd0948. +// XFAIL: target=powerpc-ibm-{{.*}} && (clang-17 || clang-18) + // https://github.com/llvm/llvm-project/issues/72893 // XFAIL: target={{x86_64-.*}} && tsan diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp index 386a393e3550..732bd4d7e5dc 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp @@ -6,9 +6,12 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// UNSUPPORTED: LIBCXX-AIX-FIXME // XFAIL: !has-64-bit-atomics +// Older versions of clang have a bug with atomic builtins affecting double and long double. +// Fixed by 5fdd0948. +// XFAIL: target=powerpc-ibm-{{.*}} && (clang-17 || clang-18) + // floating-point-type operator-=(floating-point-type) volatile noexcept; // floating-point-type operator-=(floating-point-type) noexcept; diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp index afd06d537c7a..1821aca42c79 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp @@ -6,9 +6,12 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// UNSUPPORTED: LIBCXX-AIX-FIXME // XFAIL: !has-64-bit-atomics +// Older versions of clang have a bug with atomic builtins affecting double and long double. +// Fixed by 5fdd0948. +// XFAIL: target=powerpc-ibm-{{.*}} && (clang-17 || clang-18) + // floating-point-type operator+=(floating-point-type) volatile noexcept; // floating-point-type operator+=(floating-point-type) noexcept; -- GitLab From f7018ba0eeaad8dc3e1917cfb986fc9689d72e85 Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 7 Jun 2024 16:09:57 +0100 Subject: [PATCH 261/462] [AArch64] Add patterns for add(uzp1(x,y), uzp2(x, y)) -> addp. If we are extracting the even lanes and the odd lanes and adding them, we can use an addp instruction. --- llvm/lib/Target/AArch64/AArch64InstrInfo.td | 14 +++ llvm/test/CodeGen/AArch64/addp-shuffle.ll | 49 ++------ llvm/test/CodeGen/AArch64/arm64-uzp.ll | 32 ++--- llvm/test/CodeGen/AArch64/insert-extend.ll | 116 +++++++++--------- .../sve-fixed-length-permute-zip-uzp-trn.ll | 4 +- 5 files changed, 102 insertions(+), 113 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 081d64921ee1..2ed7850404ce 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -9405,6 +9405,20 @@ def : Pat<(AArch64faddp (v4f16 (extract_subvector (v8f16 FPR128:$Rn), (i64 0))), (v4f16 (extract_subvector (v8f16 FPR128:$Rn), (i64 4)))), (v4f16 (EXTRACT_SUBREG (FADDPv8f16 $Rn, $Rn), dsub))>; +// add(uzp1(X, Y), uzp2(X, Y)) -> addp(X, Y) +def : Pat<(v2i64 (add (AArch64zip1 (v2i64 FPR128:$Rn), (v2i64 FPR128:$Rm)), + (AArch64zip2 (v2i64 FPR128:$Rn), (v2i64 FPR128:$Rm)))), + (v2i64 (ADDPv2i64 $Rn, $Rm))>; +def : Pat<(v4i32 (add (AArch64uzp1 (v4i32 FPR128:$Rn), (v4i32 FPR128:$Rm)), + (AArch64uzp2 (v4i32 FPR128:$Rn), (v4i32 FPR128:$Rm)))), + (v4i32 (ADDPv4i32 $Rn, $Rm))>; +def : Pat<(v8i16 (add (AArch64uzp1 (v8i16 FPR128:$Rn), (v8i16 FPR128:$Rm)), + (AArch64uzp2 (v8i16 FPR128:$Rn), (v8i16 FPR128:$Rm)))), + (v8i16 (ADDPv8i16 $Rn, $Rm))>; +def : Pat<(v16i8 (add (AArch64uzp1 (v16i8 FPR128:$Rn), (v16i8 FPR128:$Rm)), + (AArch64uzp2 (v16i8 FPR128:$Rn), (v16i8 FPR128:$Rm)))), + (v16i8 (ADDPv16i8 $Rn, $Rm))>; + // Scalar 64-bit shifts in FPR64 registers. def : Pat<(i64 (int_aarch64_neon_sshl (i64 FPR64:$Rn), (i64 FPR64:$Rm))), (SSHLv1i64 FPR64:$Rn, FPR64:$Rm)>; diff --git a/llvm/test/CodeGen/AArch64/addp-shuffle.ll b/llvm/test/CodeGen/AArch64/addp-shuffle.ll index c15a84c7b3a2..a187e7e94c20 100644 --- a/llvm/test/CodeGen/AArch64/addp-shuffle.ll +++ b/llvm/test/CodeGen/AArch64/addp-shuffle.ll @@ -5,9 +5,7 @@ define <4 x i32> @deinterleave_shuffle_v8i32(<8 x i32> %a) { ; CHECK-LABEL: deinterleave_shuffle_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: uzp1 v2.4s, v0.4s, v1.4s -; CHECK-NEXT: uzp2 v0.4s, v0.4s, v1.4s -; CHECK-NEXT: add v0.4s, v2.4s, v0.4s +; CHECK-NEXT: addp v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret %r0 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> %r1 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> @@ -18,9 +16,7 @@ define <4 x i32> @deinterleave_shuffle_v8i32(<8 x i32> %a) { define <4 x i32> @deinterleave_shuffle_v8i32_c(<8 x i32> %a) { ; CHECK-LABEL: deinterleave_shuffle_v8i32_c: ; CHECK: // %bb.0: -; CHECK-NEXT: uzp1 v2.4s, v0.4s, v1.4s -; CHECK-NEXT: uzp2 v0.4s, v0.4s, v1.4s -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: addp v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret %r0 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> %r1 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> @@ -45,9 +41,7 @@ define <2 x i32> @deinterleave_shuffle_v4i32(<4 x i32> %a) { define <8 x i16> @deinterleave_shuffle_v16i16(<16 x i16> %a) { ; CHECK-LABEL: deinterleave_shuffle_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: uzp1 v2.8h, v0.8h, v1.8h -; CHECK-NEXT: uzp2 v0.8h, v0.8h, v1.8h -; CHECK-NEXT: add v0.8h, v2.8h, v0.8h +; CHECK-NEXT: addp v0.8h, v0.8h, v1.8h ; CHECK-NEXT: ret %r0 = shufflevector <16 x i16> %a, <16 x i16> poison, <8 x i32> %r1 = shufflevector <16 x i16> %a, <16 x i16> poison, <8 x i32> @@ -58,9 +52,7 @@ define <8 x i16> @deinterleave_shuffle_v16i16(<16 x i16> %a) { define <16 x i8> @deinterleave_shuffle_v32i8(<32 x i8> %a) { ; CHECK-LABEL: deinterleave_shuffle_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: uzp1 v2.16b, v0.16b, v1.16b -; CHECK-NEXT: uzp2 v0.16b, v0.16b, v1.16b -; CHECK-NEXT: add v0.16b, v2.16b, v0.16b +; CHECK-NEXT: addp v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %r0 = shufflevector <32 x i8> %a, <32 x i8> poison, <16 x i32> %r1 = shufflevector <32 x i8> %a, <32 x i8> poison, <16 x i32> @@ -71,12 +63,9 @@ define <16 x i8> @deinterleave_shuffle_v32i8(<32 x i8> %a) { define <4 x i64> @deinterleave_shuffle_v8i64(<8 x i64> %a) { ; CHECK-LABEL: deinterleave_shuffle_v8i64: ; CHECK: // %bb.0: -; CHECK-NEXT: zip1 v4.2d, v2.2d, v3.2d -; CHECK-NEXT: zip1 v5.2d, v0.2d, v1.2d -; CHECK-NEXT: zip2 v2.2d, v2.2d, v3.2d -; CHECK-NEXT: zip2 v0.2d, v0.2d, v1.2d -; CHECK-NEXT: add v1.2d, v4.2d, v2.2d -; CHECK-NEXT: add v0.2d, v5.2d, v0.2d +; CHECK-NEXT: addp v2.2d, v2.2d, v3.2d +; CHECK-NEXT: addp v0.2d, v0.2d, v1.2d +; CHECK-NEXT: mov v1.16b, v2.16b ; CHECK-NEXT: ret %r0 = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> %r1 = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> @@ -164,15 +153,9 @@ define <4 x i32> @udot(<4 x i32> %z, <16 x i8> %a, <16 x i8> %b) { ; CHECK-NEXT: umull v3.4s, v3.4h, v4.4h ; CHECK-NEXT: umull2 v4.4s, v1.8h, v2.8h ; CHECK-NEXT: umull v1.4s, v1.4h, v2.4h -; CHECK-NEXT: uzp1 v2.4s, v3.4s, v5.4s -; CHECK-NEXT: uzp2 v3.4s, v3.4s, v5.4s -; CHECK-NEXT: uzp1 v6.4s, v1.4s, v4.4s -; CHECK-NEXT: uzp2 v1.4s, v1.4s, v4.4s -; CHECK-NEXT: add v2.4s, v2.4s, v3.4s -; CHECK-NEXT: add v1.4s, v6.4s, v1.4s -; CHECK-NEXT: uzp1 v3.4s, v2.4s, v1.4s -; CHECK-NEXT: uzp2 v1.4s, v2.4s, v1.4s -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s +; CHECK-NEXT: addp v2.4s, v3.4s, v5.4s +; CHECK-NEXT: addp v1.4s, v1.4s, v4.4s +; CHECK-NEXT: addp v1.4s, v2.4s, v1.4s ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret %za = zext <16 x i8> %a to <16 x i32> @@ -199,15 +182,9 @@ define <4 x i32> @sdot(<4 x i32> %z, <16 x i8> %a, <16 x i8> %b) { ; CHECK-NEXT: smull v3.4s, v3.4h, v4.4h ; CHECK-NEXT: smull2 v4.4s, v1.8h, v2.8h ; CHECK-NEXT: smull v1.4s, v1.4h, v2.4h -; CHECK-NEXT: uzp1 v2.4s, v3.4s, v5.4s -; CHECK-NEXT: uzp2 v3.4s, v3.4s, v5.4s -; CHECK-NEXT: uzp1 v6.4s, v1.4s, v4.4s -; CHECK-NEXT: uzp2 v1.4s, v1.4s, v4.4s -; CHECK-NEXT: add v2.4s, v2.4s, v3.4s -; CHECK-NEXT: add v1.4s, v6.4s, v1.4s -; CHECK-NEXT: uzp1 v3.4s, v2.4s, v1.4s -; CHECK-NEXT: uzp2 v1.4s, v2.4s, v1.4s -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s +; CHECK-NEXT: addp v2.4s, v3.4s, v5.4s +; CHECK-NEXT: addp v1.4s, v1.4s, v4.4s +; CHECK-NEXT: addp v1.4s, v2.4s, v1.4s ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret %za = sext <16 x i8> %a to <16 x i32> diff --git a/llvm/test/CodeGen/AArch64/arm64-uzp.ll b/llvm/test/CodeGen/AArch64/arm64-uzp.ll index bd6bf1bf1578..02068428c3a4 100644 --- a/llvm/test/CodeGen/AArch64/arm64-uzp.ll +++ b/llvm/test/CodeGen/AArch64/arm64-uzp.ll @@ -33,11 +33,11 @@ define <16 x i8> @vuzpQi8(<16 x i8> %A, <16 x i8> %B) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: uzp1.16b v2, v0, v1 ; CHECK-NEXT: uzp2.16b v0, v0, v1 -; CHECK-NEXT: add.16b v0, v2, v0 +; CHECK-NEXT: eor.16b v0, v2, v0 ; CHECK-NEXT: ret %tmp3 = shufflevector <16 x i8> %A, <16 x i8> %B, <16 x i32> %tmp4 = shufflevector <16 x i8> %A, <16 x i8> %B, <16 x i32> - %tmp5 = add <16 x i8> %tmp3, %tmp4 + %tmp5 = xor <16 x i8> %tmp3, %tmp4 ret <16 x i8> %tmp5 } @@ -46,11 +46,11 @@ define <8 x i16> @vuzpQi16(<8 x i16> %A, <8 x i16> %B) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: uzp1.8h v2, v0, v1 ; CHECK-NEXT: uzp2.8h v0, v0, v1 -; CHECK-NEXT: add.8h v0, v2, v0 +; CHECK-NEXT: eor.16b v0, v2, v0 ; CHECK-NEXT: ret %tmp3 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> %tmp4 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> - %tmp5 = add <8 x i16> %tmp3, %tmp4 + %tmp5 = xor <8 x i16> %tmp3, %tmp4 ret <8 x i16> %tmp5 } @@ -59,11 +59,11 @@ define <4 x i32> @vuzpQi32(<4 x i32> %A, <4 x i32> %B) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: uzp1.4s v2, v0, v1 ; CHECK-NEXT: uzp2.4s v0, v0, v1 -; CHECK-NEXT: add.4s v0, v2, v0 +; CHECK-NEXT: eor.16b v0, v2, v0 ; CHECK-NEXT: ret %tmp3 = shufflevector <4 x i32> %A, <4 x i32> %B, <4 x i32> %tmp4 = shufflevector <4 x i32> %A, <4 x i32> %B, <4 x i32> - %tmp5 = add <4 x i32> %tmp3, %tmp4 + %tmp5 = xor <4 x i32> %tmp3, %tmp4 ret <4 x i32> %tmp5 } @@ -72,11 +72,11 @@ define <4 x float> @vuzpQf(<4 x float> %A, <4 x float> %B) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: uzp1.4s v2, v0, v1 ; CHECK-NEXT: uzp2.4s v0, v0, v1 -; CHECK-NEXT: fadd.4s v0, v2, v0 +; CHECK-NEXT: fsub.4s v0, v2, v0 ; CHECK-NEXT: ret %tmp3 = shufflevector <4 x float> %A, <4 x float> %B, <4 x i32> %tmp4 = shufflevector <4 x float> %A, <4 x float> %B, <4 x i32> - %tmp5 = fadd <4 x float> %tmp3, %tmp4 + %tmp5 = fsub <4 x float> %tmp3, %tmp4 ret <4 x float> %tmp5 } @@ -100,11 +100,11 @@ define <8 x i16> @vuzpQi16_undef1(<8 x i16> %A, <8 x i16> %B) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: uzp1.8h v2, v0, v1 ; CHECK-NEXT: uzp2.8h v0, v0, v1 -; CHECK-NEXT: add.8h v0, v2, v0 +; CHECK-NEXT: eor.16b v0, v2, v0 ; CHECK-NEXT: ret %tmp3 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> %tmp4 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> - %tmp5 = add <8 x i16> %tmp3, %tmp4 + %tmp5 = xor <8 x i16> %tmp3, %tmp4 ret <8 x i16> %tmp5 } @@ -113,11 +113,11 @@ define <8 x i16> @vuzpQi16_undef0(<8 x i16> %A, <8 x i16> %B) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: uzp1.8h v2, v0, v1 ; CHECK-NEXT: uzp2.8h v0, v0, v1 -; CHECK-NEXT: add.8h v0, v2, v0 +; CHECK-NEXT: eor.16b v0, v2, v0 ; CHECK-NEXT: ret %tmp3 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> %tmp4 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> - %tmp5 = add <8 x i16> %tmp3, %tmp4 + %tmp5 = xor <8 x i16> %tmp3, %tmp4 ret <8 x i16> %tmp5 } @@ -126,11 +126,11 @@ define <8 x i16> @vuzpQi16_undef01(<8 x i16> %A, <8 x i16> %B) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: uzp1.8h v2, v0, v1 ; CHECK-NEXT: uzp2.8h v0, v0, v1 -; CHECK-NEXT: add.8h v0, v2, v0 +; CHECK-NEXT: eor.16b v0, v2, v0 ; CHECK-NEXT: ret %tmp3 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> %tmp4 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> - %tmp5 = add <8 x i16> %tmp3, %tmp4 + %tmp5 = xor <8 x i16> %tmp3, %tmp4 ret <8 x i16> %tmp5 } @@ -139,10 +139,10 @@ define <8 x i16> @vuzpQi16_undef012(<8 x i16> %A, <8 x i16> %B) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: uzp1.8h v2, v0, v1 ; CHECK-NEXT: uzp2.8h v0, v0, v1 -; CHECK-NEXT: add.8h v0, v2, v0 +; CHECK-NEXT: eor.16b v0, v2, v0 ; CHECK-NEXT: ret %tmp3 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> %tmp4 = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> - %tmp5 = add <8 x i16> %tmp3, %tmp4 + %tmp5 = xor <8 x i16> %tmp3, %tmp4 ret <8 x i16> %tmp5 } diff --git a/llvm/test/CodeGen/AArch64/insert-extend.ll b/llvm/test/CodeGen/AArch64/insert-extend.ll index 0b730f6e7715..851fb0d03e8a 100644 --- a/llvm/test/CodeGen/AArch64/insert-extend.ll +++ b/llvm/test/CodeGen/AArch64/insert-extend.ll @@ -94,75 +94,73 @@ define i32 @large(ptr nocapture noundef readonly %p1, i32 noundef %st1, ptr noca ; CHECK-NEXT: mov v3.d[1], v5.d[1] ; CHECK-NEXT: uzp1 v1.4s, v4.4s, v0.4s ; CHECK-NEXT: uzp2 v4.4s, v4.4s, v0.4s -; CHECK-NEXT: uzp2 v5.4s, v2.4s, v0.4s -; CHECK-NEXT: uzp1 v0.4s, v2.4s, v0.4s -; CHECK-NEXT: add v2.4s, v3.4s, v6.4s +; CHECK-NEXT: addp v0.4s, v2.4s, v0.4s +; CHECK-NEXT: add v5.4s, v3.4s, v6.4s ; CHECK-NEXT: sub v3.4s, v6.4s, v3.4s -; CHECK-NEXT: sub v1.4s, v1.4s, v4.4s -; CHECK-NEXT: add v0.4s, v5.4s, v0.4s -; CHECK-NEXT: rev64 v4.4s, v2.4s -; CHECK-NEXT: rev64 v5.4s, v3.4s -; CHECK-NEXT: rev64 v6.4s, v1.4s ; CHECK-NEXT: rev64 v7.4s, v0.4s -; CHECK-NEXT: addp v16.4s, v1.4s, v3.4s -; CHECK-NEXT: addp v17.4s, v0.4s, v2.4s -; CHECK-NEXT: sub v3.4s, v3.4s, v5.4s -; CHECK-NEXT: sub v2.4s, v2.4s, v4.4s -; CHECK-NEXT: sub v1.4s, v1.4s, v6.4s +; CHECK-NEXT: sub v1.4s, v1.4s, v4.4s +; CHECK-NEXT: rev64 v4.4s, v5.4s +; CHECK-NEXT: rev64 v6.4s, v3.4s +; CHECK-NEXT: addp v16.4s, v0.4s, v5.4s +; CHECK-NEXT: rev64 v2.4s, v1.4s ; CHECK-NEXT: sub v0.4s, v0.4s, v7.4s -; CHECK-NEXT: zip1 v18.4s, v17.4s, v17.4s -; CHECK-NEXT: ext v4.16b, v17.16b, v2.16b, #4 -; CHECK-NEXT: ext v5.16b, v16.16b, v3.16b, #4 +; CHECK-NEXT: zip1 v21.4s, v16.4s, v16.4s +; CHECK-NEXT: sub v4.4s, v5.4s, v4.4s +; CHECK-NEXT: addp v5.4s, v1.4s, v3.4s +; CHECK-NEXT: sub v3.4s, v3.4s, v6.4s +; CHECK-NEXT: sub v1.4s, v1.4s, v2.4s +; CHECK-NEXT: ext v7.16b, v0.16b, v16.16b, #4 +; CHECK-NEXT: ext v2.16b, v16.16b, v4.16b, #4 +; CHECK-NEXT: ext v6.16b, v5.16b, v3.16b, #4 +; CHECK-NEXT: mov v19.16b, v4.16b +; CHECK-NEXT: ext v17.16b, v1.16b, v5.16b, #8 ; CHECK-NEXT: mov v20.16b, v3.16b -; CHECK-NEXT: ext v6.16b, v1.16b, v16.16b, #8 -; CHECK-NEXT: ext v7.16b, v0.16b, v17.16b, #4 -; CHECK-NEXT: mov v21.16b, v2.16b -; CHECK-NEXT: trn2 v0.4s, v18.4s, v0.4s -; CHECK-NEXT: mov v20.s[2], v16.s[3] -; CHECK-NEXT: zip2 v4.4s, v4.4s, v17.4s -; CHECK-NEXT: zip2 v5.4s, v5.4s, v16.4s -; CHECK-NEXT: mov v21.s[2], v17.s[3] -; CHECK-NEXT: ext v19.16b, v6.16b, v1.16b, #4 +; CHECK-NEXT: trn2 v0.4s, v21.4s, v0.4s ; CHECK-NEXT: ext v7.16b, v7.16b, v7.16b, #4 -; CHECK-NEXT: mov v1.s[2], v16.s[1] -; CHECK-NEXT: ext v2.16b, v2.16b, v4.16b, #12 -; CHECK-NEXT: ext v3.16b, v3.16b, v5.16b, #12 -; CHECK-NEXT: uzp2 v4.4s, v6.4s, v19.4s -; CHECK-NEXT: mov v5.16b, v7.16b -; CHECK-NEXT: mov v6.16b, v20.16b -; CHECK-NEXT: mov v18.16b, v1.16b -; CHECK-NEXT: mov v19.16b, v21.16b +; CHECK-NEXT: mov v19.s[2], v16.s[3] +; CHECK-NEXT: zip2 v2.4s, v2.4s, v16.4s +; CHECK-NEXT: zip2 v6.4s, v6.4s, v5.4s +; CHECK-NEXT: mov v20.s[2], v5.s[3] +; CHECK-NEXT: ext v18.16b, v17.16b, v1.16b, #4 +; CHECK-NEXT: mov v1.s[2], v5.s[1] +; CHECK-NEXT: mov v21.16b, v7.16b ; CHECK-NEXT: sub v7.4s, v0.4s, v7.4s -; CHECK-NEXT: mov v6.s[1], v16.s[2] -; CHECK-NEXT: mov v5.s[0], v17.s[1] -; CHECK-NEXT: mov v18.s[1], v16.s[0] -; CHECK-NEXT: mov v19.s[1], v17.s[2] +; CHECK-NEXT: ext v2.16b, v4.16b, v2.16b, #12 +; CHECK-NEXT: ext v3.16b, v3.16b, v6.16b, #12 +; CHECK-NEXT: uzp2 v4.4s, v17.4s, v18.4s +; CHECK-NEXT: mov v6.16b, v1.16b +; CHECK-NEXT: mov v17.16b, v19.16b +; CHECK-NEXT: mov v18.16b, v20.16b +; CHECK-NEXT: mov v21.s[0], v16.s[1] +; CHECK-NEXT: mov v6.s[1], v5.s[0] +; CHECK-NEXT: mov v17.s[1], v16.s[2] +; CHECK-NEXT: sub v16.4s, v19.4s, v2.4s +; CHECK-NEXT: mov v18.s[1], v5.s[2] ; CHECK-NEXT: sub v1.4s, v1.4s, v4.4s -; CHECK-NEXT: sub v16.4s, v20.4s, v3.4s -; CHECK-NEXT: sub v17.4s, v21.4s, v2.4s -; CHECK-NEXT: add v3.4s, v6.4s, v3.4s -; CHECK-NEXT: add v0.4s, v0.4s, v5.4s -; CHECK-NEXT: add v4.4s, v18.4s, v4.4s -; CHECK-NEXT: add v2.4s, v19.4s, v2.4s -; CHECK-NEXT: mov v3.d[1], v16.d[1] +; CHECK-NEXT: sub v5.4s, v20.4s, v3.4s +; CHECK-NEXT: add v0.4s, v0.4s, v21.4s +; CHECK-NEXT: add v4.4s, v6.4s, v4.4s +; CHECK-NEXT: add v2.4s, v17.4s, v2.4s +; CHECK-NEXT: add v3.4s, v18.4s, v3.4s ; CHECK-NEXT: mov v0.d[1], v7.d[1] ; CHECK-NEXT: mov v4.d[1], v1.d[1] -; CHECK-NEXT: mov v2.d[1], v17.d[1] -; CHECK-NEXT: cmlt v1.8h, v3.8h, #0 -; CHECK-NEXT: cmlt v5.8h, v0.8h, #0 -; CHECK-NEXT: cmlt v6.8h, v4.8h, #0 -; CHECK-NEXT: cmlt v7.8h, v2.8h, #0 -; CHECK-NEXT: add v3.4s, v1.4s, v3.4s -; CHECK-NEXT: add v0.4s, v5.4s, v0.4s -; CHECK-NEXT: add v4.4s, v6.4s, v4.4s -; CHECK-NEXT: add v2.4s, v7.4s, v2.4s -; CHECK-NEXT: eor v1.16b, v3.16b, v1.16b -; CHECK-NEXT: eor v0.16b, v0.16b, v5.16b -; CHECK-NEXT: eor v2.16b, v2.16b, v7.16b -; CHECK-NEXT: eor v3.16b, v4.16b, v6.16b -; CHECK-NEXT: add v0.4s, v0.4s, v3.4s -; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: mov v2.d[1], v16.d[1] +; CHECK-NEXT: mov v3.d[1], v5.d[1] +; CHECK-NEXT: cmlt v7.8h, v0.8h, #0 +; CHECK-NEXT: cmlt v1.8h, v4.8h, #0 +; CHECK-NEXT: cmlt v6.8h, v2.8h, #0 +; CHECK-NEXT: cmlt v5.8h, v3.8h, #0 +; CHECK-NEXT: add v0.4s, v7.4s, v0.4s +; CHECK-NEXT: add v4.4s, v1.4s, v4.4s +; CHECK-NEXT: add v2.4s, v6.4s, v2.4s +; CHECK-NEXT: add v3.4s, v5.4s, v3.4s +; CHECK-NEXT: eor v0.16b, v0.16b, v7.16b +; CHECK-NEXT: eor v1.16b, v4.16b, v1.16b +; CHECK-NEXT: eor v2.16b, v2.16b, v6.16b +; CHECK-NEXT: eor v3.16b, v3.16b, v5.16b ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-NEXT: add v2.4s, v2.4s, v3.4s +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: lsr w9, w8, #16 diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-zip-uzp-trn.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-zip-uzp-trn.ll index e07645c27df7..fba324cfd74a 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-zip-uzp-trn.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-zip-uzp-trn.ll @@ -611,14 +611,14 @@ define void @uzp_v8i16(ptr %a, ptr %b) #1 { ; CHECK-NEXT: ldr q1, [x1] ; CHECK-NEXT: uzp1 v2.8h, v0.8h, v1.8h ; CHECK-NEXT: uzp2 v0.8h, v0.8h, v1.8h -; CHECK-NEXT: add v0.8h, v2.8h, v0.8h +; CHECK-NEXT: eor v0.16b, v2.16b, v0.16b ; CHECK-NEXT: str q0, [x0] ; CHECK-NEXT: ret %tmp1 = load <8 x i16>, ptr %a %tmp2 = load <8 x i16>, ptr %b %tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> %tmp4 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> - %tmp5 = add <8 x i16> %tmp3, %tmp4 + %tmp5 = xor <8 x i16> %tmp3, %tmp4 store <8 x i16> %tmp5, ptr %a ret void } -- GitLab From 4f9c0fa22374d50643177adc4fb2706fa4b9b163 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 7 Jun 2024 16:14:01 +0100 Subject: [PATCH 262/462] [LV] Add test with dead load and vector pointer. --- .../LoopVectorize/dead_instructions.ll | 187 ++++++++++++++++-- 1 file changed, 170 insertions(+), 17 deletions(-) diff --git a/llvm/test/Transforms/LoopVectorize/dead_instructions.ll b/llvm/test/Transforms/LoopVectorize/dead_instructions.ll index eeb270c4be54..3374e2e67e35 100644 --- a/llvm/test/Transforms/LoopVectorize/dead_instructions.ll +++ b/llvm/test/Transforms/LoopVectorize/dead_instructions.ll @@ -1,28 +1,64 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 ; RUN: opt < %s -force-vector-width=2 -force-vector-interleave=2 -passes=loop-vectorize -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" -; CHECK-LABEL: @dead_instructions_01 ; ; This test ensures that we don't generate trivially dead instructions prior to ; instruction simplification. We don't need to generate instructions ; corresponding to the original induction variable update or branch condition, ; since we rewrite the loop structure. ; -; CHECK: vector.body: -; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] -; CHECK: %[[I0:.+]] = add i64 %index, 0 -; CHECK: %[[I2:.+]] = add i64 %index, 2 -; CHECK: getelementptr inbounds i64, ptr %a, i64 %[[I0]] -; CHECK: getelementptr inbounds i64, ptr %a, i64 %[[I2]] -; CHECK-NOT: add nuw nsw i64 %[[I0]], 1 -; CHECK-NOT: add nuw nsw i64 %[[I2]], 1 -; CHECK-NOT: icmp slt i64 {{.*}}, %n -; CHECK: %index.next = add nuw i64 %index, 4 -; CHECK: %[[CMP:.+]] = icmp eq i64 %index.next, %n.vec -; CHECK: br i1 %[[CMP]], label %middle.block, label %vector.body -; define i64 @dead_instructions_01(ptr %a, i64 %n) { +; CHECK-LABEL: define i64 @dead_instructions_01( +; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]] +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 2 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP4]], align 8 +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8 +; CHECK-NEXT: [[TMP6]] = add <2 x i64> [[WIDE_LOAD]], [[VEC_PHI]] +; CHECK-NEXT: [[TMP7]] = add <2 x i64> [[WIDE_LOAD2]], [[VEC_PHI1]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[BIN_RDX:%.*]] = add <2 x i64> [[TMP7]], [[TMP6]] +; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX]]) +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ] +; CHECK-NEXT: br label %[[FOR_BODY:.*]] +; CHECK: [[FOR_BODY]]: +; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] +; CHECK-NEXT: [[R:%.*]] = phi i64 [ [[TMP2:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]] +; CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr [[TMP0]], align 8 +; CHECK-NEXT: [[TMP2]] = add i64 [[TMP1]], [[R]] +; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 +; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK: [[FOR_END]]: +; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ [[TMP2]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret i64 [[TMP3]] +; entry: br label %for.body @@ -42,16 +78,44 @@ for.end: } -; CHECK-LABEL: @pr47390 ; ; This test ensures that the primary induction is not considered dead when ; acting as the 'add' of another induction, and otherwise feeding only its own ; 'add' (recognized earlier as 'dead'), when the tail of the loop is folded by ; masking. Such masking uses the primary induction. ; -; CHECK: vector.body: -; define void @pr47390(ptr %a) { +; CHECK-LABEL: define void @pr47390( +; CHECK-SAME: ptr [[A:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 +; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8 +; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 8, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 7, %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ] +; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ 9, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ] +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[PRIMARY:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PRIMARY_ADD:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[USE_PRIMARY:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[PRIMARY]], %[[LOOP]] ] +; CHECK-NEXT: [[SECONDARY:%.*]] = phi i32 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[SECONDARY_ADD:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[PRIMARY_ADD]] = add i32 [[PRIMARY]], 1 +; CHECK-NEXT: [[SECONDARY_ADD]] = add i32 [[SECONDARY]], 1 +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[SECONDARY]] +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 8 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[SECONDARY]], 5 +; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]] +; entry: br label %loop @@ -69,3 +133,92 @@ loop: %cmp = icmp eq i32 %secondary, 5 br i1 %cmp, label %exit, label %loop } + +; Test with a dead load and dead vector poiner. +define void @dead_load_and_vector_pointer(ptr %a, ptr %b) { +; CHECK-LABEL: define void @dead_load_and_vector_pointer( +; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; CHECK: [[VECTOR_MEMCHECK]]: +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 516 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 516 +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]] +; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 2 +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP0]] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP1]] +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 2 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP4]], align 8, !alias.scope [[META6:![0-9]+]], !noalias [[META9:![0-9]+]] +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP5]], align 8, !alias.scope [[META6]], !noalias [[META9]] +; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i32> [[WIDE_LOAD]], +; CHECK-NEXT: [[TMP7:%.*]] = add <2 x i32> [[WIDE_LOAD2]], +; CHECK-NEXT: store <2 x i32> [[TMP6]], ptr [[TMP4]], align 4, !alias.scope [[META6]], !noalias [[META9]] +; CHECK-NEXT: store <2 x i32> [[TMP7]], ptr [[TMP5]], align 4, !alias.scope [[META6]], !noalias [[META9]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP0]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP1]] +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 +; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 128, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[PRIMARY:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PRIMARY_ADD:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[PRIMARY_ADD]] = add i32 [[PRIMARY]], 1 +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[PRIMARY]] +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 8 +; CHECK-NEXT: [[ADD:%.*]] = add i32 [[LOAD]], 1 +; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP]], align 4 +; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[PRIMARY]] +; CHECK-NEXT: [[LOAD2:%.*]] = load i32, ptr [[GEP_B]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[PRIMARY]], 128 +; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP12:![0-9]+]] +; +entry: + br label %loop + +exit: + ret void + +loop: + %primary = phi i32 [ 0, %entry ], [ %primary_add, %loop ] + %primary_add = add i32 %primary, 1 + %gep = getelementptr inbounds i32, ptr %a, i32 %primary + %load = load i32, ptr %gep, align 8 + %add = add i32 %load, 1 + store i32 %add, ptr %gep + %gep.b = getelementptr inbounds i32, ptr %b, i32 %primary + %load2 = load i32, ptr %gep.b + %cmp = icmp eq i32 %primary, 128 + br i1 %cmp, label %exit, label %loop +} +;. +; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} +; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} +; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} +; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]} +; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]} +; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]} +; CHECK: [[META6]] = !{[[META7:![0-9]+]]} +; CHECK: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]]} +; CHECK: [[META8]] = distinct !{[[META8]], !"LVerDomain"} +; CHECK: [[META9]] = !{[[META10:![0-9]+]]} +; CHECK: [[META10]] = distinct !{[[META10]], [[META8]]} +; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]} +; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]]} +;. -- GitLab From e9adcc488f96a9f2b8c4344f5e3c7ca6639b9562 Mon Sep 17 00:00:00 2001 From: Konstantin Varlamov Date: Fri, 7 Jun 2024 08:15:02 -0700 Subject: [PATCH 263/462] [libc++][regex] Correctly adjust match prefix for zero-length matches. (#94550) For regex patterns that produce zero-length matches, there is one (imaginary) match in-between every character in the sequence being searched (as well as before the first character and after the last character). It's easiest to demonstrate using replacement: `std::regex_replace("abc"s, "!", "")` should produce `!a!b!c!`, where each exclamation mark makes a zero-length match visible. Currently our implementation doesn't correctly set the prefix of each zero-length match, "swallowing" the characters separating the imaginary matches -- e.g. when going through zero-length matches within `abc`, the corresponding prefixes should be `{'', 'a', 'b', 'c'}`, but before this patch they will all be empty (`{'', '', '', ''}`). This happens in the implementation of `regex_iterator::operator++`. Note that the Standard spells out quite explicitly that the prefix might need to be adjusted when dealing with zero-length matches in [`re.regiter.incr`](http://eel.is/c++draft/re.regiter.incr): > In all cases in which the call to `regex_search` returns `true`, `match.prefix().first` shall be equal to the previous value of `match[0].second`... It is unspecified how the implementation makes these adjustments. [Reproduction example](https://godbolt.org/z/8ve6G3dav) ```cpp #include #include #include int main() { std::string str = "abc"; std::regex empty_matching_pattern(""); { // The underlying problem is that `regex_iterator::operator++` doesn't update // the prefix correctly. std::sregex_iterator i(str.begin(), str.end(), empty_matching_pattern), e; std::cout << "\""; for (; i != e; ++i) { const std::ssub_match& prefix = i->prefix(); std::cout << prefix.str(); } std::cout << "\"\n"; // Before the patch: "" // After the patch: "abc" } { // `regex_replace` makes the problem very visible. std::string replaced = std::regex_replace(str, empty_matching_pattern, "!"); std::cout << "\"" << replaced << "\"\n"; // Before the patch: "!!!!" // After the patch: "!a!b!c!" } } ``` Fixes #64451 rdar://119912002 --- libcxx/include/regex | 21 +- .../zero_length_matches.pass.cpp | 37 +++ .../re.regiter/re.regiter.incr/post.pass.cpp | 239 +++++++++++------- 3 files changed, 200 insertions(+), 97 deletions(-) create mode 100644 libcxx/test/std/re/re.alg/re.alg.replace/zero_length_matches.pass.cpp diff --git a/libcxx/include/regex b/libcxx/include/regex index b3869d36de1d..46968357d397 100644 --- a/libcxx/include/regex +++ b/libcxx/include/regex @@ -4700,6 +4700,9 @@ private: template friend class __lookahead; + + template + friend class regex_iterator; }; template @@ -5410,7 +5413,9 @@ template regex_iterator<_BidirectionalIterator, _CharT, _Traits>& regex_iterator<_BidirectionalIterator, _CharT, _Traits>::operator++() { __flags_ |= regex_constants::__no_update_pos; - _BidirectionalIterator __start = __match_[0].second; + _BidirectionalIterator __start = __match_[0].second; + _BidirectionalIterator __prefix_start = __start; + if (__match_[0].first == __match_[0].second) { if (__start == __end_) { __match_ = value_type(); @@ -5424,9 +5429,21 @@ regex_iterator<_BidirectionalIterator, _CharT, _Traits>::operator++() { else ++__start; } + __flags_ |= regex_constants::match_prev_avail; - if (!std::regex_search(__start, __end_, __match_, *__pregex_, __flags_)) + if (!std::regex_search(__start, __end_, __match_, *__pregex_, __flags_)) { __match_ = value_type(); + + } else { + // The Standard mandates that if `regex_search` returns true ([re.regiter.incr]), "`match.prefix().first` shall be + // equal to the previous value of `match[0].second`... It is unspecified how the implementation makes these + // adjustments." The adjustment is necessary if we incremented `__start` above (the branch that deals with + // zero-length matches). + auto& __prefix = __match_.__prefix_; + __prefix.first = __prefix_start; + __prefix.matched = __prefix.first != __prefix.second; + } + return *this; } diff --git a/libcxx/test/std/re/re.alg/re.alg.replace/zero_length_matches.pass.cpp b/libcxx/test/std/re/re.alg/re.alg.replace/zero_length_matches.pass.cpp new file mode 100644 index 000000000000..2f1b6280deb7 --- /dev/null +++ b/libcxx/test/std/re/re.alg/re.alg.replace/zero_length_matches.pass.cpp @@ -0,0 +1,37 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +// Test that replacing zero-length matches works correctly. + +#include +#include +#include +#include "test_macros.h" + +int main(int, char**) { + // Various patterns that produce zero-length matches. + assert(std::regex_replace("abc", std::regex(""), "!") == "!a!b!c!"); + assert(std::regex_replace("abc", std::regex("X*"), "!") == "!a!b!c!"); + assert(std::regex_replace("abc", std::regex("X{0,3}"), "!") == "!a!b!c!"); + + // Replacement string has several characters. + assert(std::regex_replace("abc", std::regex(""), "[!]") == "[!]a[!]b[!]c[!]"); + + // Empty replacement string. + assert(std::regex_replace("abc", std::regex(""), "") == "abc"); + + // Empty input. + assert(std::regex_replace("", std::regex(""), "!") == "!"); + + // Not all matches are zero-length. + assert(std::regex_replace("abCabCa", std::regex("C*"), "!") == "!a!b!!a!b!!a!"); + + return 0; +} diff --git a/libcxx/test/std/re/re.iter/re.regiter/re.regiter.incr/post.pass.cpp b/libcxx/test/std/re/re.iter/re.regiter/re.regiter.incr/post.pass.cpp index 9332158f0de9..aefe86f15298 100644 --- a/libcxx/test/std/re/re.iter/re.regiter/re.regiter.incr/post.pass.cpp +++ b/libcxx/test/std/re/re.iter/re.regiter/re.regiter.incr/post.pass.cpp @@ -17,102 +17,151 @@ #include #include "test_macros.h" -int main(int, char**) -{ - { - std::regex phone_numbers("\\d{3}-\\d{4}"); - const char phone_book[] = "555-1234, 555-2345, 555-3456"; - std::cregex_iterator i(std::begin(phone_book), std::end(phone_book), phone_numbers); - std::cregex_iterator i2 = i; - assert(i != std::cregex_iterator()); - assert(i2!= std::cregex_iterator()); - assert((*i).size() == 1); - assert((*i).position() == 0); - assert((*i).str() == "555-1234"); - assert((*i2).size() == 1); - assert((*i2).position() == 0); - assert((*i2).str() == "555-1234"); - i++; - assert(i != std::cregex_iterator()); - assert(i2!= std::cregex_iterator()); - assert((*i).size() == 1); - assert((*i).position() == 10); - assert((*i).str() == "555-2345"); - assert((*i2).size() == 1); - assert((*i2).position() == 0); - assert((*i2).str() == "555-1234"); - i++; - assert(i != std::cregex_iterator()); - assert(i2!= std::cregex_iterator()); - assert((*i).size() == 1); - assert((*i).position() == 20); - assert((*i).str() == "555-3456"); - assert((*i2).size() == 1); - assert((*i2).position() == 0); - assert((*i2).str() == "555-1234"); - i++; - assert(i == std::cregex_iterator()); - assert(i2!= std::cregex_iterator()); - assert((*i2).size() == 1); - assert((*i2).position() == 0); - assert((*i2).str() == "555-1234"); - } - { - std::regex phone_numbers("\\d{3}-\\d{4}"); - const char phone_book[] = "555-1234, 555-2345, 555-3456"; - std::cregex_iterator i(std::begin(phone_book), std::end(phone_book), phone_numbers); - std::cregex_iterator i2 = i; - assert(i != std::cregex_iterator()); - assert(i2!= std::cregex_iterator()); - assert((*i).size() == 1); - assert((*i).position() == 0); - assert((*i).str() == "555-1234"); - assert((*i2).size() == 1); - assert((*i2).position() == 0); - assert((*i2).str() == "555-1234"); - ++i; - assert(i != std::cregex_iterator()); - assert(i2!= std::cregex_iterator()); - assert((*i).size() == 1); - assert((*i).position() == 10); - assert((*i).str() == "555-2345"); - assert((*i2).size() == 1); - assert((*i2).position() == 0); - assert((*i2).str() == "555-1234"); - ++i; - assert(i != std::cregex_iterator()); - assert(i2!= std::cregex_iterator()); - assert((*i).size() == 1); - assert((*i).position() == 20); - assert((*i).str() == "555-3456"); - assert((*i2).size() == 1); - assert((*i2).position() == 0); - assert((*i2).str() == "555-1234"); - ++i; - assert(i == std::cregex_iterator()); - assert(i2!= std::cregex_iterator()); - assert((*i2).size() == 1); - assert((*i2).position() == 0); - assert((*i2).str() == "555-1234"); - } - { // https://llvm.org/PR33681 - std::regex rex(".*"); - const char foo[] = "foo"; +void validate_prefixes(const std::regex& empty_matching_pattern) { + const char source[] = "abc"; + + std::cregex_iterator i(source, source + 3, empty_matching_pattern); + assert(!i->prefix().matched); + assert(i->prefix().length() == 0); + assert(i->prefix().first == source); + assert(i->prefix().second == source); + + ++i; + assert(i->prefix().matched); + assert(i->prefix().length() == 1); + assert(i->prefix().first == source); + assert(i->prefix().second == source + 1); + assert(i->prefix().str() == "a"); + + ++i; + assert(i->prefix().matched); + assert(i->prefix().length() == 1); + assert(i->prefix().first == source + 1); + assert(i->prefix().second == source + 2); + assert(i->prefix().str() == "b"); + + ++i; + assert(i->prefix().matched); + assert(i->prefix().length() == 1); + assert(i->prefix().first == source + 2); + assert(i->prefix().second == source + 3); + assert(i->prefix().str() == "c"); + + ++i; + assert(i == std::cregex_iterator()); +} + +void test_prefix_adjustment() { + // Check that we correctly adjust the match prefix when dealing with zero-length matches -- this is explicitly + // required by the Standard ([re.regiter.incr]: "In all cases in which the call to `regex_search` returns true, + // `match.prefix().first` shall be equal to the previous value of `match[0].second`"). For a pattern that matches + // empty sequences, there is an implicit zero-length match between every character in a string -- make sure the + // prefix of each of these matches (except the first one) is the preceding character. + + // An empty pattern produces zero-length matches. + validate_prefixes(std::regex("")); + // Any character repeated zero or more times can produce zero-length matches. + validate_prefixes(std::regex("X*")); + validate_prefixes(std::regex("X{0,3}")); +} + +int main(int, char**) { + { + std::regex phone_numbers("\\d{3}-\\d{4}"); + const char phone_book[] = "555-1234, 555-2345, 555-3456"; + std::cregex_iterator i(std::begin(phone_book), std::end(phone_book), phone_numbers); + std::cregex_iterator i2 = i; + assert(i != std::cregex_iterator()); + assert(i2 != std::cregex_iterator()); + assert((*i).size() == 1); + assert((*i).position() == 0); + assert((*i).str() == "555-1234"); + assert((*i2).size() == 1); + assert((*i2).position() == 0); + assert((*i2).str() == "555-1234"); + i++; + assert(i != std::cregex_iterator()); + assert(i2 != std::cregex_iterator()); + assert((*i).size() == 1); + assert((*i).position() == 10); + assert((*i).str() == "555-2345"); + assert((*i2).size() == 1); + assert((*i2).position() == 0); + assert((*i2).str() == "555-1234"); + i++; + assert(i != std::cregex_iterator()); + assert(i2 != std::cregex_iterator()); + assert((*i).size() == 1); + assert((*i).position() == 20); + assert((*i).str() == "555-3456"); + assert((*i2).size() == 1); + assert((*i2).position() == 0); + assert((*i2).str() == "555-1234"); + i++; + assert(i == std::cregex_iterator()); + assert(i2 != std::cregex_iterator()); + assert((*i2).size() == 1); + assert((*i2).position() == 0); + assert((*i2).str() == "555-1234"); + } + { + std::regex phone_numbers("\\d{3}-\\d{4}"); + const char phone_book[] = "555-1234, 555-2345, 555-3456"; + std::cregex_iterator i(std::begin(phone_book), std::end(phone_book), phone_numbers); + std::cregex_iterator i2 = i; + assert(i != std::cregex_iterator()); + assert(i2 != std::cregex_iterator()); + assert((*i).size() == 1); + assert((*i).position() == 0); + assert((*i).str() == "555-1234"); + assert((*i2).size() == 1); + assert((*i2).position() == 0); + assert((*i2).str() == "555-1234"); + ++i; + assert(i != std::cregex_iterator()); + assert(i2 != std::cregex_iterator()); + assert((*i).size() == 1); + assert((*i).position() == 10); + assert((*i).str() == "555-2345"); + assert((*i2).size() == 1); + assert((*i2).position() == 0); + assert((*i2).str() == "555-1234"); + ++i; + assert(i != std::cregex_iterator()); + assert(i2 != std::cregex_iterator()); + assert((*i).size() == 1); + assert((*i).position() == 20); + assert((*i).str() == "555-3456"); + assert((*i2).size() == 1); + assert((*i2).position() == 0); + assert((*i2).str() == "555-1234"); + ++i; + assert(i == std::cregex_iterator()); + assert(i2 != std::cregex_iterator()); + assert((*i2).size() == 1); + assert((*i2).position() == 0); + assert((*i2).str() == "555-1234"); + } + { // https://llvm.org/PR33681 + std::regex rex(".*"); + const char foo[] = "foo"; // The -1 is because we don't want the implicit null from the array. - std::cregex_iterator i(std::begin(foo), std::end(foo) - 1, rex); - std::cregex_iterator e; - assert(i != e); - assert((*i).size() == 1); - assert((*i).str() == "foo"); - - ++i; - assert(i != e); - assert((*i).size() == 1); - assert((*i).str() == ""); - - ++i; - assert(i == e); - } + std::cregex_iterator i(std::begin(foo), std::end(foo) - 1, rex); + std::cregex_iterator e; + assert(i != e); + assert((*i).size() == 1); + assert((*i).str() == "foo"); + + ++i; + assert(i != e); + assert((*i).size() == 1); + assert((*i).str() == ""); + + ++i; + assert(i == e); + } + + test_prefix_adjustment(); return 0; } -- GitLab From 35fa2ded2ac52151be22c206fc92b983d1fd8e30 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Fri, 7 Jun 2024 11:27:52 -0400 Subject: [PATCH 264/462] Reapply PR/87550 (#94625) Re-apply https://github.com/llvm/llvm-project/pull/87550 with fixes. Details: Some tests in fuchsia failed because of the newly added assertion. This was because `GetExceptionBreakpoint()` could be called before `g_dap.debugger` was initted. The fix here is to just lazily populate the list in GetExceptionBreakpoint() rather than assuming it's already been initted. (There is some nuisance here because we can't simply just populate it in DAP::DAP(), which is a global ctor and is called before `SBDebugger::Initialize()` is called. ) --- lldb/include/lldb/API/SBDebugger.h | 2 + lldb/include/lldb/Symbol/TypeSystem.h | 1 + lldb/source/API/SBDebugger.cpp | 4 ++ lldb/source/Symbol/TypeSystem.cpp | 11 +++++ lldb/tools/lldb-dap/DAP.cpp | 61 ++++++++++++++++++++++----- lldb/tools/lldb-dap/DAP.h | 5 ++- lldb/tools/lldb-dap/lldb-dap.cpp | 6 ++- 7 files changed, 77 insertions(+), 13 deletions(-) diff --git a/lldb/include/lldb/API/SBDebugger.h b/lldb/include/lldb/API/SBDebugger.h index af19b1faf3bf..84ea9c0f772e 100644 --- a/lldb/include/lldb/API/SBDebugger.h +++ b/lldb/include/lldb/API/SBDebugger.h @@ -57,6 +57,8 @@ public: static const char *GetBroadcasterClass(); + static bool SupportsLanguage(lldb::LanguageType language); + lldb::SBBroadcaster GetBroadcaster(); /// Get progress data from a SBEvent whose type is eBroadcastBitProgress. diff --git a/lldb/include/lldb/Symbol/TypeSystem.h b/lldb/include/lldb/Symbol/TypeSystem.h index b4025c173a18..7d48f9b31613 100644 --- a/lldb/include/lldb/Symbol/TypeSystem.h +++ b/lldb/include/lldb/Symbol/TypeSystem.h @@ -209,6 +209,7 @@ public: // TypeSystems can support more than one language virtual bool SupportsLanguage(lldb::LanguageType language) = 0; + static bool SupportsLanguageStatic(lldb::LanguageType language); // Type Completion virtual bool GetCompleteType(lldb::opaque_compiler_type_t type) = 0; diff --git a/lldb/source/API/SBDebugger.cpp b/lldb/source/API/SBDebugger.cpp index 7ef0d6efd4aa..29da7d33dd80 100644 --- a/lldb/source/API/SBDebugger.cpp +++ b/lldb/source/API/SBDebugger.cpp @@ -1742,3 +1742,7 @@ bool SBDebugger::InterruptRequested() { return m_opaque_sp->InterruptRequested(); return false; } + +bool SBDebugger::SupportsLanguage(lldb::LanguageType language) { + return TypeSystem::SupportsLanguageStatic(language); +} diff --git a/lldb/source/Symbol/TypeSystem.cpp b/lldb/source/Symbol/TypeSystem.cpp index 4956f10a0b0a..5d56d9b1829d 100644 --- a/lldb/source/Symbol/TypeSystem.cpp +++ b/lldb/source/Symbol/TypeSystem.cpp @@ -335,3 +335,14 @@ TypeSystemMap::GetTypeSystemForLanguage(lldb::LanguageType language, } return GetTypeSystemForLanguage(language); } + +bool TypeSystem::SupportsLanguageStatic(lldb::LanguageType language) { + if (language == eLanguageTypeUnknown) + return false; + + LanguageSet languages = + PluginManager::GetAllTypeSystemSupportedLanguagesForTypes(); + if (languages.Empty()) + return false; + return languages[language]; +} diff --git a/lldb/tools/lldb-dap/DAP.cpp b/lldb/tools/lldb-dap/DAP.cpp index d419f821999e..263e841b7254 100644 --- a/lldb/tools/lldb-dap/DAP.cpp +++ b/lldb/tools/lldb-dap/DAP.cpp @@ -32,14 +32,7 @@ namespace lldb_dap { DAP g_dap; DAP::DAP() - : broadcaster("lldb-dap"), - exception_breakpoints( - {{"cpp_catch", "C++ Catch", lldb::eLanguageTypeC_plus_plus}, - {"cpp_throw", "C++ Throw", lldb::eLanguageTypeC_plus_plus}, - {"objc_catch", "Objective-C Catch", lldb::eLanguageTypeObjC}, - {"objc_throw", "Objective-C Throw", lldb::eLanguageTypeObjC}, - {"swift_catch", "Swift Catch", lldb::eLanguageTypeSwift}, - {"swift_throw", "Swift Throw", lldb::eLanguageTypeSwift}}), + : broadcaster("lldb-dap"), exception_breakpoints(), focus_tid(LLDB_INVALID_THREAD_ID), stop_at_entry(false), is_attach(false), enable_auto_variable_summaries(false), enable_synthetic_child_debugging(false), @@ -65,8 +58,51 @@ DAP::DAP() DAP::~DAP() = default; +void DAP::PopulateExceptionBreakpoints() { + llvm::call_once(initExceptionBreakpoints, [this]() { + exception_breakpoints = {}; + if (lldb::SBDebugger::SupportsLanguage(lldb::eLanguageTypeC_plus_plus)) { + exception_breakpoints->emplace_back("cpp_catch", "C++ Catch", + lldb::eLanguageTypeC_plus_plus); + exception_breakpoints->emplace_back("cpp_throw", "C++ Throw", + lldb::eLanguageTypeC_plus_plus); + } + if (lldb::SBDebugger::SupportsLanguage(lldb::eLanguageTypeObjC)) { + exception_breakpoints->emplace_back("objc_catch", "Objective-C Catch", + lldb::eLanguageTypeObjC); + exception_breakpoints->emplace_back("objc_throw", "Objective-C Throw", + lldb::eLanguageTypeObjC); + } + if (lldb::SBDebugger::SupportsLanguage(lldb::eLanguageTypeSwift)) { + exception_breakpoints->emplace_back("swift_catch", "Swift Catch", + lldb::eLanguageTypeSwift); + exception_breakpoints->emplace_back("swift_throw", "Swift Throw", + lldb::eLanguageTypeSwift); + } + }); +} + ExceptionBreakpoint *DAP::GetExceptionBreakpoint(const std::string &filter) { - for (auto &bp : exception_breakpoints) { + // PopulateExceptionBreakpoints() is called after g_dap.debugger is created + // in a request-initialize. + // + // But this GetExceptionBreakpoint() method may be called before attaching, in + // which case, we may not have populated the filter yet. + // + // We also cannot call PopulateExceptionBreakpoints() in DAP::DAP() because + // we need SBDebugger::Initialize() to have been called before this. + // + // So just calling PopulateExceptionBreakoints(),which does lazy-populating + // seems easiest. Two other options include: + // + call g_dap.PopulateExceptionBreakpoints() in lldb-dap.cpp::main() + // right after the call to SBDebugger::Initialize() + // + Just call PopulateExceptionBreakpoints() to get a fresh list everytime + // we query (a bit overkill since it's not likely to change?) + PopulateExceptionBreakpoints(); + assert(exception_breakpoints.has_value() && + "exception_breakpoints must have been populated"); + + for (auto &bp : *exception_breakpoints) { if (bp.filter == filter) return &bp; } @@ -74,7 +110,12 @@ ExceptionBreakpoint *DAP::GetExceptionBreakpoint(const std::string &filter) { } ExceptionBreakpoint *DAP::GetExceptionBreakpoint(const lldb::break_id_t bp_id) { - for (auto &bp : exception_breakpoints) { + // See comment in the other GetExceptionBreakpoint(). + PopulateExceptionBreakpoints(); + assert(exception_breakpoints.has_value() && + "exception_breakpoints must have been populated"); + + for (auto &bp : *exception_breakpoints) { if (bp.bp.GetID() == bp_id) return &bp; } diff --git a/lldb/tools/lldb-dap/DAP.h b/lldb/tools/lldb-dap/DAP.h index a88ee3e1dec6..daa0d9f1aa7f 100644 --- a/lldb/tools/lldb-dap/DAP.h +++ b/lldb/tools/lldb-dap/DAP.h @@ -156,7 +156,8 @@ struct DAP { std::unique_ptr log; llvm::StringMap source_breakpoints; FunctionBreakpointMap function_breakpoints; - std::vector exception_breakpoints; + std::optional> exception_breakpoints; + llvm::once_flag initExceptionBreakpoints; std::vector init_commands; std::vector pre_run_commands; std::vector post_run_commands; @@ -228,6 +229,8 @@ struct DAP { llvm::json::Value CreateTopLevelScopes(); + void PopulateExceptionBreakpoints(); + /// \return /// Attempt to determine if an expression is a variable expression or /// lldb command using a hueristic based on the first term of the diff --git a/lldb/tools/lldb-dap/lldb-dap.cpp b/lldb/tools/lldb-dap/lldb-dap.cpp index 7746afb6cbbf..470c9f84c6a2 100644 --- a/lldb/tools/lldb-dap/lldb-dap.cpp +++ b/lldb/tools/lldb-dap/lldb-dap.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #if defined(_WIN32) @@ -1586,6 +1587,7 @@ void request_initialize(const llvm::json::Object &request) { bool source_init_file = GetBoolean(arguments, "sourceInitFile", true); g_dap.debugger = lldb::SBDebugger::Create(source_init_file, log_cb, nullptr); + g_dap.PopulateExceptionBreakpoints(); auto cmd = g_dap.debugger.GetCommandInterpreter().AddMultiwordCommand( "lldb-dap", "Commands for managing lldb-dap."); if (GetBoolean(arguments, "supportsStartDebuggingRequest", false)) { @@ -1621,7 +1623,7 @@ void request_initialize(const llvm::json::Object &request) { body.try_emplace("supportsEvaluateForHovers", true); // Available filters or options for the setExceptionBreakpoints request. llvm::json::Array filters; - for (const auto &exc_bp : g_dap.exception_breakpoints) { + for (const auto &exc_bp : *g_dap.exception_breakpoints) { filters.emplace_back(CreateExceptionBreakpointFilter(exc_bp)); } body.try_emplace("exceptionBreakpointFilters", std::move(filters)); @@ -2476,7 +2478,7 @@ void request_setExceptionBreakpoints(const llvm::json::Object &request) { // Keep a list of any exception breakpoint filter names that weren't set // so we can clear any exception breakpoints if needed. std::set unset_filters; - for (const auto &bp : g_dap.exception_breakpoints) + for (const auto &bp : *g_dap.exception_breakpoints) unset_filters.insert(bp.filter); for (const auto &value : *filters) { -- GitLab From 716ed5fccd2a960981fec2c5acb17292a1502435 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Fri, 7 Jun 2024 11:31:17 -0400 Subject: [PATCH 265/462] [libc++] Undeprecate shared_ptr atomic access APIs (#92920) This patch reverts 9b832b72 (#87111): - [libc++] Deprecated `shared_ptr` Atomic Access APIs as per P0718R2 - [libc++] Implemented P2869R3: Remove Deprecated `shared_ptr` Atomic Access APIs from C++26 As explained in [1], the suggested replacement in P2869R3 is `__cpp_lib_atomic_shared_ptr`, which libc++ does not yet implement. Let's not deprecate the old way of doing things before the new way of doing things exists. [1]: https://github.com/llvm/llvm-project/pull/87111#issuecomment-2112740039 --- libcxx/docs/ReleaseNotes/19.rst | 4 --- libcxx/docs/Status/Cxx20.rst | 2 -- libcxx/docs/Status/Cxx20Papers.csv | 2 +- libcxx/docs/Status/Cxx2cPapers.csv | 2 +- libcxx/docs/UsingLibcxx.rst | 4 --- libcxx/include/__memory/shared_ptr.h | 29 ++++++---------- libcxx/include/memory | 22 ++++++------ libcxx/modules/std/memory.inc | 6 ---- ..._exchange_strong.depr_in_cxx20..verify.cpp | 32 ----------------- ...e_strong_explicit.depr_in_cxx20.verify.cpp | 34 ------------------- ...re_exchange_weak.depr_in_cxx20..verify.cpp | 31 ----------------- ...ge_weak_explicit.depr_in_cxx20..verify.cpp | 34 ------------------- .../atomic_exchange.depr_in_cxx20..verify.cpp | 29 ---------------- ...xchange_explicit.verify.depr_in_cxx20..cpp | 30 ---------------- ...mic_is_lock_free.depr_in_cxx20..verify.cpp | 28 --------------- .../atomic_load.depr_in_cxx20..verify.cpp | 28 --------------- ...ic_load_explicit.depr_in_cxx20..verify.cpp | 29 ---------------- .../atomic_store.depr_in_cxx20..verify.cpp | 28 --------------- ...c_store_explicit.depr_in_cxx20..verify.cpp | 29 ---------------- .../atomic_compare_exchange_strong.pass.cpp | 6 ++-- ..._compare_exchange_strong_explicit.pass.cpp | 6 ++-- .../atomic_compare_exchange_weak.pass.cpp | 6 ++-- ...ic_compare_exchange_weak_explicit.pass.cpp | 6 ++-- .../atomic_exchange.pass.cpp | 6 ++-- .../atomic_exchange_explicit.pass.cpp | 6 ++-- .../atomic_is_lock_free.pass.cpp | 6 ++-- .../atomic_load.pass.cpp | 6 ++-- .../atomic_load_explicit.pass.cpp | 6 ++-- .../atomic_store.pass.cpp | 6 ++-- .../atomic_store_explicit.pass.cpp | 6 ++-- 30 files changed, 46 insertions(+), 423 deletions(-) delete mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong.depr_in_cxx20..verify.cpp delete mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.depr_in_cxx20.verify.cpp delete mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak.depr_in_cxx20..verify.cpp delete mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.depr_in_cxx20..verify.cpp delete mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange.depr_in_cxx20..verify.cpp delete mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange_explicit.verify.depr_in_cxx20..cpp delete mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_is_lock_free.depr_in_cxx20..verify.cpp delete mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load.depr_in_cxx20..verify.cpp delete mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load_explicit.depr_in_cxx20..verify.cpp delete mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store.depr_in_cxx20..verify.cpp delete mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store_explicit.depr_in_cxx20..verify.cpp diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst index 0bc343acd281..6ab4cafd2bb7 100644 --- a/libcxx/docs/ReleaseNotes/19.rst +++ b/libcxx/docs/ReleaseNotes/19.rst @@ -43,7 +43,6 @@ Implemented Papers - P2819R2 - Add ``tuple`` protocol to ``complex`` - P2495R3 - Interfacing ``stringstream``\s with ``string_view`` - P2867R2 - Remove Deprecated ``strstream``\s From C++26 -- P2869R4 - Remove Deprecated ``shared_ptr`` Atomic Access APIs from C++26 - P2872R3 - Remove ``wstring_convert`` From C++26 - P3142R0 - Printing Blank Lines with ``println`` (as DR against C++23) - P2944R3 - Comparisons for ``reference_wrapper`` (comparison operators for ``reference_wrapper`` only) @@ -69,9 +68,6 @@ Improvements and New Features - The ``_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM`` macro has been added to make the declarations in ```` available. -- The ``_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS`` macro has been added to make the declarations in ```` - available. - - The ``_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT`` macro has been added to make the declarations in ```` available. diff --git a/libcxx/docs/Status/Cxx20.rst b/libcxx/docs/Status/Cxx20.rst index b08b99394fbb..c00d6fb23728 100644 --- a/libcxx/docs/Status/Cxx20.rst +++ b/libcxx/docs/Status/Cxx20.rst @@ -58,8 +58,6 @@ Paper Status * ``GPS clock`` not done * ``UTC clock`` not done - .. [#note-P0718] P0718: Implemented deprecation of ``shared_ptr`` atomic access APIs only. - .. _issues-status-cxx20: Library Working Group Issues Status diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv index 6598cd18358f..462c7eff55d5 100644 --- a/libcxx/docs/Status/Cxx20Papers.csv +++ b/libcxx/docs/Status/Cxx20Papers.csv @@ -12,7 +12,7 @@ "`P0600R1 `__","LWG","nodiscard in the Library","Albuquerque","|Complete|","16.0" "`P0616R0 `__","LWG","de-pessimize legacy algorithms with std::move","Albuquerque","|Complete|","12.0" "`P0653R2 `__","LWG","Utility to convert a pointer to a raw pointer","Albuquerque","|Complete|","6.0" -"`P0718R2 `__","LWG","Atomic shared_ptr","Albuquerque","|Partial| [#note-P0718]_","" +"`P0718R2 `__","LWG","Atomic shared_ptr","Albuquerque","","" "`P0767R1 `__","CWG","Deprecate POD","Albuquerque","|Complete|","7.0" "`P0768R1 `__","CWG","Library Support for the Spaceship (Comparison) Operator","Albuquerque","|Complete|","" "`P0777R1 `__","LWG","Treating Unnecessary ``decay``\ ","Albuquerque","|Complete|","7.0" diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv index 30a601858b63..ea060f18acde 100644 --- a/libcxx/docs/Status/Cxx2cPapers.csv +++ b/libcxx/docs/Status/Cxx2cPapers.csv @@ -48,7 +48,7 @@ "","","","","","","" "`P2875R4 `__","LWG","Undeprecate ``polymorphic_allocator::destroy`` for C++26","Tokyo March 2024","|Complete|","15.0","" "`P2867R2 `__","LWG","Remove Deprecated ``strstreams`` From C++26","Tokyo March 2024","|Complete|","19.0","" -"`P2869R4 `__","LWG","Remove Deprecated ``shared_ptr`` Atomic Access APIs from C++26","Tokyo March 2024","|Complete|","19.0","" +"`P2869R4 `__","LWG","Remove Deprecated ``shared_ptr`` Atomic Access APIs from C++26","Tokyo March 2024","","","" "`P2872R3 `__","LWG","Remove ``wstring_convert`` From C++26","Tokyo March 2024","|Complete|","19.0","" "`P3107R5 `__","LWG","Permit an efficient implementation of ``std::print``","Tokyo March 2024","","","|format| |DR|" "`P3142R0 `__","LWG","Printing Blank Lines with ``println``","Tokyo March 2024","|Complete| [#note-P3142R0]_","19.0","|format|" diff --git a/libcxx/docs/UsingLibcxx.rst b/libcxx/docs/UsingLibcxx.rst index 0fdaeb433ac6..df08875c13be 100644 --- a/libcxx/docs/UsingLibcxx.rst +++ b/libcxx/docs/UsingLibcxx.rst @@ -252,14 +252,10 @@ C++26 Specific Configuration Macros **_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM**: This macro is used to re-enable all named declarations in ````. -**_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS**: - This macro is used to re-enable all ``shared_ptr`` atomic access APIs in ````. - **_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT**: This macro is used to re-enable the ``wstring_convert`` and ``wbuffer_convert`` in ````. - Libc++ Extensions ================= diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h index de5707c4a67b..00db96185be7 100644 --- a/libcxx/include/__memory/shared_ptr.h +++ b/libcxx/include/__memory/shared_ptr.h @@ -1583,15 +1583,13 @@ private: _LIBCPP_EXPORTED_FROM_ABI __sp_mut& __get_sp_mut(const void*); -# if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS) - template -_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const shared_ptr<_Tp>*) { +inline _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const shared_ptr<_Tp>*) { return false; } template -_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load(const shared_ptr<_Tp>* __p) { +_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load(const shared_ptr<_Tp>* __p) { __sp_mut& __m = std::__get_sp_mut(__p); __m.lock(); shared_ptr<_Tp> __q = *__p; @@ -1600,13 +1598,12 @@ _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load(co } template -_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> -atomic_load_explicit(const shared_ptr<_Tp>* __p, memory_order) { +inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load_explicit(const shared_ptr<_Tp>* __p, memory_order) { return std::atomic_load(__p); } template -_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void atomic_store(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) { +_LIBCPP_HIDE_FROM_ABI void atomic_store(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) { __sp_mut& __m = std::__get_sp_mut(__p); __m.lock(); __p->swap(__r); @@ -1614,14 +1611,12 @@ _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void atomic_store(shared_ptr<_ } template -_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void -atomic_store_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r, memory_order) { +inline _LIBCPP_HIDE_FROM_ABI void atomic_store_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r, memory_order) { std::atomic_store(__p, __r); } template -_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> -atomic_exchange(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) { +_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_exchange(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) { __sp_mut& __m = std::__get_sp_mut(__p); __m.lock(); __p->swap(__r); @@ -1630,13 +1625,13 @@ atomic_exchange(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) { } template -_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> +inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_exchange_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r, memory_order) { return std::atomic_exchange(__p, __r); } template -_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w) { shared_ptr<_Tp> __temp; __sp_mut& __m = std::__get_sp_mut(__p); @@ -1654,25 +1649,23 @@ atomic_compare_exchange_strong(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, share } template -_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool +inline _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w) { return std::atomic_compare_exchange_strong(__p, __v, __w); } template -_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit( +inline _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit( shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w, memory_order, memory_order) { return std::atomic_compare_exchange_strong(__p, __v, __w); } template -_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit( +inline _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit( shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w, memory_order, memory_order) { return std::atomic_compare_exchange_weak(__p, __v, __w); } -# endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS) - #endif // !defined(_LIBCPP_HAS_NO_THREADS) _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/memory b/libcxx/include/memory index 30fadce760d7..a8c0264eb9eb 100644 --- a/libcxx/include/memory +++ b/libcxx/include/memory @@ -830,34 +830,34 @@ public: }; template - bool atomic_is_lock_free(const shared_ptr* p); // Deprecated in C++20, removed in C++26 + bool atomic_is_lock_free(const shared_ptr* p); template - shared_ptr atomic_load(const shared_ptr* p); // Deprecated in C++20, removed in C++26 + shared_ptr atomic_load(const shared_ptr* p); template - shared_ptr atomic_load_explicit(const shared_ptr* p, memory_order mo); // Deprecated in C++20, removed in C++26 + shared_ptr atomic_load_explicit(const shared_ptr* p, memory_order mo); template - void atomic_store(shared_ptr* p, shared_ptr r); // Deprecated in C++20, removed in C++26 + void atomic_store(shared_ptr* p, shared_ptr r); template - void atomic_store_explicit(shared_ptr* p, shared_ptr r, memory_order mo); // Deprecated in C++20, removed in C++26 + void atomic_store_explicit(shared_ptr* p, shared_ptr r, memory_order mo); template - shared_ptr atomic_exchange(shared_ptr* p, shared_ptr r); // Deprecated in C++20, removed in C++26 + shared_ptr atomic_exchange(shared_ptr* p, shared_ptr r); template shared_ptr - atomic_exchange_explicit(shared_ptr* p, shared_ptr r, memory_order mo); // Deprecated in C++20, removed in C++26 + atomic_exchange_explicit(shared_ptr* p, shared_ptr r, memory_order mo); template bool - atomic_compare_exchange_weak(shared_ptr* p, shared_ptr* v, shared_ptr w); // Deprecated in C++20, removed in C++26 + atomic_compare_exchange_weak(shared_ptr* p, shared_ptr* v, shared_ptr w); template bool - atomic_compare_exchange_strong( shared_ptr* p, shared_ptr* v, shared_ptr w); // Deprecated in C++20, removed in C++26 + atomic_compare_exchange_strong( shared_ptr* p, shared_ptr* v, shared_ptr w); template bool - atomic_compare_exchange_weak_explicit(shared_ptr* p, shared_ptr* v, // Deprecated in C++20, removed in C++26 + atomic_compare_exchange_weak_explicit(shared_ptr* p, shared_ptr* v, shared_ptr w, memory_order success, memory_order failure); template bool - atomic_compare_exchange_strong_explicit(shared_ptr* p, shared_ptr* v, // Deprecated in C++20, removed in C++26 + atomic_compare_exchange_strong_explicit(shared_ptr* p, shared_ptr* v, shared_ptr w, memory_order success, memory_order failure); // Hash support diff --git a/libcxx/modules/std/memory.inc b/libcxx/modules/std/memory.inc index b23c27707afd..56c621c0cf17 100644 --- a/libcxx/modules/std/memory.inc +++ b/libcxx/modules/std/memory.inc @@ -190,9 +190,6 @@ export namespace std { // using std::inout_ptr; #ifndef _LIBCPP_HAS_NO_THREADS - -# if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS) - // [depr.util.smartptr.shared.atomic] using std::atomic_is_lock_free; @@ -209,8 +206,5 @@ export namespace std { using std::atomic_compare_exchange_strong_explicit; using std::atomic_compare_exchange_weak; using std::atomic_compare_exchange_weak_explicit; - -# endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS) - #endif // _LIBCPP_HAS_NO_THREADS } // namespace std diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong.depr_in_cxx20..verify.cpp deleted file mode 100644 index 3ca01cad451a..000000000000 --- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong.depr_in_cxx20..verify.cpp +++ /dev/null @@ -1,32 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// UNSUPPORTED: no-threads -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// - -// shared_ptr - -// template -// bool -// atomic_compare_exchange_strong(shared_ptr* p, shared_ptr* v, -// shared_ptr w); // Deprecated in C++20, removed in C++26 - -#include -#include -#include - -void test() { - std::shared_ptr p(new int(4)); - std::shared_ptr v(new int(3)); - std::shared_ptr w(new int(2)); - // expected-warning@+1 {{'atomic_compare_exchange_strong' is deprecated}} - std::ignore = std::atomic_compare_exchange_strong(&p, &v, w); -} diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.depr_in_cxx20.verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.depr_in_cxx20.verify.cpp deleted file mode 100644 index 93384772c8af..000000000000 --- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.depr_in_cxx20.verify.cpp +++ /dev/null @@ -1,34 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// UNSUPPORTED: no-threads -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// - -// shared_ptr - -// template -// bool -// atomic_compare_exchange_strong_explicit(shared_ptr* p, shared_ptr* v, -// shared_ptr w, memory_order success, -// memory_order failure); // Deprecated in C++20, removed in C++26 - -#include -#include -#include - -void test() { - std::shared_ptr p(new int(4)); - std::shared_ptr v(new int(3)); - std::shared_ptr w(new int(2)); - // expected-warning@+2 {{'atomic_compare_exchange_strong_explicit' is deprecated}} - std::ignore = - std::atomic_compare_exchange_strong_explicit(&p, &v, w, std::memory_order_seq_cst, std::memory_order_seq_cst); -} diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak.depr_in_cxx20..verify.cpp deleted file mode 100644 index 30ca70523fb6..000000000000 --- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak.depr_in_cxx20..verify.cpp +++ /dev/null @@ -1,31 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// UNSUPPORTED: no-threads -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// - -// shared_ptr - -// template -// bool -// atomic_compare_exchange_weak(shared_ptr* p, shared_ptr* v, -// shared_ptr w); // Deprecated in C++20, removed in C++26 - -#include -#include - -int main(int, char**) { - std::shared_ptr p(new int(4)); - std::shared_ptr v(new int(3)); - std::shared_ptr w(new int(2)); - // expected-warning@+1 {{'atomic_compare_exchange_weak' is deprecated}} - std::ignore = std::atomic_compare_exchange_weak(&p, &v, w); -} diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.depr_in_cxx20..verify.cpp deleted file mode 100644 index ae6e98d0692d..000000000000 --- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.depr_in_cxx20..verify.cpp +++ /dev/null @@ -1,34 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// UNSUPPORTED: no-threads -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// - -// shared_ptr - -// template -// bool -// atomic_compare_exchange_weak_explicit(shared_ptr* p, shared_ptr* v, -// shared_ptr w, memory_order success, -// memory_order failure); // Deprecated in C++20, removed in C++26 - -#include -#include -#include - -void test() { - std::shared_ptr p(new int(4)); - std::shared_ptr v(new int(3)); - std::shared_ptr w(new int(2)); - // expected-warning@+2 {{'atomic_compare_exchange_weak_explicit' is deprecated}} - std::ignore = - std::atomic_compare_exchange_weak_explicit(&p, &v, w, std::memory_order_seq_cst, std::memory_order_seq_cst); -} diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange.depr_in_cxx20..verify.cpp deleted file mode 100644 index 0e4bc1bc50b1..000000000000 --- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange.depr_in_cxx20..verify.cpp +++ /dev/null @@ -1,29 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// UNSUPPORTED: no-threads -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// - -// shared_ptr - -// template -// shared_ptr -// atomic_exchange(shared_ptr* p, shared_ptr r) // Deprecated in C++20, removed in C++26 - -#include -#include - -void test() { - std::shared_ptr p(new int(4)); - std::shared_ptr r(new int(3)); - // expected-warning@+1 {{'atomic_exchange' is deprecated}} - std::ignore = std::atomic_exchange(&p, r); -} diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange_explicit.verify.depr_in_cxx20..cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange_explicit.verify.depr_in_cxx20..cpp deleted file mode 100644 index 5a6a2bfe4c1e..000000000000 --- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange_explicit.verify.depr_in_cxx20..cpp +++ /dev/null @@ -1,30 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// UNSUPPORTED: no-threads -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// - -// shared_ptr - -// template -// shared_ptr -// atomic_exchange_explicit(shared_ptr* p, shared_ptr r) // Deprecated in C++20, removed in C++26 - -#include -#include -#include - -void test() { - std::shared_ptr p(new int(4)); - std::shared_ptr r(new int(3)); - // expected-warning@+1 {{'atomic_exchange_explicit' is deprecated}} - std::ignore = std::atomic_exchange_explicit(&p, r, std::memory_order_seq_cst); -} diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_is_lock_free.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_is_lock_free.depr_in_cxx20..verify.cpp deleted file mode 100644 index 6799d31ce1fa..000000000000 --- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_is_lock_free.depr_in_cxx20..verify.cpp +++ /dev/null @@ -1,28 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// UNSUPPORTED: no-threads -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// - -// shared_ptr - -// template -// bool -// atomic_is_lock_free(const shared_ptr* p); // Deprecated in C++20, removed in C++26 - -#include -#include - -void test() { - const std::shared_ptr p(new int(3)); - // expected-warning@+1 {{'atomic_is_lock_free' is deprecated}} - std::ignore = std::atomic_is_lock_free(&p); -} diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load.depr_in_cxx20..verify.cpp deleted file mode 100644 index 768ad0ded983..000000000000 --- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load.depr_in_cxx20..verify.cpp +++ /dev/null @@ -1,28 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// UNSUPPORTED: no-threads -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// - -// shared_ptr - -// template -// shared_ptr -// atomic_load(const shared_ptr* p) // Deprecated in C++20, removed in C++26 - -#include -#include - -void test() { - std::shared_ptr p(new int(3)); - // expected-warning@+1 {{'atomic_load' is deprecated}} - std::ignore = std::atomic_load(&p); -} diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load_explicit.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load_explicit.depr_in_cxx20..verify.cpp deleted file mode 100644 index dc56193324f0..000000000000 --- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load_explicit.depr_in_cxx20..verify.cpp +++ /dev/null @@ -1,29 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// UNSUPPORTED: no-threads -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// - -// shared_ptr - -// template -// shared_ptr -// atomic_load_explicit(const shared_ptr* p, memory_order mo) // Deprecated in C++20, removed in C++26 - -#include -#include -#include - -void test() { - const std::shared_ptr p(new int(3)); - // expected-warning@+1 {{'atomic_load_explicit' is deprecated}} - std::ignore = std::atomic_load_explicit(&p, std::memory_order_relaxed); -} diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store.depr_in_cxx20..verify.cpp deleted file mode 100644 index 38b8df6f2439..000000000000 --- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store.depr_in_cxx20..verify.cpp +++ /dev/null @@ -1,28 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// UNSUPPORTED: no-threads -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// - -// shared_ptr - -// template -// void -// atomic_store(shared_ptr* p, shared_ptr r) // Deprecated in C++20, removed in C++26 - -#include - -void test() { - std::shared_ptr p; - std::shared_ptr r(new int(3)); - // expected-warning@+1 {{'atomic_store' is deprecated}} - std::atomic_store(&p, r); -} diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store_explicit.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store_explicit.depr_in_cxx20..verify.cpp deleted file mode 100644 index 18f59c461869..000000000000 --- a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store_explicit.depr_in_cxx20..verify.cpp +++ /dev/null @@ -1,29 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// UNSUPPORTED: no-threads -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// - -// shared_ptr - -// template -// void -// atomic_store_explicit(shared_ptr* p, shared_ptr r, memory_order mo) // Deprecated in C++20, removed in C++26 - -#include -#include - -void test() { - std::shared_ptr p; - std::shared_ptr r(new int(3)); - // expected-warning@+1 {{'atomic_store_explicit' is deprecated}} - std::atomic_store_explicit(&p, r, std::memory_order_seq_cst); -} diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong.pass.cpp index ad9d2a9d9ccc..37be6ceea3e0 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong.pass.cpp @@ -5,9 +5,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +// // UNSUPPORTED: no-threads // @@ -17,7 +15,7 @@ // template // bool // atomic_compare_exchange_strong(shared_ptr* p, shared_ptr* v, -// shared_ptr w); // Deprecated in C++20, removed in C++26 +// shared_ptr w); // UNSUPPORTED: c++03 diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.pass.cpp index 26496df96d47..3965863b86cc 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.pass.cpp @@ -5,9 +5,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +// // UNSUPPORTED: no-threads // @@ -18,7 +16,7 @@ // bool // atomic_compare_exchange_strong_explicit(shared_ptr* p, shared_ptr* v, // shared_ptr w, memory_order success, -// memory_order failure); // Deprecated in C++20, removed in C++26 +// memory_order failure); // UNSUPPORTED: c++03 diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak.pass.cpp index 5cdc0ea3312f..6dd04f924a10 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak.pass.cpp @@ -5,9 +5,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +// // UNSUPPORTED: no-threads // @@ -17,7 +15,7 @@ // template // bool // atomic_compare_exchange_weak(shared_ptr* p, shared_ptr* v, -// shared_ptr w); // Deprecated in C++20, removed in C++26 +// shared_ptr w); // UNSUPPORTED: c++03 diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.pass.cpp index e4cc4cf77a64..4837fa9793a5 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.pass.cpp @@ -5,9 +5,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +// // UNSUPPORTED: no-threads // @@ -18,7 +16,7 @@ // bool // atomic_compare_exchange_weak_explicit(shared_ptr* p, shared_ptr* v, // shared_ptr w, memory_order success, -// memory_order failure); // Deprecated in C++20, removed in C++26 +// memory_order failure); // UNSUPPORTED: c++03 diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange.pass.cpp index db02c9d9f75b..f488e0ed1d14 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange.pass.cpp @@ -5,9 +5,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +// // UNSUPPORTED: no-threads // @@ -16,7 +14,7 @@ // template // shared_ptr -// atomic_exchange(shared_ptr* p, shared_ptr r) // Deprecated in C++20, removed in C++26 +// atomic_exchange(shared_ptr* p, shared_ptr r) // UNSUPPORTED: c++03 diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange_explicit.pass.cpp index 704232e71fa4..1945f7bba6dc 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange_explicit.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange_explicit.pass.cpp @@ -5,9 +5,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +// // UNSUPPORTED: no-threads // @@ -16,7 +14,7 @@ // template // shared_ptr -// atomic_exchange_explicit(shared_ptr* p, shared_ptr r) // Deprecated in C++20, removed in C++26 +// atomic_exchange_explicit(shared_ptr* p, shared_ptr r) // UNSUPPORTED: c++03 diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_is_lock_free.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_is_lock_free.pass.cpp index 8f3b5bbb5155..37f7d12eda2c 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_is_lock_free.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_is_lock_free.pass.cpp @@ -5,9 +5,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +// // UNSUPPORTED: no-threads // @@ -16,7 +14,7 @@ // template // bool -// atomic_is_lock_free(const shared_ptr* p); // Deprecated in C++20, removed in C++26 +// atomic_is_lock_free(const shared_ptr* p); // UNSUPPORTED: c++03 diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load.pass.cpp index 9d81e81b4392..1b9a15ac92ac 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load.pass.cpp @@ -5,9 +5,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +// // UNSUPPORTED: no-threads // @@ -16,7 +14,7 @@ // template // shared_ptr -// atomic_load(const shared_ptr* p) // Deprecated in C++20, removed in C++26 +// atomic_load(const shared_ptr* p) // UNSUPPORTED: c++03 diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load_explicit.pass.cpp index f4875dce38a8..5c2970133328 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load_explicit.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load_explicit.pass.cpp @@ -5,9 +5,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +// // UNSUPPORTED: no-threads // @@ -16,7 +14,7 @@ // template // shared_ptr -// atomic_load_explicit(const shared_ptr* p, memory_order mo) // Deprecated in C++20, removed in C++26 +// atomic_load_explicit(const shared_ptr* p, memory_order mo) // UNSUPPORTED: c++03 diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store.pass.cpp index 836dcb16ff32..5b7bd5fad69c 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store.pass.cpp @@ -5,9 +5,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +// // UNSUPPORTED: no-threads // @@ -16,7 +14,7 @@ // template // void -// atomic_store(shared_ptr* p, shared_ptr r) // Deprecated in C++20, removed in C++26 +// atomic_store(shared_ptr* p, shared_ptr r) // UNSUPPORTED: c++03 diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store_explicit.pass.cpp index 85b31c017d73..571219042130 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store_explicit.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store_explicit.pass.cpp @@ -5,9 +5,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +// // UNSUPPORTED: no-threads // @@ -16,7 +14,7 @@ // template // void -// atomic_store_explicit(shared_ptr* p, shared_ptr r, memory_order mo) // Deprecated in C++20, removed in C++26 +// atomic_store_explicit(shared_ptr* p, shared_ptr r, memory_order mo) // UNSUPPORTED: c++03 -- GitLab From 97b12df2cc27e1ef376692a0c5b62e1f2e793970 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 7 Jun 2024 14:20:23 +0100 Subject: [PATCH 266/462] [Reassociate] shifttest.ll - generate test checks to replace custom grep expression (and remove an unused argument) --- llvm/test/Transforms/Reassociate/shifttest.ll | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/llvm/test/Transforms/Reassociate/shifttest.ll b/llvm/test/Transforms/Reassociate/shifttest.ll index 102166e16c05..84ac5396a3a2 100644 --- a/llvm/test/Transforms/Reassociate/shifttest.ll +++ b/llvm/test/Transforms/Reassociate/shifttest.ll @@ -1,12 +1,17 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt < %s -passes=reassociate,instcombine -S | FileCheck %s + ; With shl->mul reassociation, we can see that this is (shl A, 9) * A +define i32 @test(i32 %A) { +; CHECK-LABEL: define i32 @test( +; CHECK-SAME: i32 [[A:%.*]]) { +; CHECK-NEXT: [[Y:%.*]] = shl i32 [[A]], 9 +; CHECK-NEXT: [[Z:%.*]] = mul i32 [[Y]], [[A]] +; CHECK-NEXT: ret i32 [[Z]] ; -; RUN: opt < %s -passes=reassociate,instcombine -S |\ -; RUN: grep "shl .*, 9" - -define i32 @test(i32 %A, i32 %B) { - %X = shl i32 %A, 5 ; [#uses=1] - %Y = shl i32 %A, 4 ; [#uses=1] - %Z = mul i32 %Y, %X ; [#uses=1] - ret i32 %Z + %X = shl i32 %A, 5 ; [#uses=1] + %Y = shl i32 %A, 4 ; [#uses=1] + %Z = mul i32 %Y, %X ; [#uses=1] + ret i32 %Z } -- GitLab From b01ac5137c28fa5e1b44a5d850cb7a6ace7d8799 Mon Sep 17 00:00:00 2001 From: jeanPerier Date: Fri, 7 Jun 2024 17:38:11 +0200 Subject: [PATCH 267/462] [flang][runtime] add SHAPE runtime interface (#94702) Add SHAPE runtime API (will be used for assumed-rank, lowering is generating other cases inline). I tried to make it in a way were there is no dynamic allocation in the runtime/deallocation expected to be inserted by inline code for arrays that we know are small (lowering will just always stack allocate a rank 15 array to avoid dynamic stack allocation or heap allocation). --- flang/include/flang/Runtime/inquiry.h | 9 ++++++-- flang/runtime/inquiry.cpp | 19 ++++++++++++++++ flang/unittests/Runtime/Inquiry.cpp | 31 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/flang/include/flang/Runtime/inquiry.h b/flang/include/flang/Runtime/inquiry.h index 3fe670b0fae3..7161d1e41c4b 100644 --- a/flang/include/flang/Runtime/inquiry.h +++ b/flang/include/flang/Runtime/inquiry.h @@ -23,13 +23,18 @@ extern "C" { std::int64_t RTDECL(LboundDim)(const Descriptor &array, int dim, const char *sourceFile = nullptr, int line = 0); -void RTDECL(Ubound)(Descriptor &result, const Descriptor &array, int kind, - const char *sourceFile = nullptr, int line = 0); + +void RTDECL(Shape)(void *result, const Descriptor &array, int kind); + std::int64_t RTDECL(Size)( const Descriptor &array, const char *sourceFile = nullptr, int line = 0); + std::int64_t RTDECL(SizeDim)(const Descriptor &array, int dim, const char *sourceFile = nullptr, int line = 0); +void RTDECL(Ubound)(Descriptor &result, const Descriptor &array, int kind, + const char *sourceFile = nullptr, int line = 0); + } // extern "C" } // namespace Fortran::runtime #endif // FORTRAN_RUNTIME_INQUIRY_H_ diff --git a/flang/runtime/inquiry.cpp b/flang/runtime/inquiry.cpp index 2b59a1cfab1a..ea114174de7f 100644 --- a/flang/runtime/inquiry.cpp +++ b/flang/runtime/inquiry.cpp @@ -18,6 +18,15 @@ namespace Fortran::runtime { +template struct RawStoreIntegerAt { + RT_API_ATTRS void operator()( + void *contiguousIntegerArray, std::size_t at, std::int64_t value) const { + reinterpret_cast *>( + contiguousIntegerArray)[at] = value; + } +}; + extern "C" { std::int64_t RTDEF(LboundDim)( const Descriptor &array, int dim, const char *sourceFile, int line) { @@ -76,5 +85,15 @@ std::int64_t RTDEF(SizeDim)( return static_cast(dimension.Extent()); } +void RTDEF(Shape)(void *result, const Descriptor &array, int kind) { + Terminator terminator{__FILE__, __LINE__}; + INTERNAL_CHECK(array.rank() <= common::maxRank); + for (SubscriptValue i{0}; i < array.rank(); ++i) { + const Dimension &dimension{array.GetDimension(i)}; + Fortran::runtime::ApplyIntegerKind( + kind, terminator, result, i, dimension.Extent()); + } +} + } // extern "C" } // namespace Fortran::runtime diff --git a/flang/unittests/Runtime/Inquiry.cpp b/flang/unittests/Runtime/Inquiry.cpp index 5b97bb239f49..665a930ee4ff 100644 --- a/flang/unittests/Runtime/Inquiry.cpp +++ b/flang/unittests/Runtime/Inquiry.cpp @@ -76,3 +76,34 @@ TEST(Inquiry, Size) { EXPECT_EQ(RTNAME(SizeDim)(*array, 2, __FILE__, __LINE__), std::int64_t{3}); EXPECT_EQ(RTNAME(Size)(*array, __FILE__, __LINE__), std::int64_t{6}); } + +TEST(Inquiry, Shape) { + // ARRAY 1 3 5 + // 2 4 6 + auto array{MakeArray( + std::vector{2, 3}, std::vector{1, 2, 3, 4, 5, 6})}; + + // SHAPE(ARRAY, KIND=1) + auto int8Result{ + MakeArray(std::vector{array->rank()}, + std::vector(array->rank(), 0))}; + RTNAME(Shape)(int8Result->raw().base_addr, *array, /*KIND=*/1); + EXPECT_EQ(*int8Result->ZeroBasedIndexedElement(0), 2); + EXPECT_EQ(*int8Result->ZeroBasedIndexedElement(1), 3); + + // SHAPE(ARRAY, KIND=4) + auto int32Result{ + MakeArray(std::vector{array->rank()}, + std::vector(array->rank(), 0))}; + RTNAME(Shape)(int32Result->raw().base_addr, *array, /*KIND=*/4); + EXPECT_EQ(*int32Result->ZeroBasedIndexedElement(0), 2); + EXPECT_EQ(*int32Result->ZeroBasedIndexedElement(1), 3); + + // SHAPE(ARRAY, KIND=8) + auto int64Result{ + MakeArray(std::vector{array->rank()}, + std::vector(array->rank(), 0))}; + RTNAME(Shape)(int64Result->raw().base_addr, *array, /*KIND=*/8); + EXPECT_EQ(*int64Result->ZeroBasedIndexedElement(0), 2); + EXPECT_EQ(*int64Result->ZeroBasedIndexedElement(1), 3); +} -- GitLab From 1539da4601448711fcfa622e26e596973d58c670 Mon Sep 17 00:00:00 2001 From: Kareem Ergawy Date: Fri, 7 Jun 2024 18:08:25 +0200 Subject: [PATCH 268/462] [flang][OpenMP] Add `--openmp-enable-delayed-privatization-staging` flag (#94749) --- flang/lib/Lower/OpenMP/OpenMP.cpp | 2 +- flang/lib/Lower/OpenMP/Utils.cpp | 6 ++++++ flang/lib/Lower/OpenMP/Utils.h | 1 + .../DelayedPrivatization/target-private-allocatable.f90 | 4 ++-- .../target-private-multiple-variables.f90 | 4 ++-- .../OpenMP/DelayedPrivatization/target-private-simple.f90 | 4 ++-- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index 184c43ff9fe9..6b391e11beb4 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -1114,7 +1114,7 @@ static void genTargetClauses( llvm::omp::Directive::OMPD_target); // `target private(..)` is only supported in delayed privatization mode. - if (!enableDelayedPrivatization) + if (!enableDelayedPrivatizationStaging) cp.processTODO(loc, llvm::omp::Directive::OMPD_target); } diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp index da94352a84a7..36d96f37ff36 100644 --- a/flang/lib/Lower/OpenMP/Utils.cpp +++ b/flang/lib/Lower/OpenMP/Utils.cpp @@ -36,6 +36,12 @@ llvm::cl::opt enableDelayedPrivatization( "Emit `[first]private` variables as clauses on the MLIR ops."), llvm::cl::init(false)); +llvm::cl::opt enableDelayedPrivatizationStaging( + "openmp-enable-delayed-privatization-staging", + llvm::cl::desc("For partially supported constructs, emit `[first]private` " + "variables as clauses on the MLIR ops."), + llvm::cl::init(false)); + namespace Fortran { namespace lower { namespace omp { diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h index d20f9187640f..0b4fe9044bfa 100644 --- a/flang/lib/Lower/OpenMP/Utils.h +++ b/flang/lib/Lower/OpenMP/Utils.h @@ -17,6 +17,7 @@ extern llvm::cl::opt treatIndexAsSection; extern llvm::cl::opt enableDelayedPrivatization; +extern llvm::cl::opt enableDelayedPrivatizationStaging; namespace fir { class FirOpBuilder; diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90 index 17a28c6a5ab7..a27de1152ce1 100644 --- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90 +++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-allocatable.f90 @@ -1,8 +1,8 @@ ! Tests delayed privatization for `targets ... private(..)` for allocatables. -! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization \ +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization-staging \ ! RUN: -o - %s 2>&1 | FileCheck %s -! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 \ +! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization-staging -o - %s 2>&1 \ ! RUN: | FileCheck %s subroutine target_allocatable diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 index 8682a420e89d..6e8282b2af62 100644 --- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 +++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 @@ -1,8 +1,8 @@ ! Tests delayed privatization for `targets ... private(..)` for allocatables. -! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization \ +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization-staging \ ! RUN: -o - %s 2>&1 | FileCheck %s -! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 \ +! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization-staging -o - %s 2>&1 \ ! RUN: | FileCheck %s subroutine target_allocatable(lb, ub, l) diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90 index 94edeaa5a7ef..524e973780c4 100644 --- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90 +++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-simple.f90 @@ -1,8 +1,8 @@ ! Tests delayed privatization for `targets ... private(..)` for simple variables. -! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization \ +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization-staging \ ! RUN: -o - %s 2>&1 | FileCheck %s -! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 \ +! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization-staging -o - %s 2>&1 \ ! RUN: | FileCheck %s subroutine target_simple -- GitLab From 374f6554c3e409e4b9b5fd0ec90c5272768f5ab9 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 7 Jun 2024 11:14:16 -0500 Subject: [PATCH 269/462] [OpenMP] Fix passing target id features to AMDGPU offloading (#94765) Summary: AMDGPU supports a `target-id` feature which is used to qualify targets with different incompatible features. These are both rules and target features. Currently, we pass `-target-cpu` twice when offloading to OpenMP, and do not pass the target-id features at all. The effect was that passing something like `--offload-arch=gfx90a:xnack+` would show up as `-target-cpu=gfx90a:xnack+ -target-cpu=gfx90a`. Thus ignoring the xnack completely and passing it twice. This patch fixes that to pass it once and then separate it like how HIP does. --- clang/lib/Driver/ToolChains/AMDGPU.cpp | 6 +++++- clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp | 5 ----- clang/test/Driver/amdgpu-openmp-toolchain.c | 3 ++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp index 11a98a0ec314..20f879e2f75c 100644 --- a/clang/lib/Driver/ToolChains/AMDGPU.cpp +++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp @@ -645,7 +645,11 @@ void amdgpu::getAMDGPUTargetFeatures(const Driver &D, std::vector &Features) { // Add target ID features to -target-feature options. No diagnostics should // be emitted here since invalid target ID is diagnosed at other places. - StringRef TargetID = Args.getLastArgValue(options::OPT_mcpu_EQ); + StringRef TargetID; + if (Args.hasArg(options::OPT_mcpu_EQ)) + TargetID = Args.getLastArgValue(options::OPT_mcpu_EQ); + else if (Args.hasArg(options::OPT_march_EQ)) + TargetID = Args.getLastArgValue(options::OPT_march_EQ); if (!TargetID.empty()) { llvm::StringMap FeatureMap; auto OptionalGpuArch = parseTargetID(Triple, TargetID, &FeatureMap); diff --git a/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp b/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp index cca18431ff77..d17ecb15c820 100644 --- a/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp +++ b/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp @@ -44,14 +44,9 @@ void AMDGPUOpenMPToolChain::addClangTargetOptions( Action::OffloadKind DeviceOffloadingKind) const { HostTC.addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadingKind); - StringRef GPUArch = DriverArgs.getLastArgValue(options::OPT_march_EQ); - assert(!GPUArch.empty() && "Must have an explicit GPU arch."); - assert(DeviceOffloadingKind == Action::OFK_OpenMP && "Only OpenMP offloading kinds are supported."); - CC1Args.push_back("-target-cpu"); - CC1Args.push_back(DriverArgs.MakeArgStringRef(GPUArch)); CC1Args.push_back("-fcuda-is-device"); if (DriverArgs.hasArg(options::OPT_nogpulib)) diff --git a/clang/test/Driver/amdgpu-openmp-toolchain.c b/clang/test/Driver/amdgpu-openmp-toolchain.c index ef58c2c4e3f3..49af04acc463 100644 --- a/clang/test/Driver/amdgpu-openmp-toolchain.c +++ b/clang/test/Driver/amdgpu-openmp-toolchain.c @@ -7,7 +7,7 @@ // verify the tools invocations // CHECK: "-cc1" "-triple" "x86_64-unknown-linux-gnu"{{.*}}"-emit-llvm-bc"{{.*}}"-x" "c" -// CHECK: "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"{{.*}}"-target-cpu" "gfx906"{{.*}}"-fcuda-is-device"{{.*}} +// CHECK: "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"{{.*}}"-fcuda-is-device"{{.*}}"-target-cpu" "gfx906" // CHECK: "-cc1" "-triple" "x86_64-unknown-linux-gnu"{{.*}}"-emit-obj" // CHECK: clang-linker-wrapper{{.*}} "-o" "a.out" @@ -63,6 +63,7 @@ // RUN: %clang -### -target x86_64-pc-linux-gnu -fopenmp --offload-arch=gfx90a:sramecc-:xnack+ \ // RUN: -nogpulib %s 2>&1 | FileCheck %s --check-prefix=CHECK-TARGET-ID +// CHECK-TARGET-ID: "-cc1" "-triple" "amdgcn-amd-amdhsa" {{.*}} "-target-cpu" "gfx90a" "-target-feature" "-sramecc" "-target-feature" "+xnack" // CHECK-TARGET-ID: clang-offload-packager{{.*}}arch=gfx90a:sramecc-:xnack+,kind=openmp,feature=-sramecc,feature=+xnack // RUN: not %clang -### -target x86_64-pc-linux-gnu -fopenmp --offload-arch=gfx90a,gfx90a:xnack+ \ -- GitLab From bbddedb3bf7b17c5caa4732c4a94dde8824c5e3a Mon Sep 17 00:00:00 2001 From: kper Date: Fri, 7 Jun 2024 18:20:52 +0200 Subject: [PATCH 270/462] Fixed grammatical error in "enum specifier" error msg #94443 (#94592) As discussed in #94443, this PR changes the wording to be more correct. --- clang/include/clang/Basic/DiagnosticSemaKinds.td | 6 +++--- clang/test/CXX/basic/basic.lookup/basic.lookup.elab/p2.cpp | 4 ++-- .../CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.elab/p2-0x.cpp | 6 +++--- clang/test/CXX/drs/cwg2xx.cpp | 6 +++--- clang/test/CXX/drs/cwg4xx.cpp | 2 +- clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp | 2 +- clang/test/CXX/temp/temp.spec/no-body.cpp | 2 +- clang/test/SemaCXX/PR8755.cpp | 2 +- clang/test/SemaCXX/using-decl-templates.cpp | 2 +- clang/test/SemaTemplate/template-id-expr.cpp | 2 +- 10 files changed, 17 insertions(+), 17 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 9f0b6f5a3638..ae78534ddec2 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -6088,9 +6088,9 @@ def err_redefinition_different_concept : Error< "redefinition of concept %0 with different template parameters or requirements">; def err_tag_reference_non_tag : Error< "%select{non-struct type|non-class type|non-union type|non-enum " - "type|typedef|type alias|template|type alias template|template " - "template argument}1 %0 cannot be referenced with a " - "%select{struct|interface|union|class|enum}2 specifier">; + "type|typedef|type alias|template|alias template|template " + "template argument}1 %0 cannot be referenced with the '" + "%select{struct|interface|union|class|enum}2' specifier">; def err_tag_reference_conflict : Error< "implicit declaration introduced by elaborated type conflicts with a " "%select{non-struct type|non-class type|non-union type|non-enum " diff --git a/clang/test/CXX/basic/basic.lookup/basic.lookup.elab/p2.cpp b/clang/test/CXX/basic/basic.lookup/basic.lookup.elab/p2.cpp index a908518f02ea..c58cbfefc799 100644 --- a/clang/test/CXX/basic/basic.lookup/basic.lookup.elab/p2.cpp +++ b/clang/test/CXX/basic/basic.lookup/basic.lookup.elab/p2.cpp @@ -9,7 +9,7 @@ namespace test0 { typedef int A; // expected-note {{declared here}} int test() { - struct A a; // expected-error {{typedef 'A' cannot be referenced with a struct specifier}} + struct A a; // expected-error {{typedef 'A' cannot be referenced with the 'struct' specifier}} return a.foo; } } @@ -18,7 +18,7 @@ namespace test0 { template class A; // expected-note {{declared here}} int test() { - struct A a; // expected-error {{template 'A' cannot be referenced with a struct specifier}} + struct A a; // expected-error {{template 'A' cannot be referenced with the 'struct' specifier}} return a.foo; } } diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.elab/p2-0x.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.elab/p2-0x.cpp index f3e79c0aae44..98c9b915c6ce 100644 --- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.elab/p2-0x.cpp +++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.elab/p2-0x.cpp @@ -2,18 +2,18 @@ struct A { typedef int type; }; template using X = A; // expected-note {{declared here}} -struct X* p2; // expected-error {{type alias template 'X' cannot be referenced with a struct specifier}} +struct X* p2; // expected-error {{alias template 'X' cannot be referenced with the 'struct' specifier}} template using Id = T; // expected-note {{declared here}} template class F> struct Y { - struct F i; // expected-error {{type alias template 'Id' cannot be referenced with a struct specifier}} + struct F i; // expected-error {{alias template 'Id' cannot be referenced with the 'struct' specifier}} typename F::type j; // ok // FIXME: don't produce the diagnostic both for the definition and the instantiation. template using U = F; // expected-note 2{{declared here}} - struct Y::template U k; // expected-error 2{{type alias template 'U' cannot be referenced with a struct specifier}} + struct Y::template U k; // expected-error 2{{alias template 'U' cannot be referenced with the 'struct' specifier}} typename Y::template U l; // ok }; template struct Y; // expected-note {{requested here}} diff --git a/clang/test/CXX/drs/cwg2xx.cpp b/clang/test/CXX/drs/cwg2xx.cpp index 2b3131be3305..99916dea9a91 100644 --- a/clang/test/CXX/drs/cwg2xx.cpp +++ b/clang/test/CXX/drs/cwg2xx.cpp @@ -759,7 +759,7 @@ namespace cwg254 { // cwg254: 2.9 typedef typename T::type type; // ok even if this is a typedef-name, because // it's not an elaborated-type-specifier typedef struct T::type foo; - // expected-error@-1 {{typedef 'type' cannot be referenced with a struct specifier}} + // expected-error@-1 {{typedef 'type' cannot be referenced with the 'struct' specifier}} // expected-note@#cwg254-instantiation {{in instantiation of template class 'cwg254::A' requested here}} // expected-note@#cwg254-C {{declared here}} }; @@ -1264,10 +1264,10 @@ namespace cwg298 { // cwg298: 3.1 struct A a; struct B b; - // expected-error@-1 {{typedef 'B' cannot be referenced with a struct specifier}} + // expected-error@-1 {{typedef 'B' cannot be referenced with the 'struct' specifier}} // expected-note@#cwg298-B {{declared here}} struct C c; - // expected-error@-1 {{typedef 'C' cannot be referenced with a struct specifier}} + // expected-error@-1 {{typedef 'C' cannot be referenced with the 'struct' specifier}} // expected-note@#cwg298-C {{declared here}} B::B() {} diff --git a/clang/test/CXX/drs/cwg4xx.cpp b/clang/test/CXX/drs/cwg4xx.cpp index 07162cc28f6b..6ada25237287 100644 --- a/clang/test/CXX/drs/cwg4xx.cpp +++ b/clang/test/CXX/drs/cwg4xx.cpp @@ -170,7 +170,7 @@ namespace cwg407 { // cwg407: 3.8 { typedef struct S S; // #cwg407-typedef-S struct S *p; - // expected-error@-1 {{typedef 'S' cannot be referenced with a struct specifier}} + // expected-error@-1 {{typedef 'S' cannot be referenced with the 'struct' specifier}} // expected-note@#cwg407-typedef-S {{declared here}} } } diff --git a/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp b/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp index 1cf9e1c9f9c0..b8092afaffef 100644 --- a/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp +++ b/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp @@ -174,7 +174,7 @@ namespace test7 { // This shouldn't crash. template class D { - friend class A; // expected-error {{template 'A' cannot be referenced with a class specifier}} + friend class A; // expected-error {{template 'A' cannot be referenced with the 'class' specifier}} }; template class D; } diff --git a/clang/test/CXX/temp/temp.spec/no-body.cpp b/clang/test/CXX/temp/temp.spec/no-body.cpp index 6d1b82fe1898..5b0c4be30033 100644 --- a/clang/test/CXX/temp/temp.spec/no-body.cpp +++ b/clang/test/CXX/temp/temp.spec/no-body.cpp @@ -43,7 +43,7 @@ namespace good { // Only good in C++98/03 namespace unsupported { #ifndef FIXING - template struct y; // expected-error {{template 'y' cannot be referenced with a struct specifier}} + template struct y; // expected-error {{template 'y' cannot be referenced with the 'struct' specifier}} #endif } diff --git a/clang/test/SemaCXX/PR8755.cpp b/clang/test/SemaCXX/PR8755.cpp index 6818f3f0a822..c0bcab3537d6 100644 --- a/clang/test/SemaCXX/PR8755.cpp +++ b/clang/test/SemaCXX/PR8755.cpp @@ -7,7 +7,7 @@ struct A { template void f() { - class A ::iterator foo; // expected-error{{typedef 'iterator' cannot be referenced with a class specifier}} + class A ::iterator foo; // expected-error{{typedef 'iterator' cannot be referenced with the 'class' specifier}} } void g() { diff --git a/clang/test/SemaCXX/using-decl-templates.cpp b/clang/test/SemaCXX/using-decl-templates.cpp index 77dc596fdfc9..1cf4caee1c0d 100644 --- a/clang/test/SemaCXX/using-decl-templates.cpp +++ b/clang/test/SemaCXX/using-decl-templates.cpp @@ -90,7 +90,7 @@ namespace aliastemplateinst { template struct A { }; template using APtr = A; // expected-note{{previous use is here}} - template struct APtr; // expected-error{{type alias template 'APtr' cannot be referenced with a struct specifier}} + template struct APtr; // expected-error{{alias template 'APtr' cannot be referenced with the 'struct' specifier}} } namespace DontDiagnoseInvalidTest { diff --git a/clang/test/SemaTemplate/template-id-expr.cpp b/clang/test/SemaTemplate/template-id-expr.cpp index 6c98e29cdaa9..dc12823ae307 100644 --- a/clang/test/SemaTemplate/template-id-expr.cpp +++ b/clang/test/SemaTemplate/template-id-expr.cpp @@ -179,7 +179,7 @@ template