From 0fe271c35368a9190661bcca87101fc0916d6a8b Mon Sep 17 00:00:00 2001 From: Fanbo Meng Date: Tue, 12 Mar 2024 10:56:51 -0400 Subject: [PATCH 0001/1407] [SystemZ][z/OS] Add missing include header to AutoConvert.cpp to fix build (#84909) ba13fa2a5d57581bff1a7e9322234af30f4882f6 added usages of `errnoAsErrorCode()` to AutoConvert.cpp, need to include Error.h header to fix build failure. --- llvm/lib/Support/AutoConvert.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Support/AutoConvert.cpp b/llvm/lib/Support/AutoConvert.cpp index 74842e9167bd..c509284ee916 100644 --- a/llvm/lib/Support/AutoConvert.cpp +++ b/llvm/lib/Support/AutoConvert.cpp @@ -14,6 +14,7 @@ #ifdef __MVS__ #include "llvm/Support/AutoConvert.h" +#include "llvm/Support/Error.h" #include #include #include -- GitLab From f32b04d4ea91ad1018c25a1d4178cc4392d34968 Mon Sep 17 00:00:00 2001 From: NagyDonat Date: Tue, 12 Mar 2024 16:01:04 +0100 Subject: [PATCH 0002/1407] Revert "[analyzer] Accept C library functions from the `std` namespace" (#84926) Reverts llvm/llvm-project#84469 because it causes buildbot failures. I'll examine them and re-submit the change. --- .../Core/PathSensitive/CallDescription.h | 8 +- .../StaticAnalyzer/Core/CheckerContext.cpp | 8 +- clang/unittests/StaticAnalyzer/CMakeLists.txt | 1 - .../StaticAnalyzer/IsCLibraryFunctionTest.cpp | 89 ------------------- .../clang/unittests/StaticAnalyzer/BUILD.gn | 1 - 5 files changed, 9 insertions(+), 98 deletions(-) delete mode 100644 clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h index b4e1636130ca..3432d2648633 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h @@ -41,8 +41,12 @@ public: /// - We also accept calls where the number of arguments or parameters is /// greater than the specified value. /// For the exact heuristics, see CheckerContext::isCLibraryFunction(). - /// (This mode only matches functions that are declared either directly - /// within a TU or in the namespace `std`.) + /// Note that functions whose declaration context is not a TU (e.g. + /// methods, functions in namespaces) are not accepted as C library + /// functions. + /// FIXME: If I understand it correctly, this discards calls where C++ code + /// refers a C library function through the namespace `std::` via headers + /// like . CLibrary, /// Matches "simple" functions that are not methods. (Static methods are diff --git a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp index 1a9bff529e9b..d6d4cec9dd3d 100644 --- a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp +++ b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp @@ -87,11 +87,9 @@ bool CheckerContext::isCLibraryFunction(const FunctionDecl *FD, if (!II) return false; - // C library functions are either declared directly within a TU (the common - // case) or they are accessed through the namespace `std` (when they are used - // in C++ via headers like ). - const DeclContext *DC = FD->getDeclContext()->getRedeclContext(); - if (!(DC->isTranslationUnit() || DC->isStdNamespace())) + // Look through 'extern "C"' and anything similar invented in the future. + // If this function is not in TU directly, it is not a C library function. + if (!FD->getDeclContext()->getRedeclContext()->isTranslationUnit()) return false; // If this function is not externally visible, it is not a C library function. diff --git a/clang/unittests/StaticAnalyzer/CMakeLists.txt b/clang/unittests/StaticAnalyzer/CMakeLists.txt index db56e77331b8..775f0f8486b8 100644 --- a/clang/unittests/StaticAnalyzer/CMakeLists.txt +++ b/clang/unittests/StaticAnalyzer/CMakeLists.txt @@ -11,7 +11,6 @@ add_clang_unittest(StaticAnalysisTests CallEventTest.cpp ConflictingEvalCallsTest.cpp FalsePositiveRefutationBRVisitorTest.cpp - IsCLibraryFunctionTest.cpp NoStateChangeFuncVisitorTest.cpp ParamRegionTest.cpp RangeSetTest.cpp diff --git a/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp b/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp deleted file mode 100644 index 19c66cc6bee1..000000000000 --- a/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp +++ /dev/null @@ -1,89 +0,0 @@ -#include "clang/ASTMatchers/ASTMatchFinder.h" -#include "clang/ASTMatchers/ASTMatchers.h" -#include "clang/Analysis/AnalysisDeclContext.h" -#include "clang/Frontend/ASTUnit.h" -#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" -#include "clang/Tooling/Tooling.h" -#include "gtest/gtest.h" - -#include - -using namespace clang; -using namespace ento; -using namespace ast_matchers; - -testing::AssertionResult extractFunctionDecl(StringRef Code, - const FunctionDecl *&Result) { - auto ASTUnit = tooling::buildASTFromCode(Code); - if (!ASTUnit) - return testing::AssertionFailure() << "AST construction failed"; - - ASTContext &Context = ASTUnit->getASTContext(); - if (Context.getDiagnostics().hasErrorOccurred()) - return testing::AssertionFailure() << "Compilation error"; - - auto Matches = ast_matchers::match(functionDecl().bind("fn"), Context); - if (Matches.empty()) - return testing::AssertionFailure() << "No function declaration found"; - - if (Matches.size() > 1) - return testing::AssertionFailure() - << "Multiple function declarations found"; - - Result = Matches[0].getNodeAs("fn"); - return testing::AssertionSuccess(); -} - -TEST(IsCLibraryFunctionTest, AcceptsGlobal) { - const FunctionDecl *Result; - ASSERT_TRUE(extractFunctionDecl(R"cpp(void fun();)cpp", Result)); - EXPECT_TRUE(CheckerContext::isCLibraryFunction(Result)); -} - -TEST(IsCLibraryFunctionTest, AcceptsExternCGlobal) { - const FunctionDecl *Result; - ASSERT_TRUE( - extractFunctionDecl(R"cpp(extern "C" { void fun(); })cpp", Result)); - EXPECT_TRUE(CheckerContext::isCLibraryFunction(Result)); -} - -TEST(IsCLibraryFunctionTest, RejectsNoInlineNoExternalLinkage) { - // Functions that are neither inlined nor externally visible cannot be C library functions. - const FunctionDecl *Result; - ASSERT_TRUE(extractFunctionDecl(R"cpp(static void fun();)cpp", Result)); - EXPECT_FALSE(CheckerContext::isCLibraryFunction(Result)); -} - -TEST(IsCLibraryFunctionTest, RejectsAnonymousNamespace) { - const FunctionDecl *Result; - ASSERT_TRUE( - extractFunctionDecl(R"cpp(namespace { void fun(); })cpp", Result)); - EXPECT_FALSE(CheckerContext::isCLibraryFunction(Result)); -} - -TEST(IsCLibraryFunctionTest, AcceptsStdNamespace) { - const FunctionDecl *Result; - ASSERT_TRUE( - extractFunctionDecl(R"cpp(namespace std { void fun(); })cpp", Result)); - EXPECT_TRUE(CheckerContext::isCLibraryFunction(Result)); -} - -TEST(IsCLibraryFunctionTest, RejectsOtherNamespaces) { - const FunctionDecl *Result; - ASSERT_TRUE( - extractFunctionDecl(R"cpp(namespace stdx { void fun(); })cpp", Result)); - EXPECT_FALSE(CheckerContext::isCLibraryFunction(Result)); -} - -TEST(IsCLibraryFunctionTest, RejectsClassStatic) { - const FunctionDecl *Result; - ASSERT_TRUE( - extractFunctionDecl(R"cpp(class A { static void fun(); };)cpp", Result)); - EXPECT_FALSE(CheckerContext::isCLibraryFunction(Result)); -} - -TEST(IsCLibraryFunctionTest, RejectsClassMember) { - const FunctionDecl *Result; - ASSERT_TRUE(extractFunctionDecl(R"cpp(class A { void fun(); };)cpp", Result)); - EXPECT_FALSE(CheckerContext::isCLibraryFunction(Result)); -} diff --git a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn index 9c240cff1816..01c2b6ced336 100644 --- a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn @@ -19,7 +19,6 @@ unittest("StaticAnalysisTests") { "CallEventTest.cpp", "ConflictingEvalCallsTest.cpp", "FalsePositiveRefutationBRVisitorTest.cpp", - "IsCLibraryFunctionTest.cpp", "NoStateChangeFuncVisitorTest.cpp", "ParamRegionTest.cpp", "RangeSetTest.cpp", -- GitLab From 083da46ff07170471f8bb9ed2947f6ebc725670b Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Tue, 12 Mar 2024 08:02:54 -0700 Subject: [PATCH 0003/1407] [clang][deps] Fix dependency scanning with -working-directory (#84525) Stop overriding -working-directory to CWD during argument parsing, which should no longer necessary after we set the VFS working directory, and set FSOpts correctly after parsing arguments so that working-directory behaves correctly. --- .../DependencyScanningWorker.cpp | 14 ++++----- .../ClangScanDeps/working-directory-option.c | 30 +++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 clang/test/ClangScanDeps/working-directory-option.c diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 2b882f8a5e07..76f3d950a13b 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -296,7 +296,7 @@ public: DisableFree(DisableFree), ModuleName(ModuleName) {} bool runInvocation(std::shared_ptr Invocation, - FileManager *FileMgr, + FileManager *DriverFileMgr, std::shared_ptr PCHContainerOps, DiagnosticConsumer *DiagConsumer) override { // Make a deep copy of the original Clang invocation. @@ -342,12 +342,13 @@ public: ScanInstance.getHeaderSearchOpts().ModulesIncludeVFSUsage = any(OptimizeArgs & ScanningOptimizations::VFS); - ScanInstance.setFileManager(FileMgr); // Support for virtual file system overlays. - FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation( + auto FS = createVFSFromCompilerInvocation( ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), - FileMgr->getVirtualFileSystemPtr())); + DriverFileMgr->getVirtualFileSystemPtr()); + // Create a new FileManager to match the invocation's FileSystemOptions. + auto *FileMgr = ScanInstance.createFileManager(FS); ScanInstance.createSourceManager(*FileMgr); // Store the list of prebuilt module files into header search options. This @@ -624,9 +625,8 @@ bool DependencyScanningWorker::computeDependencies( ModifiedCommandLine ? *ModifiedCommandLine : CommandLine; auto &FinalFS = ModifiedFS ? ModifiedFS : BaseFS; - FileSystemOptions FSOpts; - FSOpts.WorkingDir = WorkingDirectory.str(); - auto FileMgr = llvm::makeIntrusiveRefCnt(FSOpts, FinalFS); + auto FileMgr = + llvm::makeIntrusiveRefCnt(FileSystemOptions{}, FinalFS); std::vector FinalCCommandLine(FinalCommandLine.size(), nullptr); llvm::transform(FinalCommandLine, FinalCCommandLine.begin(), diff --git a/clang/test/ClangScanDeps/working-directory-option.c b/clang/test/ClangScanDeps/working-directory-option.c new file mode 100644 index 000000000000..d57497d405d3 --- /dev/null +++ b/clang/test/ClangScanDeps/working-directory-option.c @@ -0,0 +1,30 @@ +// Test that -working-directory works even when it differs from the working +// directory of the filesystem. + +// RUN: rm -rf %t +// RUN: mkdir -p %t/other +// RUN: split-file %s %t +// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \ +// RUN: > %t/deps.json + +// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t + +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/cwd/t.c" +// CHECK-NEXT: "[[PREFIX]]/cwd/relative/h1.h" +// CHECK-NEXT: ] +// CHECK-NEXT: "input-file": "[[PREFIX]]/cwd/t.c" + +//--- cdb.json.template +[{ + "directory": "DIR/other", + "command": "clang -c t.c -I relative -working-directory DIR/cwd", + "file": "DIR/cwd/t.c" +}] + +//--- cwd/relative/h1.h + +//--- cwd/t.c +#include "h1.h" -- GitLab From 3238b92142c8fb70a1b2c72ee06bb47d57229dc9 Mon Sep 17 00:00:00 2001 From: Bjorn Pettersson Date: Tue, 12 Mar 2024 15:19:30 +0100 Subject: [PATCH 0004/1407] [LoopSimplifyCFG] Drop no longer needed DependenceAnalysis.h include --- llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp index 028a487ecdbc..ae9103d0608a 100644 --- a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp +++ b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp @@ -16,7 +16,6 @@ #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" -#include "llvm/Analysis/DependenceAnalysis.h" #include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopIterator.h" -- GitLab From 4d0f79e346ceb0ddb25a94053c612a5b34a72100 Mon Sep 17 00:00:00 2001 From: Bjorn Pettersson Date: Tue, 12 Mar 2024 16:02:02 +0100 Subject: [PATCH 0005/1407] Pre commit test cases SRL/SRA support in canCreateUndefOrPoison. NFC Add test cases to show that we can't push freeze through SRA/SRL with 'exact' flag when there are multiple uses. --- llvm/test/CodeGen/X86/freeze-binary.ll | 48 ++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/llvm/test/CodeGen/X86/freeze-binary.ll b/llvm/test/CodeGen/X86/freeze-binary.ll index defd81e6ab77..b212e9438e1b 100644 --- a/llvm/test/CodeGen/X86/freeze-binary.ll +++ b/llvm/test/CodeGen/X86/freeze-binary.ll @@ -488,6 +488,30 @@ define i32 @freeze_ashr_exact(i32 %a0) nounwind { ret i32 %z } +define i32 @freeze_ashr_exact_extra_use(i32 %a0, ptr %escape) nounwind { +; X86-LABEL: freeze_ashr_exact_extra_use: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: sarl $3, %eax +; X86-NEXT: movl %eax, (%ecx) +; X86-NEXT: sarl $6, %eax +; X86-NEXT: retl +; +; X64-LABEL: freeze_ashr_exact_extra_use: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: sarl $3, %eax +; X64-NEXT: movl %eax, (%rsi) +; X64-NEXT: sarl $6, %eax +; X64-NEXT: retq + %x = ashr exact i32 %a0, 3 + %y = freeze i32 %x + %z = ashr i32 %y, 6 + store i32 %x, ptr %escape + ret i32 %z +} + define i32 @freeze_ashr_outofrange(i32 %a0) nounwind { ; X86-LABEL: freeze_ashr_outofrange: ; X86: # %bb.0: @@ -597,6 +621,30 @@ define i32 @freeze_lshr_exact(i32 %a0) nounwind { ret i32 %z } +define i32 @freeze_lshr_exact_extra_use(i32 %a0, ptr %escape) nounwind { +; X86-LABEL: freeze_lshr_exact_extra_use: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shrl $3, %eax +; X86-NEXT: movl %eax, (%ecx) +; X86-NEXT: shrl $5, %eax +; X86-NEXT: retl +; +; X64-LABEL: freeze_lshr_exact_extra_use: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: shrl $3, %eax +; X64-NEXT: movl %eax, (%rsi) +; X64-NEXT: shrl $5, %eax +; X64-NEXT: retq + %x = lshr exact i32 %a0, 3 + %y = freeze i32 %x + %z = lshr i32 %y, 5 + store i32 %x, ptr %escape + ret i32 %z +} + define i32 @freeze_lshr_outofrange(i32 %a0) nounwind { ; X86-LABEL: freeze_lshr_outofrange: ; X86: # %bb.0: -- GitLab From beba307c5bc206168bdea3b893e02ea31579fe62 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Tue, 12 Mar 2024 16:23:25 +0100 Subject: [PATCH 0006/1407] [LSR] Clear SCEVExpander before deleting phi nodes Fixes https://github.com/llvm/llvm-project/issues/84709. --- .../Transforms/Scalar/LoopStrengthReduce.cpp | 2 ++ .../Transforms/LoopStrengthReduce/pr84709.ll | 34 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 llvm/test/Transforms/LoopStrengthReduce/pr84709.ll diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp index 8b078ddc4e74..c4e1a0db8b32 100644 --- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -6971,6 +6971,7 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE, Rewriter.setDebugType(DEBUG_TYPE); #endif unsigned numFolded = Rewriter.replaceCongruentIVs(L, &DT, DeadInsts, &TTI); + Rewriter.clear(); if (numFolded) { Changed = true; RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts, &TLI, @@ -6989,6 +6990,7 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE, SCEVExpander Rewriter(SE, DL, "lsr", true); int Rewrites = rewriteLoopExitValues(L, &LI, &TLI, &SE, &TTI, Rewriter, &DT, UnusedIndVarInLoop, DeadInsts); + Rewriter.clear(); if (Rewrites) { Changed = true; RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts, &TLI, diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr84709.ll b/llvm/test/Transforms/LoopStrengthReduce/pr84709.ll new file mode 100644 index 000000000000..99794d01242c --- /dev/null +++ b/llvm/test/Transforms/LoopStrengthReduce/pr84709.ll @@ -0,0 +1,34 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes=loop-reduce < %s | FileCheck %s + +; Make sure it does not assert. +define i64 @test() { +; CHECK-LABEL: define i64 @test() { +; CHECK-NEXT: bb: +; CHECK-NEXT: br label [[BB1:%.*]] +; CHECK: bb1: +; CHECK-NEXT: br label [[BB2:%.*]] +; CHECK: bb2: +; CHECK-NEXT: br i1 true, label [[BB5:%.*]], label [[BB2]] +; CHECK: bb5: +; CHECK-NEXT: br label [[BB1]] +; +bb: + br label %bb1 + +bb1: + %phi = phi i8 [ %zext6, %bb5 ], [ 0, %bb ] + br label %bb2 + +bb2: + %phi3 = phi i8 [ %add, %bb2 ], [ %phi, %bb1 ] + %phi4 = phi i32 [ 0, %bb2 ], [ 1, %bb1 ] + %add = add i8 %phi3, 1 + br i1 true, label %bb5, label %bb2 + +bb5: + %zext = zext i8 %add to i32 + %icmp = icmp sge i32 %phi4, 0 + %zext6 = zext i1 %icmp to i8 + br label %bb1 +} -- GitLab From 08dd645c15a091a53313e278d8f3c090e7c385d1 Mon Sep 17 00:00:00 2001 From: Nemanja Ivanovic Date: Tue, 12 Mar 2024 16:26:49 +0100 Subject: [PATCH 0007/1407] [RISC-V] Bad immediate value for Zcmp instructions with E extension (#84925) When we are using the Zcmp extension together with the E extension in 32-bit mode and we need to spill both callee-saved registers as well as needing a couple of 32-bit stack slots, we emit a meaningless stack adjustment with cm.push/cm.popret. Furthermore this leads to the stack slot for the ra being clobbered so control returns to a random location. This is just a pre-commit test so that the PR for the fix shows the difference in code generation. --- .../CodeGen/RISCV/zcmp-additional-stack.ll | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll diff --git a/llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll b/llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll new file mode 100644 index 000000000000..e5c2e0180ee0 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll @@ -0,0 +1,49 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=riscv32 -mattr=+zcmp,+e -target-abi ilp32e -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV32 +define ptr @func(ptr %s, i32 %_c, ptr %incdec.ptr, i1 %0, i8 %conv14) #0 { +; RV32-LABEL: func: +; RV32: # %bb.0: # %entry +; RV32-NEXT: cm.push {ra, s0-s1}, -24 +; RV32-NEXT: .cfi_def_cfa_offset 24 +; RV32-NEXT: .cfi_offset ra, -12 +; RV32-NEXT: .cfi_offset s0, -8 +; RV32-NEXT: .cfi_offset s1, -4 +; RV32-NEXT: sw a4, 4(sp) # 4-byte Folded Spill +; RV32-NEXT: sw a2, 0(sp) # 4-byte Folded Spill +; RV32-NEXT: mv a2, a1 +; RV32-NEXT: mv s1, a0 +; RV32-NEXT: li a0, 1 +; RV32-NEXT: andi a3, a3, 1 +; RV32-NEXT: .LBB0_1: # %while.body +; RV32-NEXT: # =>This Inner Loop Header: Depth=1 +; RV32-NEXT: mv s0, a0 +; RV32-NEXT: li a0, 0 +; RV32-NEXT: bnez a3, .LBB0_1 +; RV32-NEXT: # %bb.2: # %while.end +; RV32-NEXT: lui a0, 4112 +; RV32-NEXT: addi a1, a0, 257 +; RV32-NEXT: mv a0, a2 +; RV32-NEXT: call __mulsi3 +; RV32-NEXT: sw a0, 0(zero) +; RV32-NEXT: andi s0, s0, 1 +; RV32-NEXT: lw a0, 0(sp) # 4-byte Folded Reload +; RV32-NEXT: add s0, s0, a0 +; RV32-NEXT: lw a0, 4(sp) # 4-byte Folded Reload +; RV32-NEXT: sb a0, 0(s0) +; RV32-NEXT: mv a0, s1 +; RV32-NEXT: cm.popret {ra, s0-s1}, 24 +entry: + br label %while.body + +while.body: ; preds = %while.body, %entry + %n.addr.042 = phi i32 [ 1, %entry ], [ 0, %while.body ] + br i1 %0, label %while.body, label %while.end + +while.end: ; preds = %while.body + %or5 = mul i32 %_c, 16843009 + store i32 %or5, ptr null, align 4 + %1 = and i32 %n.addr.042, 1 + %scevgep = getelementptr i8, ptr %incdec.ptr, i32 %1 + store i8 %conv14, ptr %scevgep, align 1 + ret ptr %s +} -- GitLab From bae47d48b632a4fa1dce5591bc0783360cf69e28 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Tue, 12 Mar 2024 08:38:34 -0700 Subject: [PATCH 0008/1407] [libc] fix another build failure from using limits.h (#84827) My GCC build is failing with issues similar why we added our own. Looks like we missed one spot. See also: commit 72ce62941579 ("[libc] Add C23 limits.h header. (#78887)") --- libc/test/src/__support/CMakeLists.txt | 1 + libc/test/src/__support/integer_to_string_test.cpp | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt index 91dd0dc4decf..51b897f8b595 100644 --- a/libc/test/src/__support/CMakeLists.txt +++ b/libc/test/src/__support/CMakeLists.txt @@ -78,6 +78,7 @@ add_libc_test( SRCS integer_to_string_test.cpp DEPENDS + libc.src.__support.CPP.limits libc.src.__support.CPP.string_view libc.src.__support.integer_literals libc.src.__support.integer_to_string diff --git a/libc/test/src/__support/integer_to_string_test.cpp b/libc/test/src/__support/integer_to_string_test.cpp index a2a80c81b9f6..270fddd828b6 100644 --- a/libc/test/src/__support/integer_to_string_test.cpp +++ b/libc/test/src/__support/integer_to_string_test.cpp @@ -6,6 +6,7 @@ // //===----------------------------------------------------------------------===// +#include "src/__support/CPP/limits.h" #include "src/__support/CPP/span.h" #include "src/__support/CPP/string_view.h" #include "src/__support/UInt.h" @@ -15,8 +16,6 @@ #include "test/UnitTest/Test.h" -#include "limits.h" - using LIBC_NAMESPACE::IntegerToString; using LIBC_NAMESPACE::cpp::span; using LIBC_NAMESPACE::cpp::string_view; -- GitLab From f0c0ddae45ec929d023232d2ff0b75b7f09853c2 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Tue, 12 Mar 2024 08:39:17 -0700 Subject: [PATCH 0009/1407] [libc] implement the final macros for stdbit.h support (#84798) Relevant sections of n3096: - 7.18.1p1 - 7.18.2 --- libc/docs/c23.rst | 2 +- libc/docs/stdbit.rst | 8 ++++---- libc/include/llvm-libc-macros/stdbit-macros.h | 5 +++++ libc/spec/stdc.td | 4 ++++ libc/test/include/stdbit_test.cpp | 17 +++++++++++++++++ 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/libc/docs/c23.rst b/libc/docs/c23.rst index 24cef8539393..3f64722bc8e6 100644 --- a/libc/docs/c23.rst +++ b/libc/docs/c23.rst @@ -75,7 +75,7 @@ Additions: * dfmal * fsqrt* * dsqrtl -* stdbit.h (New header) +* stdbit.h (New header) |check| * stdckdint.h (New header) |check| * stddef.h diff --git a/libc/docs/stdbit.rst b/libc/docs/stdbit.rst index 9b4974cf1479..d42f79382462 100644 --- a/libc/docs/stdbit.rst +++ b/libc/docs/stdbit.rst @@ -110,10 +110,10 @@ Macros ========================= ========= Macro Name Available ========================= ========= -__STDC_VERSION_STDBIT_H__ -__STDC_ENDIAN_LITTLE__ -__STDC_ENDIAN_BIG__ -__STDC_ENDIAN_NATIVE__ +__STDC_VERSION_STDBIT_H__ |check| +__STDC_ENDIAN_LITTLE__ |check| +__STDC_ENDIAN_BIG__ |check| +__STDC_ENDIAN_NATIVE__ |check| stdc_leading_zeros |check| stdc_leading_ones |check| stdc_trailing_zeros |check| diff --git a/libc/include/llvm-libc-macros/stdbit-macros.h b/libc/include/llvm-libc-macros/stdbit-macros.h index 10c0fac3c8dd..c5b2f0977834 100644 --- a/libc/include/llvm-libc-macros/stdbit-macros.h +++ b/libc/include/llvm-libc-macros/stdbit-macros.h @@ -9,6 +9,11 @@ #ifndef __LLVM_LIBC_MACROS_STDBIT_MACROS_H #define __LLVM_LIBC_MACROS_STDBIT_MACROS_H +#define __STDC_VERSION_STDBIT_H__ 202311L +#define __STDC_ENDIAN_LITTLE__ __ORDER_LITTLE_ENDIAN__ +#define __STDC_ENDIAN_BIG__ __ORDER_BIG_ENDIAN__ +#define __STDC_ENDIAN_NATIVE__ __BYTE_ORDER__ + // TODO(https://github.com/llvm/llvm-project/issues/80509): support _BitInt(). #ifdef __cplusplus inline unsigned stdc_leading_zeros(unsigned char x) { diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 1f14fe758130..1f9917b1f073 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -805,6 +805,10 @@ def StdC : StandardSpec<"stdc"> { HeaderSpec StdBit = HeaderSpec< "stdbit.h", [ + Macro<"__STDC_VERSION_STDBIT_H__">, + Macro<"__STDC_ENDIAN_LITTLE__">, + Macro<"__STDC_ENDIAN_BIG__">, + Macro<"__STDC_ENDIAN_NATIVE__">, Macro<"stdc_leading_zeros">, Macro<"stdc_leading_ones">, Macro<"stdc_trailing_zeros">, diff --git a/libc/test/include/stdbit_test.cpp b/libc/test/include/stdbit_test.cpp index f3227eb86959..bee1a19f9c03 100644 --- a/libc/test/include/stdbit_test.cpp +++ b/libc/test/include/stdbit_test.cpp @@ -141,3 +141,20 @@ TEST(LlvmLibcStdbitTest, TypeGenericMacroBitCeil) { EXPECT_EQ(stdc_bit_ceil(0UL), 0x6DUL); EXPECT_EQ(stdc_bit_ceil(0ULL), 0x6EULL); } + +TEST(LlvmLibcStdbitTest, VersionMacro) { + // 7.18.1p2 an integer constant expression with a value equivalent to 202311L. + EXPECT_EQ(__STDC_VERSION_STDBIT_H__, 202311L); +} + +TEST(LlvmLibcStdbitTest, EndianMacros) { + // 7.18.2p3 The values of the integer constant expressions for + // __STDC_ENDIAN_LITTLE__ and __STDC_ENDIAN_BIG__ are not equal. + EXPECT_NE(__STDC_ENDIAN_LITTLE__, __STDC_ENDIAN_BIG__); + // The standard does allow for __STDC_ENDIAN_NATIVE__ to be an integer + // constant expression with an implementation defined value for non-big or + // little endianness environments. I assert such machines are no longer + // relevant. + EXPECT_TRUE(__STDC_ENDIAN_NATIVE__ == __STDC_ENDIAN_LITTLE__ || + __STDC_ENDIAN_NATIVE__ == __STDC_ENDIAN_BIG__); +} -- GitLab From 9f69d3cf88905df5006f93dce536b7e73c0b1735 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Tue, 12 Mar 2024 10:39:40 -0500 Subject: [PATCH 0010/1407] [Libomptarget] Use NVPTX lane id intrinsic in DeviceRTL (#84928) Summary: We are currently taking the lower 5 bites of the thread ID as the warp ID. This doesn't work in non-1D grids and is also slower than just using the dedicated hardware register. --- openmp/libomptarget/DeviceRTL/src/Mapping.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/openmp/libomptarget/DeviceRTL/src/Mapping.cpp b/openmp/libomptarget/DeviceRTL/src/Mapping.cpp index 31dd8054dec3..b2028a8fb4f5 100644 --- a/openmp/libomptarget/DeviceRTL/src/Mapping.cpp +++ b/openmp/libomptarget/DeviceRTL/src/Mapping.cpp @@ -172,10 +172,7 @@ uint32_t getThreadIdInBlock(int32_t Dim) { UNREACHABLE("Dim outside range!"); } -uint32_t getThreadIdInWarp() { - return impl::getThreadIdInBlock(mapping::DIM_X) & - (mapping::getWarpSize() - 1); -} +uint32_t getThreadIdInWarp() { return __nvvm_read_ptx_sreg_laneid(); } uint32_t getBlockIdInKernel(int32_t Dim) { switch (Dim) { -- GitLab From 392436383a52bc5e188bd28bec5bc71b3cb5384a Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Tue, 12 Mar 2024 08:39:50 -0700 Subject: [PATCH 0011/1407] [libc] fix typo in stdbit.h macro spec files (#84780) --- libc/spec/stdc.td | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 1f9917b1f073..e012d0dee089 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -820,9 +820,9 @@ def StdC : StandardSpec<"stdc"> { Macro<"stdc_count_zeros">, Macro<"stdc_count_ones">, Macro<"stdc_has_single_bit">, - Macro<"std_bit_width">, - Macro<"std_bit_floor">, - Macro<"std_bit_ceil"> + Macro<"stdc_bit_width">, + Macro<"stdc_bit_floor">, + Macro<"stdc_bit_ceil"> ], // Macros [], // Types [], // Enumerations -- GitLab From c167a2588737613558bd7be4c9280603e89281ac Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Tue, 12 Mar 2024 10:40:35 -0500 Subject: [PATCH 0012/1407] [libc] Fix lane-id utility function not using built-in (#84902) Summary: Previously we got the lane-id from taking the global thread ID and taking off the bottom 5 bits. This works but is inefficient compared to the NVPTX intrinsic simply dedicated to get this value. --- libc/src/__support/GPU/nvptx/utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/src/__support/GPU/nvptx/utils.h b/libc/src/__support/GPU/nvptx/utils.h index a92c8847b6ec..fe9da4e8e6cb 100644 --- a/libc/src/__support/GPU/nvptx/utils.h +++ b/libc/src/__support/GPU/nvptx/utils.h @@ -97,7 +97,7 @@ LIBC_INLINE uint32_t get_lane_size() { return 32; } /// Returns the id of the thread inside of a CUDA warp executing together. [[clang::convergent]] LIBC_INLINE uint32_t get_lane_id() { - return get_thread_id() & (get_lane_size() - 1); + return __nvvm_read_ptx_sreg_laneid(); } /// Returns the bit-mask of active threads in the current warp. -- GitLab From 261e5648e70b363aecf86acfcd7fb416eb48fb7b Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Tue, 12 Mar 2024 10:40:49 -0500 Subject: [PATCH 0013/1407] [libc] Add utility functions for warp-level scan and reduction (#84866) Summary: The GPU uses a SIMT execution model. That means that each value actually belongs to a group of 32 or 64 other lanes executing next to it. These platforms offer some intrinsic fuctions to actually take elements from neighboring lanes. With these we can do parallel scans or reductions. These functions do not have an immediate user, but will be used in the allocator interface that is in-progress and are generally good to have. This patch is a precommit for these new utilitly functions. --- libc/src/__support/GPU/amdgpu/utils.h | 6 ++ libc/src/__support/GPU/generic/utils.h | 2 + libc/src/__support/GPU/nvptx/utils.h | 8 +++ libc/src/__support/GPU/utils.h | 19 ++++++ .../integration/src/__support/CMakeLists.txt | 3 + .../src/__support/GPU/CMakeLists.txt | 11 ++++ .../src/__support/GPU/scan_reduce.cpp | 62 +++++++++++++++++++ 7 files changed, 111 insertions(+) create mode 100644 libc/test/integration/src/__support/GPU/CMakeLists.txt create mode 100644 libc/test/integration/src/__support/GPU/scan_reduce.cpp diff --git a/libc/src/__support/GPU/amdgpu/utils.h b/libc/src/__support/GPU/amdgpu/utils.h index 75f0b5744ebd..9b520a6bcf38 100644 --- a/libc/src/__support/GPU/amdgpu/utils.h +++ b/libc/src/__support/GPU/amdgpu/utils.h @@ -145,6 +145,12 @@ LIBC_INLINE uint32_t get_lane_size() { __builtin_amdgcn_wave_barrier(); } +/// Shuffles the the lanes inside the wavefront according to the given index. +[[clang::convergent]] LIBC_INLINE uint32_t shuffle(uint64_t, uint32_t idx, + uint32_t x) { + return __builtin_amdgcn_ds_bpermute(idx << 2, x); +} + /// Returns the current value of the GPU's processor clock. /// NOTE: The RDNA3 and RDNA2 architectures use a 20-bit cycle counter. LIBC_INLINE uint64_t processor_clock() { return __builtin_readcyclecounter(); } diff --git a/libc/src/__support/GPU/generic/utils.h b/libc/src/__support/GPU/generic/utils.h index c6c3c01cf7d5..b6df59f7aa9e 100644 --- a/libc/src/__support/GPU/generic/utils.h +++ b/libc/src/__support/GPU/generic/utils.h @@ -67,6 +67,8 @@ LIBC_INLINE void sync_threads() {} LIBC_INLINE void sync_lane(uint64_t) {} +LIBC_INLINE uint32_t shuffle(uint64_t, uint32_t, uint32_t x) { return x; } + LIBC_INLINE uint64_t processor_clock() { return 0; } LIBC_INLINE uint64_t fixed_frequency_clock() { return 0; } diff --git a/libc/src/__support/GPU/nvptx/utils.h b/libc/src/__support/GPU/nvptx/utils.h index fe9da4e8e6cb..3f19afb83648 100644 --- a/libc/src/__support/GPU/nvptx/utils.h +++ b/libc/src/__support/GPU/nvptx/utils.h @@ -126,6 +126,14 @@ LIBC_INLINE uint32_t get_lane_size() { return 32; } __nvvm_bar_warp_sync(static_cast(mask)); } +/// Shuffles the the lanes inside the warp according to the given index. +[[clang::convergent]] LIBC_INLINE uint32_t shuffle(uint64_t lane_mask, + uint32_t idx, uint32_t x) { + uint32_t mask = static_cast(lane_mask); + uint32_t bitmask = (mask >> idx) & 1; + return -bitmask & __nvvm_shfl_sync_idx_i32(mask, x, idx, get_lane_size() - 1); +} + /// Returns the current value of the GPU's processor clock. LIBC_INLINE uint64_t processor_clock() { return __builtin_readcyclecounter(); } diff --git a/libc/src/__support/GPU/utils.h b/libc/src/__support/GPU/utils.h index 0f9167cdee06..93022e8de811 100644 --- a/libc/src/__support/GPU/utils.h +++ b/libc/src/__support/GPU/utils.h @@ -31,6 +31,25 @@ LIBC_INLINE bool is_first_lane(uint64_t lane_mask) { return gpu::get_lane_id() == get_first_lane_id(lane_mask); } +/// Gets the sum of all lanes inside the warp or wavefront. +LIBC_INLINE uint32_t reduce(uint64_t lane_mask, uint32_t x) { + for (uint32_t step = gpu::get_lane_size() / 2; step > 0; step /= 2) { + uint32_t index = step + gpu::get_lane_id(); + x += gpu::shuffle(lane_mask, index, x); + } + return gpu::broadcast_value(lane_mask, x); +} + +/// Gets the accumulator scan of the threads in the warp or wavefront. +LIBC_INLINE uint32_t scan(uint64_t lane_mask, uint32_t x) { + for (uint32_t step = 1; step < gpu::get_lane_size(); step *= 2) { + uint32_t index = gpu::get_lane_id() - step; + uint32_t bitmask = gpu::get_lane_id() >= step; + x += -bitmask & gpu::shuffle(lane_mask, index, x); + } + return x; +} + } // namespace gpu } // namespace LIBC_NAMESPACE diff --git a/libc/test/integration/src/__support/CMakeLists.txt b/libc/test/integration/src/__support/CMakeLists.txt index 7c853ff10259..b5b6557e8d68 100644 --- a/libc/test/integration/src/__support/CMakeLists.txt +++ b/libc/test/integration/src/__support/CMakeLists.txt @@ -1 +1,4 @@ add_subdirectory(threads) +if(LIBC_TARGET_OS_IS_GPU) + add_subdirectory(GPU) +endif() diff --git a/libc/test/integration/src/__support/GPU/CMakeLists.txt b/libc/test/integration/src/__support/GPU/CMakeLists.txt new file mode 100644 index 000000000000..7811e0da45dd --- /dev/null +++ b/libc/test/integration/src/__support/GPU/CMakeLists.txt @@ -0,0 +1,11 @@ +add_custom_target(libc-support-gpu-tests) +add_dependencies(libc-integration-tests libc-support-gpu-tests) + +add_integration_test( + scan_reduce_test + SUITE libc-support-gpu-tests + SRCS + scan_reduce.cpp + LOADER_ARGS + --threads 64 +) diff --git a/libc/test/integration/src/__support/GPU/scan_reduce.cpp b/libc/test/integration/src/__support/GPU/scan_reduce.cpp new file mode 100644 index 000000000000..bc621c3300cb --- /dev/null +++ b/libc/test/integration/src/__support/GPU/scan_reduce.cpp @@ -0,0 +1,62 @@ +//===-- Test for the parallel scan and reduction operations on the GPU ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/bit.h" +#include "src/__support/GPU/utils.h" +#include "test/IntegrationTest/test.h" + +using namespace LIBC_NAMESPACE; + +static uint32_t sum(uint32_t n) { return n * (n + 1) / 2; } + +// Tests a reduction within a convergant warp or wavefront using some known +// values. For example, if every element in the lane is one, then the sum should +// be the size of the warp or wavefront, i.e. 1 + 1 + 1 ... + 1. +static void test_reduce() { + uint64_t mask = gpu::get_lane_mask(); + uint32_t x = gpu::reduce(mask, 1); + EXPECT_EQ(x, gpu::get_lane_size()); + + uint32_t y = gpu::reduce(mask, gpu::get_lane_id()); + EXPECT_EQ(y, sum(gpu::get_lane_size() - 1)); + + uint32_t z = 0; + if (gpu::get_lane_id() % 2) + z = gpu::reduce(gpu::get_lane_mask(), 1); + gpu::sync_lane(mask); + + EXPECT_EQ(z, gpu::get_lane_id() % 2 ? gpu::get_lane_size() / 2 : 0); +} + +// Tests an accumulation scan within a convergent warp or wavefront using some +// known values. For example, if every element in the lane is one, then the scan +// should have each element be equivalent to its ID, i.e. 1, 1 + 1, ... +static void test_scan() { + uint64_t mask = gpu::get_lane_mask(); + + uint32_t x = gpu::scan(mask, 1); + EXPECT_EQ(x, gpu::get_lane_id() + 1); + + uint32_t y = gpu::scan(mask, gpu::get_lane_id()); + EXPECT_EQ(y, sum(gpu::get_lane_id())); + + uint32_t z = 0; + if (gpu::get_lane_id() % 2) + z = gpu::scan(gpu::get_lane_mask(), 1); + gpu::sync_lane(mask); + + EXPECT_EQ(z, gpu::get_lane_id() % 2 ? gpu::get_lane_id() / 2 + 1 : 0); +} + +TEST_MAIN(int argc, char **argv, char **envp) { + test_reduce(); + + test_scan(); + + return 0; +} -- GitLab From 0ebf511ad011a83022edb171e044c98d9d16b1fa Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Tue, 12 Mar 2024 08:44:06 -0700 Subject: [PATCH 0014/1407] [libc] move non functions to math_extras (#84818) As per TODOs added in https://github.com/llvm/llvm-project/pull/84035/commits/48b0bc837085a38ff1de33010d9222363f70238f. --- libc/src/__support/CMakeLists.txt | 1 + libc/src/__support/CPP/bit.h | 37 --------------- libc/src/__support/math_extras.h | 37 ++++++++++++++- libc/src/stdbit/CMakeLists.txt | 46 +++++++++++-------- libc/src/stdbit/stdc_count_zeros_uc.cpp | 4 +- libc/src/stdbit/stdc_count_zeros_ui.cpp | 4 +- libc/src/stdbit/stdc_count_zeros_ul.cpp | 4 +- libc/src/stdbit/stdc_count_zeros_ull.cpp | 4 +- libc/src/stdbit/stdc_count_zeros_us.cpp | 4 +- libc/src/stdbit/stdc_first_leading_one_uc.cpp | 4 +- libc/src/stdbit/stdc_first_leading_one_ui.cpp | 4 +- libc/src/stdbit/stdc_first_leading_one_ul.cpp | 4 +- .../src/stdbit/stdc_first_leading_one_ull.cpp | 4 +- libc/src/stdbit/stdc_first_leading_one_us.cpp | 4 +- .../src/stdbit/stdc_first_leading_zero_uc.cpp | 4 +- .../src/stdbit/stdc_first_leading_zero_ui.cpp | 4 +- .../src/stdbit/stdc_first_leading_zero_ul.cpp | 4 +- .../stdbit/stdc_first_leading_zero_ull.cpp | 4 +- .../src/stdbit/stdc_first_leading_zero_us.cpp | 4 +- .../src/stdbit/stdc_first_trailing_one_uc.cpp | 4 +- .../src/stdbit/stdc_first_trailing_one_ui.cpp | 4 +- .../src/stdbit/stdc_first_trailing_one_ul.cpp | 4 +- .../stdbit/stdc_first_trailing_one_ull.cpp | 4 +- .../src/stdbit/stdc_first_trailing_one_us.cpp | 4 +- .../stdbit/stdc_first_trailing_zero_uc.cpp | 4 +- .../stdbit/stdc_first_trailing_zero_ui.cpp | 4 +- .../stdbit/stdc_first_trailing_zero_ul.cpp | 4 +- .../stdbit/stdc_first_trailing_zero_ull.cpp | 4 +- .../stdbit/stdc_first_trailing_zero_us.cpp | 4 +- libc/test/src/__support/CPP/bit_test.cpp | 32 ------------- libc/test/src/__support/math_extras_test.cpp | 40 ++++++++++++++++ 31 files changed, 154 insertions(+), 139 deletions(-) diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt index 2e5a026bf423..4c1f271e1df4 100644 --- a/libc/src/__support/CMakeLists.txt +++ b/libc/src/__support/CMakeLists.txt @@ -34,6 +34,7 @@ add_header_library( HDRS math_extras.h DEPENDS + libc.src.__support.CPP.bit libc.src.__support.CPP.limits libc.src.__support.CPP.type_traits libc.src.__support.macros.attributes diff --git a/libc/src/__support/CPP/bit.h b/libc/src/__support/CPP/bit.h index 1a05728b8506..3f2fbec94405 100644 --- a/libc/src/__support/CPP/bit.h +++ b/libc/src/__support/CPP/bit.h @@ -239,36 +239,6 @@ LIBC_INLINE constexpr To bit_or_static_cast(const From &from) { } } -// TODO: remove from 'bit.h' as it is not a standard function. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -first_leading_zero(T value) { - return value == cpp::numeric_limits::max() ? 0 : countl_one(value) + 1; -} - -// TODO: remove from 'bit.h' as it is not a standard function. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -first_leading_one(T value) { - return first_leading_zero(static_cast(~value)); -} - -// TODO: remove from 'bit.h' as it is not a standard function. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -first_trailing_zero(T value) { - return value == cpp::numeric_limits::max() - ? 0 - : countr_zero(static_cast(~value)) + 1; -} - -// TODO: remove from 'bit.h' as it is not a standard function. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -first_trailing_one(T value) { - return value == cpp::numeric_limits::max() ? 0 : countr_zero(value) + 1; -} - /// Count number of 1's aka population count or Hamming weight. /// /// Only unsigned integral types are allowed. @@ -294,13 +264,6 @@ ADD_SPECIALIZATION(unsigned long long, __builtin_popcountll) // TODO: 128b specializations? #undef ADD_SPECIALIZATION -// TODO: remove from 'bit.h' as it is not a standard function. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -count_zeros(T value) { - return popcount(static_cast(~value)); -} - } // namespace LIBC_NAMESPACE::cpp #endif // LLVM_LIBC_SRC___SUPPORT_CPP_BIT_H diff --git a/libc/src/__support/math_extras.h b/libc/src/__support/math_extras.h index c6b458ddecda..28ee1be8b999 100644 --- a/libc/src/__support/math_extras.h +++ b/libc/src/__support/math_extras.h @@ -10,7 +10,8 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_MATH_EXTRAS_H #define LLVM_LIBC_SRC___SUPPORT_MATH_EXTRAS_H -#include "src/__support/CPP/limits.h" // CHAR_BIT +#include "src/__support/CPP/bit.h" // countl_one, countr_zero +#include "src/__support/CPP/limits.h" // CHAR_BIT, numeric_limits #include "src/__support/CPP/type_traits.h" // is_unsigned_v #include "src/__support/macros/attributes.h" // LIBC_INLINE #include "src/__support/macros/config.h" // LIBC_HAS_BUILTIN @@ -226,6 +227,40 @@ sub_with_borrow(unsigned long long a, unsigned long long b, #endif // LIBC_HAS_BUILTIN(__builtin_subc) +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +first_leading_zero(T value) { + return value == cpp::numeric_limits::max() ? 0 + : cpp::countl_one(value) + 1; +} + +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +first_leading_one(T value) { + return first_leading_zero(static_cast(~value)); +} + +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +first_trailing_zero(T value) { + return value == cpp::numeric_limits::max() + ? 0 + : cpp::countr_zero(static_cast(~value)) + 1; +} + +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +first_trailing_one(T value) { + return value == cpp::numeric_limits::max() ? 0 + : cpp::countr_zero(value) + 1; +} + +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +count_zeros(T value) { + return cpp::popcount(static_cast(~value)); +} + } // namespace LIBC_NAMESPACE #endif // LLVM_LIBC_SRC___SUPPORT_MATH_EXTRAS_H diff --git a/libc/src/stdbit/CMakeLists.txt b/libc/src/stdbit/CMakeLists.txt index 2aef2029f2df..0c22b1d2617a 100644 --- a/libc/src/stdbit/CMakeLists.txt +++ b/libc/src/stdbit/CMakeLists.txt @@ -1,30 +1,38 @@ +function(declare_dependencies prefixes dependencies) + set(suffixes c s i l ll) + foreach(prefix ${prefixes}) + foreach(suffix IN LISTS suffixes) + add_entrypoint_object( + stdc_${prefix}_u${suffix} + SRCS + stdc_${prefix}_u${suffix}.cpp + HDRS + stdc_${prefix}_u${suffix}.h + DEPENDS + ${dependencies} + ) + endforeach() + endforeach() +endfunction() + + set(prefixes leading_zeros leading_ones trailing_zeros trailing_ones - first_leading_zero - first_leading_one - first_trailing_zero - first_trailing_one - count_zeros count_ones has_single_bit bit_width bit_floor bit_ceil ) -set(suffixes c s i l ll) -foreach(prefix IN LISTS prefixes) - foreach(suffix IN LISTS suffixes) - add_entrypoint_object( - stdc_${prefix}_u${suffix} - SRCS - stdc_${prefix}_u${suffix}.cpp - HDRS - stdc_${prefix}_u${suffix}.h - DEPENDS - libc.src.__support.CPP.bit - ) - endforeach() -endforeach() +declare_dependencies("${prefixes}" libc.src.__support.CPP.bit) +set(prefixes + first_leading_zero + first_leading_one + first_trailing_zero + first_trailing_one + count_zeros +) +declare_dependencies("${prefixes}" libc.src.__support.math_extras) diff --git a/libc/src/stdbit/stdc_count_zeros_uc.cpp b/libc/src/stdbit/stdc_count_zeros_uc.cpp index 22c57bd60c38..309ebb55e0fa 100644 --- a/libc/src/stdbit/stdc_count_zeros_uc.cpp +++ b/libc/src/stdbit/stdc_count_zeros_uc.cpp @@ -8,13 +8,13 @@ #include "src/stdbit/stdc_count_zeros_uc.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_count_zeros_uc, (unsigned char value)) { - return static_cast(cpp::count_zeros(value)); + return static_cast(count_zeros(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_count_zeros_ui.cpp b/libc/src/stdbit/stdc_count_zeros_ui.cpp index 6a1defd9d555..31ea907b24de 100644 --- a/libc/src/stdbit/stdc_count_zeros_ui.cpp +++ b/libc/src/stdbit/stdc_count_zeros_ui.cpp @@ -8,13 +8,13 @@ #include "src/stdbit/stdc_count_zeros_ui.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_count_zeros_ui, (unsigned value)) { - return static_cast(cpp::count_zeros(value)); + return static_cast(count_zeros(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_count_zeros_ul.cpp b/libc/src/stdbit/stdc_count_zeros_ul.cpp index ceab32ef9ac3..f5df5c49f131 100644 --- a/libc/src/stdbit/stdc_count_zeros_ul.cpp +++ b/libc/src/stdbit/stdc_count_zeros_ul.cpp @@ -8,13 +8,13 @@ #include "src/stdbit/stdc_count_zeros_ul.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_count_zeros_ul, (unsigned long value)) { - return static_cast(cpp::count_zeros(value)); + return static_cast(count_zeros(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_count_zeros_ull.cpp b/libc/src/stdbit/stdc_count_zeros_ull.cpp index 2f57f727a691..6a9c8f04a799 100644 --- a/libc/src/stdbit/stdc_count_zeros_ull.cpp +++ b/libc/src/stdbit/stdc_count_zeros_ull.cpp @@ -8,13 +8,13 @@ #include "src/stdbit/stdc_count_zeros_ull.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_count_zeros_ull, (unsigned long long value)) { - return static_cast(cpp::count_zeros(value)); + return static_cast(count_zeros(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_count_zeros_us.cpp b/libc/src/stdbit/stdc_count_zeros_us.cpp index fc06836ee292..c08186ec6e87 100644 --- a/libc/src/stdbit/stdc_count_zeros_us.cpp +++ b/libc/src/stdbit/stdc_count_zeros_us.cpp @@ -8,13 +8,13 @@ #include "src/stdbit/stdc_count_zeros_us.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_count_zeros_us, (unsigned short value)) { - return static_cast(cpp::count_zeros(value)); + return static_cast(count_zeros(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_leading_one_uc.cpp b/libc/src/stdbit/stdc_first_leading_one_uc.cpp index 02871595fdb6..2e28ed3bb6f8 100644 --- a/libc/src/stdbit/stdc_first_leading_one_uc.cpp +++ b/libc/src/stdbit/stdc_first_leading_one_uc.cpp @@ -8,13 +8,13 @@ #include "src/stdbit/stdc_first_leading_one_uc.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_leading_one_uc, (unsigned char value)) { - return static_cast(cpp::first_leading_one(value)); + return static_cast(first_leading_one(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_leading_one_ui.cpp b/libc/src/stdbit/stdc_first_leading_one_ui.cpp index a6c7ef5a8339..a07a39b09d9f 100644 --- a/libc/src/stdbit/stdc_first_leading_one_ui.cpp +++ b/libc/src/stdbit/stdc_first_leading_one_ui.cpp @@ -8,13 +8,13 @@ #include "src/stdbit/stdc_first_leading_one_ui.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_leading_one_ui, (unsigned value)) { - return static_cast(cpp::first_leading_one(value)); + return static_cast(first_leading_one(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_leading_one_ul.cpp b/libc/src/stdbit/stdc_first_leading_one_ul.cpp index d1bcab5dda02..4350fb7826b4 100644 --- a/libc/src/stdbit/stdc_first_leading_one_ul.cpp +++ b/libc/src/stdbit/stdc_first_leading_one_ul.cpp @@ -8,13 +8,13 @@ #include "src/stdbit/stdc_first_leading_one_ul.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_leading_one_ul, (unsigned long value)) { - return static_cast(cpp::first_leading_one(value)); + return static_cast(first_leading_one(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_leading_one_ull.cpp b/libc/src/stdbit/stdc_first_leading_one_ull.cpp index 7be8f1051ec2..57a5ae368e11 100644 --- a/libc/src/stdbit/stdc_first_leading_one_ull.cpp +++ b/libc/src/stdbit/stdc_first_leading_one_ull.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_leading_one_ull.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_leading_one_ull, (unsigned long long value)) { - return static_cast(cpp::first_leading_one(value)); + return static_cast(first_leading_one(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_leading_one_us.cpp b/libc/src/stdbit/stdc_first_leading_one_us.cpp index 7a4c7e673f36..f14433b13f35 100644 --- a/libc/src/stdbit/stdc_first_leading_one_us.cpp +++ b/libc/src/stdbit/stdc_first_leading_one_us.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_leading_one_us.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_leading_one_us, (unsigned short value)) { - return static_cast(cpp::first_leading_one(value)); + return static_cast(first_leading_one(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_leading_zero_uc.cpp b/libc/src/stdbit/stdc_first_leading_zero_uc.cpp index ffc1d9247406..6e2164256f17 100644 --- a/libc/src/stdbit/stdc_first_leading_zero_uc.cpp +++ b/libc/src/stdbit/stdc_first_leading_zero_uc.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_leading_zero_uc.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_leading_zero_uc, (unsigned char value)) { - return static_cast(cpp::first_leading_zero(value)); + return static_cast(first_leading_zero(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_leading_zero_ui.cpp b/libc/src/stdbit/stdc_first_leading_zero_ui.cpp index 1eeab2963e6a..cb733a94c0d8 100644 --- a/libc/src/stdbit/stdc_first_leading_zero_ui.cpp +++ b/libc/src/stdbit/stdc_first_leading_zero_ui.cpp @@ -8,13 +8,13 @@ #include "src/stdbit/stdc_first_leading_zero_ui.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_leading_zero_ui, (unsigned value)) { - return static_cast(cpp::first_leading_zero(value)); + return static_cast(first_leading_zero(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_leading_zero_ul.cpp b/libc/src/stdbit/stdc_first_leading_zero_ul.cpp index 6743d3eda516..8a3930a271ed 100644 --- a/libc/src/stdbit/stdc_first_leading_zero_ul.cpp +++ b/libc/src/stdbit/stdc_first_leading_zero_ul.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_leading_zero_ul.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_leading_zero_ul, (unsigned long value)) { - return static_cast(cpp::first_leading_zero(value)); + return static_cast(first_leading_zero(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_leading_zero_ull.cpp b/libc/src/stdbit/stdc_first_leading_zero_ull.cpp index 8128dd3d59a7..5a69197a8299 100644 --- a/libc/src/stdbit/stdc_first_leading_zero_ull.cpp +++ b/libc/src/stdbit/stdc_first_leading_zero_ull.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_leading_zero_ull.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_leading_zero_ull, (unsigned long long value)) { - return static_cast(cpp::first_leading_zero(value)); + return static_cast(first_leading_zero(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_leading_zero_us.cpp b/libc/src/stdbit/stdc_first_leading_zero_us.cpp index d931535e7690..6482c8654db3 100644 --- a/libc/src/stdbit/stdc_first_leading_zero_us.cpp +++ b/libc/src/stdbit/stdc_first_leading_zero_us.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_leading_zero_us.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_leading_zero_us, (unsigned short value)) { - return static_cast(cpp::first_leading_zero(value)); + return static_cast(first_leading_zero(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_trailing_one_uc.cpp b/libc/src/stdbit/stdc_first_trailing_one_uc.cpp index 6ed35966be61..d3e8825eef00 100644 --- a/libc/src/stdbit/stdc_first_trailing_one_uc.cpp +++ b/libc/src/stdbit/stdc_first_trailing_one_uc.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_trailing_one_uc.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_trailing_one_uc, (unsigned char value)) { - return static_cast(cpp::first_trailing_one(value)); + return static_cast(first_trailing_one(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_trailing_one_ui.cpp b/libc/src/stdbit/stdc_first_trailing_one_ui.cpp index a89083bd4950..842bd6995050 100644 --- a/libc/src/stdbit/stdc_first_trailing_one_ui.cpp +++ b/libc/src/stdbit/stdc_first_trailing_one_ui.cpp @@ -8,13 +8,13 @@ #include "src/stdbit/stdc_first_trailing_one_ui.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_trailing_one_ui, (unsigned value)) { - return static_cast(cpp::first_trailing_one(value)); + return static_cast(first_trailing_one(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_trailing_one_ul.cpp b/libc/src/stdbit/stdc_first_trailing_one_ul.cpp index f30078d0f5ff..0497d1d77811 100644 --- a/libc/src/stdbit/stdc_first_trailing_one_ul.cpp +++ b/libc/src/stdbit/stdc_first_trailing_one_ul.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_trailing_one_ul.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_trailing_one_ul, (unsigned long value)) { - return static_cast(cpp::first_trailing_one(value)); + return static_cast(first_trailing_one(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_trailing_one_ull.cpp b/libc/src/stdbit/stdc_first_trailing_one_ull.cpp index 2e526a890cda..6e062dd27cdd 100644 --- a/libc/src/stdbit/stdc_first_trailing_one_ull.cpp +++ b/libc/src/stdbit/stdc_first_trailing_one_ull.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_trailing_one_ull.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_trailing_one_ull, (unsigned long long value)) { - return static_cast(cpp::first_trailing_one(value)); + return static_cast(first_trailing_one(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_trailing_one_us.cpp b/libc/src/stdbit/stdc_first_trailing_one_us.cpp index e4c88e0d7906..e90158f10204 100644 --- a/libc/src/stdbit/stdc_first_trailing_one_us.cpp +++ b/libc/src/stdbit/stdc_first_trailing_one_us.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_trailing_one_us.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_trailing_one_us, (unsigned short value)) { - return static_cast(cpp::first_trailing_one(value)); + return static_cast(first_trailing_one(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_trailing_zero_uc.cpp b/libc/src/stdbit/stdc_first_trailing_zero_uc.cpp index 5825d5d441c5..a6939f6286b3 100644 --- a/libc/src/stdbit/stdc_first_trailing_zero_uc.cpp +++ b/libc/src/stdbit/stdc_first_trailing_zero_uc.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_trailing_zero_uc.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_trailing_zero_uc, (unsigned char value)) { - return static_cast(cpp::first_trailing_zero(value)); + return static_cast(first_trailing_zero(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_trailing_zero_ui.cpp b/libc/src/stdbit/stdc_first_trailing_zero_ui.cpp index 3b51b5fa22c3..7a50b696afff 100644 --- a/libc/src/stdbit/stdc_first_trailing_zero_ui.cpp +++ b/libc/src/stdbit/stdc_first_trailing_zero_ui.cpp @@ -8,13 +8,13 @@ #include "src/stdbit/stdc_first_trailing_zero_ui.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_trailing_zero_ui, (unsigned value)) { - return static_cast(cpp::first_trailing_zero(value)); + return static_cast(first_trailing_zero(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_trailing_zero_ul.cpp b/libc/src/stdbit/stdc_first_trailing_zero_ul.cpp index abf122944a76..88acbabdf2d9 100644 --- a/libc/src/stdbit/stdc_first_trailing_zero_ul.cpp +++ b/libc/src/stdbit/stdc_first_trailing_zero_ul.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_trailing_zero_ul.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_trailing_zero_ul, (unsigned long value)) { - return static_cast(cpp::first_trailing_zero(value)); + return static_cast(first_trailing_zero(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_trailing_zero_ull.cpp b/libc/src/stdbit/stdc_first_trailing_zero_ull.cpp index 336e7d6e075f..92df8f284e8b 100644 --- a/libc/src/stdbit/stdc_first_trailing_zero_ull.cpp +++ b/libc/src/stdbit/stdc_first_trailing_zero_ull.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_trailing_zero_ull.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_trailing_zero_ull, (unsigned long long value)) { - return static_cast(cpp::first_trailing_zero(value)); + return static_cast(first_trailing_zero(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_first_trailing_zero_us.cpp b/libc/src/stdbit/stdc_first_trailing_zero_us.cpp index b7d05047b272..86caa20dd3bd 100644 --- a/libc/src/stdbit/stdc_first_trailing_zero_us.cpp +++ b/libc/src/stdbit/stdc_first_trailing_zero_us.cpp @@ -8,14 +8,14 @@ #include "src/stdbit/stdc_first_trailing_zero_us.h" -#include "src/__support/CPP/bit.h" #include "src/__support/common.h" +#include "src/__support/math_extras.h" namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_first_trailing_zero_us, (unsigned short value)) { - return static_cast(cpp::first_trailing_zero(value)); + return static_cast(first_trailing_zero(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/test/src/__support/CPP/bit_test.cpp b/libc/test/src/__support/CPP/bit_test.cpp index 3deb1f41dcf8..cee5b90c8f4b 100644 --- a/libc/test/src/__support/CPP/bit_test.cpp +++ b/libc/test/src/__support/CPP/bit_test.cpp @@ -228,38 +228,6 @@ TEST(LlvmLibcBitTest, Rotr) { rotr(0x12345678deadbeefULL, -19)); } -TYPED_TEST(LlvmLibcBitTest, FirstLeadingZero, UnsignedTypesNoBigInt) { - EXPECT_EQ(first_leading_zero(cpp::numeric_limits::max()), 0); - for (int i = 0U; i != cpp::numeric_limits::digits; ++i) - EXPECT_EQ(first_leading_zero(~(T(1) << i)), - cpp::numeric_limits::digits - i); -} - -TYPED_TEST(LlvmLibcBitTest, FirstLeadingOne, UnsignedTypesNoBigInt) { - EXPECT_EQ(first_leading_one(static_cast(0)), 0); - for (int i = 0U; i != cpp::numeric_limits::digits; ++i) - EXPECT_EQ(first_leading_one(T(1) << i), - cpp::numeric_limits::digits - i); -} - -TYPED_TEST(LlvmLibcBitTest, FirstTrailingZero, UnsignedTypesNoBigInt) { - EXPECT_EQ(first_trailing_zero(cpp::numeric_limits::max()), 0); - for (int i = 0U; i != cpp::numeric_limits::digits; ++i) - EXPECT_EQ(first_trailing_zero(~(T(1) << i)), i + 1); -} - -TYPED_TEST(LlvmLibcBitTest, FirstTrailingOne, UnsignedTypesNoBigInt) { - EXPECT_EQ(first_trailing_one(cpp::numeric_limits::max()), 0); - for (int i = 0U; i != cpp::numeric_limits::digits; ++i) - EXPECT_EQ(first_trailing_one(T(1) << i), i + 1); -} - -TYPED_TEST(LlvmLibcBitTest, CountZeros, UnsignedTypesNoBigInt) { - EXPECT_EQ(count_zeros(T(0)), cpp::numeric_limits::digits); - for (int i = 0; i != cpp::numeric_limits::digits; ++i) - EXPECT_EQ(count_zeros(cpp::numeric_limits::max() >> i), i); -} - TYPED_TEST(LlvmLibcBitTest, CountOnes, UnsignedTypesNoBigInt) { EXPECT_EQ(popcount(T(0)), 0); for (int i = 0; i != cpp::numeric_limits::digits; ++i) diff --git a/libc/test/src/__support/math_extras_test.cpp b/libc/test/src/__support/math_extras_test.cpp index ed064363d446..e642248881a4 100644 --- a/libc/test/src/__support/math_extras_test.cpp +++ b/libc/test/src/__support/math_extras_test.cpp @@ -13,6 +13,14 @@ namespace LIBC_NAMESPACE { +// TODO: add UInt<128> support. +using UnsignedTypesNoBigInt = testing::TypeList< +#if defined(LIBC_TYPES_HAS_INT128) + __uint128_t, +#endif // LIBC_TYPES_HAS_INT128 + unsigned char, unsigned short, unsigned int, unsigned long, + unsigned long long>; + TEST(LlvmLibcBlockMathExtrasTest, mask_trailing_ones) { EXPECT_EQ(0_u8, (mask_leading_ones())); EXPECT_EQ(0_u8, (mask_trailing_ones())); @@ -61,4 +69,36 @@ TEST(LlvmLibcBlockMathExtrasTest, mask_trailing_ones) { (mask_leading_ones())); } +TYPED_TEST(LlvmLibcBitTest, FirstLeadingZero, UnsignedTypesNoBigInt) { + EXPECT_EQ(first_leading_zero(cpp::numeric_limits::max()), 0); + for (int i = 0U; i != cpp::numeric_limits::digits; ++i) + EXPECT_EQ(first_leading_zero(~(T(1) << i)), + cpp::numeric_limits::digits - i); +} + +TYPED_TEST(LlvmLibcBitTest, FirstLeadingOne, UnsignedTypesNoBigInt) { + EXPECT_EQ(first_leading_one(static_cast(0)), 0); + for (int i = 0U; i != cpp::numeric_limits::digits; ++i) + EXPECT_EQ(first_leading_one(T(1) << i), + cpp::numeric_limits::digits - i); +} + +TYPED_TEST(LlvmLibcBitTest, FirstTrailingZero, UnsignedTypesNoBigInt) { + EXPECT_EQ(first_trailing_zero(cpp::numeric_limits::max()), 0); + for (int i = 0U; i != cpp::numeric_limits::digits; ++i) + EXPECT_EQ(first_trailing_zero(~(T(1) << i)), i + 1); +} + +TYPED_TEST(LlvmLibcBitTest, FirstTrailingOne, UnsignedTypesNoBigInt) { + EXPECT_EQ(first_trailing_one(cpp::numeric_limits::max()), 0); + for (int i = 0U; i != cpp::numeric_limits::digits; ++i) + EXPECT_EQ(first_trailing_one(T(1) << i), i + 1); +} + +TYPED_TEST(LlvmLibcBitTest, CountZeros, UnsignedTypesNoBigInt) { + EXPECT_EQ(count_zeros(T(0)), cpp::numeric_limits::digits); + for (int i = 0; i != cpp::numeric_limits::digits; ++i) + EXPECT_EQ(count_zeros(cpp::numeric_limits::max() >> i), i); +} + } // namespace LIBC_NAMESPACE -- GitLab From 87dc068280aaddc98acb7865ae3df1d248f4a170 Mon Sep 17 00:00:00 2001 From: Jason Molenda Date: Tue, 12 Mar 2024 09:03:28 -0700 Subject: [PATCH 0015/1407] [lldb] [debugserver] Handle interrupted reads correctly (#84872) The first half of this patch is a long-standing annoyance, if I attach to debugserver with lldb while it is waiting for an lldb connection, the syscall is interrupted and it doesn't retry, debugserver exits immediately. The second half is a request from another tool that is communicating with debugserver, that we retry reads on our sockets in the same way. I haven't dug in to the details of how they're communicating that this is necessary, but the best I've been able to find reading the POSIX API docs, this is fine. rdar://117113298 --- lldb/tools/debugserver/source/RNBSocket.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lldb/tools/debugserver/source/RNBSocket.cpp b/lldb/tools/debugserver/source/RNBSocket.cpp index 1282ea221625..fc55dbf2e1f1 100644 --- a/lldb/tools/debugserver/source/RNBSocket.cpp +++ b/lldb/tools/debugserver/source/RNBSocket.cpp @@ -120,8 +120,13 @@ rnb_err_t RNBSocket::Listen(const char *listen_host, uint16_t port, while (!accept_connection) { struct kevent event_list[4]; - int num_events = - kevent(queue_id, events.data(), events.size(), event_list, 4, NULL); + int num_events; + do { + errno = 0; + num_events = + kevent(queue_id, events.data(), events.size(), event_list, 4, NULL); + } while (num_events == -1 && + (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)); if (num_events < 0) { err.SetError(errno, DNBError::MachKernel); @@ -291,7 +296,12 @@ rnb_err_t RNBSocket::Read(std::string &p) { // DNBLogThreadedIf(LOG_RNB_COMM, "%8u RNBSocket::%s calling read()", // (uint32_t)m_timer.ElapsedMicroSeconds(true), __FUNCTION__); DNBError err; - ssize_t bytesread = read(m_fd, buf, sizeof(buf)); + ssize_t bytesread; + do { + errno = 0; + bytesread = read(m_fd, buf, sizeof(buf)); + } while (bytesread == -1 && + (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)); if (bytesread <= 0) err.SetError(errno, DNBError::POSIX); else -- GitLab From fa1d13590cfd0be77c452cca929bc32efb456627 Mon Sep 17 00:00:00 2001 From: Jake Egan Date: Tue, 12 Mar 2024 12:03:42 -0400 Subject: [PATCH 0016/1407] [AIX][tests] Disable failing tests on AIX These new tests are failing on the AIX bot because the -I option isn't supported. Disable these tests for now until they can be fixed. --- llvm/test/CodeGen/AMDGPU/lds-run-twice-absolute-md.ll | 2 ++ llvm/test/CodeGen/AMDGPU/lds-run-twice.ll | 2 ++ 2 files changed, 4 insertions(+) diff --git a/llvm/test/CodeGen/AMDGPU/lds-run-twice-absolute-md.ll b/llvm/test/CodeGen/AMDGPU/lds-run-twice-absolute-md.ll index 52b44eea35c8..51e10d903797 100644 --- a/llvm/test/CodeGen/AMDGPU/lds-run-twice-absolute-md.ll +++ b/llvm/test/CodeGen/AMDGPU/lds-run-twice-absolute-md.ll @@ -1,3 +1,5 @@ +; XFAIL: target={{.*}}-aix{{.*}} + ; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-module-lds %s -o %t.ll ; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-module-lds %t.ll -o %t.second.ll ; RUN: diff -ub %t.ll %t.second.ll -I ".*ModuleID.*" diff --git a/llvm/test/CodeGen/AMDGPU/lds-run-twice.ll b/llvm/test/CodeGen/AMDGPU/lds-run-twice.ll index b830ccb944a2..e121f0da327d 100644 --- a/llvm/test/CodeGen/AMDGPU/lds-run-twice.ll +++ b/llvm/test/CodeGen/AMDGPU/lds-run-twice.ll @@ -1,3 +1,5 @@ +; XFAIL: target={{.*}}-aix{{.*}} + ; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-module-lds %s -o %t.ll ; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-module-lds %t.ll -o %t.second.ll ; RUN: diff -ub %t.ll %t.second.ll -I ".*ModuleID.*" -- GitLab From 65eea3e5dc907c3059c57ab4d16770522c5b9fb0 Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Tue, 12 Mar 2024 17:26:39 +0100 Subject: [PATCH 0017/1407] [libc++][TZDB] Fixes parsing interleaved rules. (#84808) Typically the rules in the database are contiguous, but that is not a requirement. This fixes the case when they are not. --------- Co-authored-by: Louis Dionne --- libcxx/src/tzdb.cpp | 27 ++++++++++++++++--- .../time.zone/time.zone.db/rules.pass.cpp | 24 +++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/libcxx/src/tzdb.cpp b/libcxx/src/tzdb.cpp index 2bb801e48694..0307f754caab 100644 --- a/libcxx/src/tzdb.cpp +++ b/libcxx/src/tzdb.cpp @@ -511,14 +511,33 @@ static string __parse_version(istream& __input) { return chrono::__parse_string(__input); } +[[nodiscard]] +static __tz::__rule& __create_entry(__tz::__rules_storage_type& __rules, const string& __name) { + auto __result = [&]() -> __tz::__rule& { + auto& __rule = __rules.emplace_back(__name, vector<__tz::__rule>{}); + return __rule.second.emplace_back(); + }; + + if (__rules.empty()) + return __result(); + + // Typically rules are in contiguous order in the database. + // But there are exceptions, some rules are interleaved. + if (__rules.back().first == __name) + return __rules.back().second.emplace_back(); + + if (auto __it = ranges::find(__rules, __name, [](const auto& __r) { return __r.first; }); + __it != ranges::end(__rules)) + return __it->second.emplace_back(); + + return __result(); +} + static void __parse_rule(tzdb& __tzdb, __tz::__rules_storage_type& __rules, istream& __input) { chrono::__skip_mandatory_whitespace(__input); string __name = chrono::__parse_string(__input); - if (__rules.empty() || __rules.back().first != __name) - __rules.emplace_back(__name, vector<__tz::__rule>{}); - - __tz::__rule& __rule = __rules.back().second.emplace_back(); + __tz::__rule& __rule = __create_entry(__rules, __name); chrono::__skip_mandatory_whitespace(__input); __rule.__from = chrono::__parse_year(__input); diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.db/rules.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.db/rules.pass.cpp index 5ae2ed1e91eb..4814f4aad87f 100644 --- a/libcxx/test/libcxx/time/time.zone/time.zone.db/rules.pass.cpp +++ b/libcxx/test/libcxx/time/time.zone/time.zone.db/rules.pass.cpp @@ -550,6 +550,29 @@ R a 0 1 - Ja Su>=31 1w 2s abc assert(result.rules[0].second[2].__letters == "abc"); } +static void test_mixed_order() { + // This is a part of the real database. The interesting part is that the + // rules NZ and Chatham are interleaved. Make sure the parse algorithm + // handles this correctly. + parse_result result{ + R"( +# Since 1957 Chatham has been 45 minutes ahead of NZ, but until 2018a +# there was no documented single notation for the date and time of this +# transition. Duplicate the Rule lines for now, to give the 2018a change +# time to percolate out. +Rule NZ 1974 only - Nov Sun>=1 2:00s 1:00 D +Rule Chatham 1974 only - Nov Sun>=1 2:45s 1:00 - +Rule NZ 1975 only - Feb lastSun 2:00s 0 S +Rule Chatham 1975 only - Feb lastSun 2:45s 0 - +Rule NZ 1975 1988 - Oct lastSun 2:00s 1:00 D +Rule Chatham 1975 1988 - Oct lastSun 2:45s 1:00 - +)"}; + + assert(result.rules.size() == 2); + assert(result.rules[0].second.size() == 3); + assert(result.rules[1].second.size() == 3); +} + int main(int, const char**) { test_invalid(); test_name(); @@ -560,6 +583,7 @@ int main(int, const char**) { test_at(); test_save(); test_letter(); + test_mixed_order(); return 0; } -- GitLab From af21659c8c5c1d16b9bc5e745aaaf49b322f64d7 Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Tue, 12 Mar 2024 17:28:15 +0100 Subject: [PATCH 0018/1407] [libc++][CI] Installs tzdata package in Docker. (#84643) This allows testing the time zone information in the CI. This is needed to let https://github.com/llvm/llvm-project/pull/82108 pass the CI. --- libcxx/utils/ci/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libcxx/utils/ci/Dockerfile b/libcxx/utils/ci/Dockerfile index 225de937cc86..178cba415933 100644 --- a/libcxx/utils/ci/Dockerfile +++ b/libcxx/utils/ci/Dockerfile @@ -65,6 +65,12 @@ RUN < Date: Tue, 12 Mar 2024 17:31:22 +0100 Subject: [PATCH 0019/1407] [clang][ASTMatchers] Fix forEachArgumentWithParam* for deducing "this" operator calls (#84887) This is a follow-up commit of #84446. In this patch, I demonstrate that `forEachArgumentWithParam` and `forEachArgumentWithParamType` did not correctly handle the presence of the explicit object parameter for operator calls. Prior to this patch, the matcher would skip the first (and only) argument of the operator call if the explicit object param was used. Note that I had to move the definition of `isExplicitObjectMemberFunction`, to be declared before the matcher I fix to be visible. I also had to do some gymnastics for passing the language standard version command-line flags to the invocation as `matchAndVerifyResultTrue` wasn't really considered for non-c++11 code. See the that it always prepends `-std=gnu++11` to the command-line arguments. I workarounded it by accepting extra args, which get appended, thus possibly overriding the hardcoded arguments. I'm not sure if this qualifies for backporting to clang-18 (probably not because its not a crash, but a semantic problem), but I figure it might be useful for some vendors (like us). But we are also happy to cherry-pick this fix to downstream. Let me know if you want this to be backported or not. CPP-5074 --- clang/docs/ReleaseNotes.rst | 2 + clang/include/clang/ASTMatchers/ASTMatchers.h | 59 ++++++++--------- clang/unittests/ASTMatchers/ASTMatchersTest.h | 36 +++++++---- .../ASTMatchers/ASTMatchersTraversalTest.cpp | 63 +++++++++++++++++++ 4 files changed, 119 insertions(+), 41 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 6c30af304d98..2842b63197ff 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -450,6 +450,8 @@ AST Matchers - ``isInStdNamespace`` now supports Decl declared with ``extern "C++"``. - Add ``isExplicitObjectMemberFunction``. +- Fixed ``forEachArgumentWithParam`` and ``forEachArgumentWithParamType`` to + not skip the explicit object parameter for operator calls. clang-format ------------ diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h index 96dbcdc344e1..2f71053d030f 100644 --- a/clang/include/clang/ASTMatchers/ASTMatchers.h +++ b/clang/include/clang/ASTMatchers/ASTMatchers.h @@ -5032,6 +5032,25 @@ AST_POLYMORPHIC_MATCHER_P2(hasParameter, && InnerMatcher.matches(*Node.parameters()[N], Finder, Builder)); } +/// Matches if the given method declaration declares a member function with an +/// explicit object parameter. +/// +/// Given +/// \code +/// struct A { +/// int operator-(this A, int); +/// void fun(this A &&self); +/// static int operator()(int); +/// int operator+(int); +/// }; +/// \endcode +/// +/// cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two +/// methods but not the last two. +AST_MATCHER(CXXMethodDecl, isExplicitObjectMemberFunction) { + return Node.isExplicitObjectMemberFunction(); +} + /// Matches all arguments and their respective ParmVarDecl. /// /// Given @@ -5060,10 +5079,12 @@ AST_POLYMORPHIC_MATCHER_P2(forEachArgumentWithParam, // argument of the method which should not be matched against a parameter, so // we skip over it here. BoundNodesTreeBuilder Matches; - unsigned ArgIndex = cxxOperatorCallExpr(callee(cxxMethodDecl())) - .matches(Node, Finder, &Matches) - ? 1 - : 0; + unsigned ArgIndex = + cxxOperatorCallExpr( + callee(cxxMethodDecl(unless(isExplicitObjectMemberFunction())))) + .matches(Node, Finder, &Matches) + ? 1 + : 0; int ParamIndex = 0; bool Matched = false; for (; ArgIndex < Node.getNumArgs(); ++ArgIndex) { @@ -5121,11 +5142,12 @@ AST_POLYMORPHIC_MATCHER_P2(forEachArgumentWithParamType, // argument of the method which should not be matched against a parameter, so // we skip over it here. BoundNodesTreeBuilder Matches; - unsigned ArgIndex = cxxOperatorCallExpr(callee(cxxMethodDecl())) - .matches(Node, Finder, &Matches) - ? 1 - : 0; - + unsigned ArgIndex = + cxxOperatorCallExpr( + callee(cxxMethodDecl(unless(isExplicitObjectMemberFunction())))) + .matches(Node, Finder, &Matches) + ? 1 + : 0; const FunctionProtoType *FProto = nullptr; if (const auto *Call = dyn_cast(&Node)) { @@ -6366,25 +6388,6 @@ AST_MATCHER(CXXMethodDecl, isConst) { return Node.isConst(); } -/// Matches if the given method declaration declares a member function with an -/// explicit object parameter. -/// -/// Given -/// \code -/// struct A { -/// int operator-(this A, int); -/// void fun(this A &&self); -/// static int operator()(int); -/// int operator+(int); -/// }; -/// \endcode -/// -/// cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two -/// methods but not the last two. -AST_MATCHER(CXXMethodDecl, isExplicitObjectMemberFunction) { - return Node.isExplicitObjectMemberFunction(); -} - /// Matches if the given method declaration declares a copy assignment /// operator. /// diff --git a/clang/unittests/ASTMatchers/ASTMatchersTest.h b/clang/unittests/ASTMatchers/ASTMatchersTest.h index 1ed1b5958a8b..e98129953157 100644 --- a/clang/unittests/ASTMatchers/ASTMatchersTest.h +++ b/clang/unittests/ASTMatchers/ASTMatchersTest.h @@ -293,7 +293,8 @@ testing::AssertionResult notMatchesWithOpenMP51(const Twine &Code, template testing::AssertionResult matchAndVerifyResultConditionally( const Twine &Code, const T &AMatcher, - std::unique_ptr FindResultVerifier, bool ExpectResult) { + std::unique_ptr FindResultVerifier, bool ExpectResult, + ArrayRef Args = {}, StringRef Filename = "input.cc") { bool VerifiedResult = false; MatchFinder Finder; VerifyMatch VerifyVerifiedResult(std::move(FindResultVerifier), @@ -304,9 +305,13 @@ testing::AssertionResult matchAndVerifyResultConditionally( // Some tests use typeof, which is a gnu extension. Using an explicit // unknown-unknown triple is good for a large speedup, because it lets us // avoid constructing a full system triple. - std::vector Args = {"-std=gnu++11", "-target", - "i386-unknown-unknown"}; - if (!runToolOnCodeWithArgs(Factory->create(), Code, Args)) { + std::vector CompileArgs = {"-std=gnu++11", "-target", + "i386-unknown-unknown"}; + // Append additional arguments at the end to allow overriding the default + // choices that we made above. + llvm::copy(Args, std::back_inserter(CompileArgs)); + + if (!runToolOnCodeWithArgs(Factory->create(), Code, CompileArgs, Filename)) { return testing::AssertionFailure() << "Parsing error in \"" << Code << "\""; } if (!VerifiedResult && ExpectResult) { @@ -319,8 +324,8 @@ testing::AssertionResult matchAndVerifyResultConditionally( VerifiedResult = false; SmallString<256> Buffer; - std::unique_ptr AST( - buildASTFromCodeWithArgs(Code.toStringRef(Buffer), Args)); + std::unique_ptr AST(buildASTFromCodeWithArgs( + Code.toStringRef(Buffer), CompileArgs, Filename)); if (!AST.get()) return testing::AssertionFailure() << "Parsing error in \"" << Code << "\" while building AST"; @@ -339,19 +344,24 @@ testing::AssertionResult matchAndVerifyResultConditionally( // FIXME: Find better names for these functions (or document what they // do more precisely). template -testing::AssertionResult matchAndVerifyResultTrue( - const Twine &Code, const T &AMatcher, - std::unique_ptr FindResultVerifier) { - return matchAndVerifyResultConditionally(Code, AMatcher, - std::move(FindResultVerifier), true); +testing::AssertionResult +matchAndVerifyResultTrue(const Twine &Code, const T &AMatcher, + std::unique_ptr FindResultVerifier, + ArrayRef Args = {}, + StringRef Filename = "input.cc") { + return matchAndVerifyResultConditionally( + Code, AMatcher, std::move(FindResultVerifier), + /*ExpectResult=*/true, Args, Filename); } template testing::AssertionResult matchAndVerifyResultFalse( const Twine &Code, const T &AMatcher, - std::unique_ptr FindResultVerifier) { + std::unique_ptr FindResultVerifier, + ArrayRef Args = {}, StringRef Filename = "input.cc") { return matchAndVerifyResultConditionally( - Code, AMatcher, std::move(FindResultVerifier), false); + Code, AMatcher, std::move(FindResultVerifier), + /*ExpectResult=*/false, Args, Filename); } // Implements a run method that returns whether BoundNodes contains a diff --git a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp index 6911d7600a71..f198dc71eb83 100644 --- a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp +++ b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp @@ -985,6 +985,38 @@ TEST(ForEachArgumentWithParam, HandlesBoundNodesForNonMatches) { std::make_unique>("v", 4))); } +TEST_P(ASTMatchersTest, + ForEachArgumentWithParamMatchesExplicitObjectParamOnOperatorCalls) { + if (!GetParam().isCXX23OrLater()) { + return; + } + + auto DeclRef = declRefExpr(to(varDecl().bind("declOfArg"))).bind("arg"); + auto SelfParam = parmVarDecl().bind("param"); + StatementMatcher CallExpr = + callExpr(forEachArgumentWithParam(DeclRef, SelfParam)); + + StringRef S = R"cpp( + struct A { + int operator()(this const A &self); + }; + A obj; + int global = obj(); + )cpp"; + + auto Args = GetParam().getCommandLineArgs(); + auto Filename = getFilenameForTesting(GetParam().Language); + + EXPECT_TRUE(matchAndVerifyResultTrue( + S, CallExpr, + std::make_unique>("param", "self"), Args, + Filename)); + EXPECT_TRUE(matchAndVerifyResultTrue( + S, CallExpr, + std::make_unique>("declOfArg", "obj"), Args, + Filename)); +} + TEST(ForEachArgumentWithParamType, ReportsNoFalsePositives) { StatementMatcher ArgumentY = declRefExpr(to(varDecl(hasName("y")))).bind("arg"); @@ -1168,6 +1200,37 @@ TEST(ForEachArgumentWithParamType, MatchesVariadicFunctionPtrCalls) { S, CallExpr, std::make_unique>("arg"))); } +TEST_P(ASTMatchersTest, + ForEachArgumentWithParamTypeMatchesExplicitObjectParamOnOperatorCalls) { + if (!GetParam().isCXX23OrLater()) { + return; + } + + auto DeclRef = declRefExpr(to(varDecl().bind("declOfArg"))).bind("arg"); + auto SelfTy = qualType(asString("const A &")).bind("selfType"); + StatementMatcher CallExpr = + callExpr(forEachArgumentWithParamType(DeclRef, SelfTy)); + + StringRef S = R"cpp( + struct A { + int operator()(this const A &self); + }; + A obj; + int global = obj(); + )cpp"; + + auto Args = GetParam().getCommandLineArgs(); + auto Filename = getFilenameForTesting(GetParam().Language); + + EXPECT_TRUE(matchAndVerifyResultTrue( + S, CallExpr, std::make_unique>("selfType"), + Args, Filename)); + EXPECT_TRUE(matchAndVerifyResultTrue( + S, CallExpr, + std::make_unique>("declOfArg", "obj"), Args, + Filename)); +} + TEST(QualType, hasCanonicalType) { EXPECT_TRUE(notMatches("typedef int &int_ref;" "int a;" -- GitLab From c8cc7903b373589cc85271987980ae277145df7c Mon Sep 17 00:00:00 2001 From: XChy Date: Wed, 13 Mar 2024 00:33:50 +0800 Subject: [PATCH 0020/1407] [SelectionDAG] Replace some basic patterns in visitADDLike with SDPatternMatch (#84759) Resolves #84745. Based on SDPatternMatch introduced by #78654, this patch replaces some of basic patterns in `visitADDLike` with corresponding patterns in SDPatternMatch. This patch only replaces original folds, instead of introducing new ones. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 55 +++++++++---------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 5476ef879714..735cec8ecc06 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -38,6 +38,7 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/RuntimeLibcalls.h" +#include "llvm/CodeGen/SDPatternMatch.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h" #include "llvm/CodeGen/SelectionDAGNodes.h" @@ -79,6 +80,7 @@ #include "MatchContext.h" using namespace llvm; +using namespace llvm::SDPatternMatch; #define DEBUG_TYPE "dagcombine" @@ -2697,52 +2699,45 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) { reassociateReduction(ISD::VECREDUCE_ADD, ISD::ADD, DL, VT, N0, N1)) return SD; } + + SDValue A, B, C; + // fold ((0-A) + B) -> B-A - if (N0.getOpcode() == ISD::SUB && isNullOrNullSplat(N0.getOperand(0))) - return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); + if (sd_match(N0, m_Sub(m_Zero(), m_Value(A)))) + return DAG.getNode(ISD::SUB, DL, VT, N1, A); // fold (A + (0-B)) -> A-B - if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0))) - return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1)); + if (sd_match(N1, m_Sub(m_Zero(), m_Value(B)))) + return DAG.getNode(ISD::SUB, DL, VT, N0, B); // fold (A+(B-A)) -> B - if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1)) - return N1.getOperand(0); + if (sd_match(N1, m_Sub(m_Value(B), m_Specific(N0)))) + return B; // fold ((B-A)+A) -> B - if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1)) - return N0.getOperand(0); + if (sd_match(N0, m_Sub(m_Value(B), m_Specific(N1)))) + return B; // fold ((A-B)+(C-A)) -> (C-B) - if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB && - N0.getOperand(0) == N1.getOperand(1)) - return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), - N0.getOperand(1)); + if (sd_match(N0, m_Sub(m_Value(A), m_Value(B))) && + sd_match(N1, m_Sub(m_Value(C), m_Specific(A)))) + return DAG.getNode(ISD::SUB, DL, VT, C, B); // fold ((A-B)+(B-C)) -> (A-C) - if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB && - N0.getOperand(1) == N1.getOperand(0)) - return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), - N1.getOperand(1)); + if (sd_match(N0, m_Sub(m_Value(A), m_Value(B))) && + sd_match(N1, m_Sub(m_Specific(B), m_Value(C)))) + return DAG.getNode(ISD::SUB, DL, VT, A, C); // fold (A+(B-(A+C))) to (B-C) - if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD && - N0 == N1.getOperand(1).getOperand(0)) - return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), - N1.getOperand(1).getOperand(1)); - // fold (A+(B-(C+A))) to (B-C) - if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD && - N0 == N1.getOperand(1).getOperand(1)) - return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), - N1.getOperand(1).getOperand(0)); + if (sd_match(N1, m_Sub(m_Value(B), m_Add(m_Specific(N0), m_Value(C))))) + return DAG.getNode(ISD::SUB, DL, VT, B, C); // fold (A+((B-A)+or-C)) to (B+or-C) - if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) && - N1.getOperand(0).getOpcode() == ISD::SUB && - N0 == N1.getOperand(0).getOperand(1)) - return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0), - N1.getOperand(1)); + if (sd_match(N1, + m_AnyOf(m_Add(m_Sub(m_Value(B), m_Specific(N0)), m_Value(C)), + m_Sub(m_Sub(m_Value(B), m_Specific(N0)), m_Value(C))))) + return DAG.getNode(N1.getOpcode(), DL, VT, B, C); // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB && -- GitLab From bd72ebd8d1ec3e97ca666623aa628d653a1d54a5 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Tue, 12 Mar 2024 22:05:47 +0530 Subject: [PATCH 0021/1407] AMDGPU: Add some more mfma hazard recognizer tests (#84727) --- .../CodeGen/AMDGPU/mai-hazards-gfx940.mir | 454 ++++++++++++++++++ 1 file changed, 454 insertions(+) diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx940.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx940.mir index 4d307a444b19..a98b02d792d9 100644 --- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx940.mir +++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx940.mir @@ -2028,3 +2028,457 @@ body: | $agpr0_agpr1 = V_MFMA_F64_4X4X4F64_e64 $agpr0_agpr1, $agpr0_agpr1, $agpr0_agpr1, 0, 0, 0, implicit $mode, implicit $exec BUFFER_STORE_DWORDX2_OFFEN_exact $vgpr2_vgpr3, $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec ... + +... +# 2 pass source +# GCN-LABEL: name: xdl_mfma_2pass_write_vgpr_xdl_mfma_read_overlap_srcc +# GCN: V_MFMA +# GCN-NEXT: S_NOP 2 +# GCN-NEXT: V_MFMA +name: xdl_mfma_2pass_write_vgpr_xdl_mfma_read_overlap_srcc +body: | + bb.0: + + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 1, 2, 3, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr2_vgpr3_vgpr4_vgpr5, 1, 2, 3, implicit $mode, implicit $exec + +... + +... +# 2 pass source +# GCN-LABEL: name: xdl_mfma_2pass_write_vgpr_xdl_mfma_read_overlap_srca +# GCN: V_MFMA +# GCN-NEXT: S_NOP 4 +# GCN-NEXT: V_MFMA +name: xdl_mfma_2pass_write_vgpr_xdl_mfma_read_overlap_srca +body: | + bb.0: + + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 1, 2, 3, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, 1, 2, 3, implicit $mode, implicit $exec + +... + +... +# 2 pass source +# GCN-LABEL: name: xdl_mfma_2pass_write_vgpr_xdl_mfma_read_overlap_srcb +# GCN: V_MFMA +# GCN-NEXT: S_NOP 4 +# GCN-NEXT: V_MFMA +name: xdl_mfma_2pass_write_vgpr_xdl_mfma_read_overlap_srcb +body: | + bb.0: + + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 1, 2, 3, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr6_vgpr7, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 1, 2, 3, implicit $mode, implicit $exec + +... + +... +# 4 pass source +# GCN-LABEL: name: xdl_mfma_4pass_write_vgpr_xdl_mfma_read_overlap_srcc +# GCN: V_MFMA +# GCN-NEXT: S_NOP 4 +# GCN-NEXT: V_MFMA +name: xdl_mfma_4pass_write_vgpr_xdl_mfma_read_overlap_srcc +body: | + bb.0: + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 1, 2, 3, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr2_vgpr3_vgpr4_vgpr5, 1, 2, 3, implicit $mode, implicit $exec + +... + +... +# 4 pass source +# GCN-LABEL: name: xdl_mfma_4pass_write_vgpr_xdl_mfma_read_overlap_srca +# GCN: V_MFMA +# GCN-NEXT: S_NOP 6 +# GCN-NEXT: V_MFMA +name: xdl_mfma_4pass_write_vgpr_xdl_mfma_read_overlap_srca +body: | + bb.0: + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 1, 2, 3, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr2_vgpr3, $vgpr10_vgpr11, $vgpr6_vgpr7_vgpr8_vgpr9, 1, 2, 3, implicit $mode, implicit $exec + +... + +... +# 4 pass source +# GCN-LABEL: name: xdl_mfma_4pass_write_vgpr_xdl_mfma_read_overlap_srcb +# GCN: V_MFMA +# GCN-NEXT: S_NOP 6 +# GCN-NEXT: V_MFMA +name: xdl_mfma_4pass_write_vgpr_xdl_mfma_read_overlap_srcb +body: | + bb.0: + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 1, 2, 3, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr10_vgpr11, $vgpr2_vgpr3, $vgpr6_vgpr7_vgpr8_vgpr9, 1, 2, 3, implicit $mode, implicit $exec + +... + +... +# 2 pass source +# GCN-LABEL: name: xdl_mfma_2pass_write_vgpr_sgemm_mfma_read_overlap_srcc +# GCN: V_MFMA +# GCN-NEXT: S_NOP 2 +# GCN-NEXT: V_MFMA +name: xdl_mfma_2pass_write_vgpr_sgemm_mfma_read_overlap_srcc +body: | + bb.0: + + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 1, 2, 3, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr6, $vgpr8, $vgpr2_vgpr3_vgpr4_vgpr5, 0, 0, 0, implicit $mode, implicit $exec + +... + +... +# 2 pass source +# GCN-LABEL: name: xdl_mfma_2pass_write_vgpr_sgemm_mfma_read_overlap_srca +# GCN: V_MFMA +# GCN-NEXT: S_NOP 4 +# GCN-NEXT: V_MFMA +name: xdl_mfma_2pass_write_vgpr_sgemm_mfma_read_overlap_srca +body: | + bb.0: + + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 1, 2, 3, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr1, $vgpr8, $vgpr6_vgpr7_vgpr8_vgpr9, 0, 0, 0, implicit $mode, implicit $exec + +... + +... +# 2 pass source +# GCN-LABEL: name: xdl_mfma_2pass_write_vgpr_sgemm_mfma_read_overlap_srcb +# GCN: V_MFMA +# GCN-NEXT: S_NOP 4 +# GCN-NEXT: V_MFMA +name: xdl_mfma_2pass_write_vgpr_sgemm_mfma_read_overlap_srcb +body: | + bb.0: + + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 1, 2, 3, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr8, $vgpr1, $vgpr6_vgpr7_vgpr8_vgpr9, 0, 0, 0, implicit $mode, implicit $exec + +... + +... +# 4 pass source +# GCN-LABEL: name: xdl_mfma_4pass_write_vgpr_sgemm_mfma_read_overlap_srcc +# GCN: V_MFMA +# GCN-NEXT: S_NOP 4 +# GCN-NEXT: V_MFMA +name: xdl_mfma_4pass_write_vgpr_sgemm_mfma_read_overlap_srcc +body: | + bb.0: + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 1, 2, 3, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr8, $vgpr9, $vgpr2_vgpr3_vgpr4_vgpr5, 0, 0, 0, implicit $mode, implicit $exec + +... + +... +# 4 pass source +# GCN-LABEL: name: xdl_mfma_4pass_write_vgpr_sgemm_mfma_read_overlap_srca +# GCN: V_MFMA +# GCN-NEXT: S_NOP 6 +# GCN-NEXT: V_MFMA +name: xdl_mfma_4pass_write_vgpr_sgemm_mfma_read_overlap_srca +body: | + bb.0: + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 1, 2, 3, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr1, $vgpr8, $vgpr6_vgpr7_vgpr8_vgpr9, 0, 0, 0, implicit $mode, implicit $exec + +... + +... +# 4 pass source +# GCN-LABEL: name: xdl_mfma_4pass_write_vgpr_sgemm_mfma_read_overlap_srcb +# GCN: V_MFMA +# GCN-NEXT: S_NOP 6 +# GCN-NEXT: V_MFMA +name: xdl_mfma_4pass_write_vgpr_sgemm_mfma_read_overlap_srcb +body: | + bb.0: + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, 1, 2, 3, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr8, $vgpr1, $vgpr6_vgpr7_vgpr8_vgpr9, 0, 0, 0, implicit $mode, implicit $exec + +... + +... +# 8 pass source +# GCN-LABEL: name: xdl_mfma_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcc +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 0 +# GCN-NEXT: V_MFMA +name: xdl_mfma_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcc +body: | + bb.0: + renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr0_vgpr1, killed $vgpr2_vgpr3, 1065353216, 0, 0, 0, implicit $mode, implicit $exec + + $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr18, $vgpr19, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec +... + +... +# 8 pass source +# GCN-LABEL: name: xdl_mfma_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srca +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 2 +# GCN-NEXT: V_MFMA +name: xdl_mfma_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srca +body: | + bb.0: + renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr0_vgpr1, killed $vgpr2_vgpr3, 1065353216, 0, 0, 0, implicit $mode, implicit $exec + + $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr0, $vgpr33, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec +... + +... +# 8 pass source +# GCN-LABEL: name: xdl_mfma_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcb +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 2 +# GCN-NEXT: V_MFMA +name: xdl_mfma_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcb +body: | + bb.0: + renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr0_vgpr1, killed $vgpr2_vgpr3, 1065353216, 0, 0, 0, implicit $mode, implicit $exec + + $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr33, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec +... + +... +# 16 pass source +# GCN-LABEL: name: xdl_16pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcc +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 0 +# GCN-NEXT: V_MFMA +name: xdl_16pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcc +body: | + bb.0: + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr126_vgpr127, $vgpr128_vgpr129, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec + + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 killed $vgpr32, killed $vgpr33, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 1, 2, 3, implicit $mode, implicit $exec + +... + +... +# 16 pass source +# GCN-LABEL: name: xdl_16pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srca +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 2 +# GCN-NEXT: V_MFMA +name: xdl_16pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srca +body: | + bb.0: + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr126_vgpr127, $vgpr128_vgpr129, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 killed $vgpr0, killed $vgpr33, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, 1, 2, 3, implicit $mode, implicit $exec + +... + +... +# 16 pass source +# GCN-LABEL: name: xdl_16pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcb +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 2 +# GCN-NEXT: V_MFMA +name: xdl_16pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcb +body: | + bb.0: + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr126_vgpr127, $vgpr128_vgpr129, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 killed $vgpr33, killed $vgpr0, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, 1, 2, 3, implicit $mode, implicit $exec + +... + +... +# 8 pass source +# GCN-LABEL: name: nonxdl_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcc +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: V_MFMA +name: nonxdl_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcc +body: | + bb.0: + $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr18, $vgpr19, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr18, $vgpr19, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec +... + +... +# 8 pass source +# GCN-LABEL: name: nonxdl_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srca +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 1 +# GCN-NEXT: V_MFMA +name: nonxdl_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srca +body: | + bb.0: + $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr18, $vgpr19, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr3, $vgpr19, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec +... + +... +# 8 pass source +# GCN-LABEL: name: nonxdl_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcb +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 1 +# GCN-NEXT: V_MFMA +name: nonxdl_8pass_write_vgpr_nonxdl_sgemm_mfma_read_overlap_srcb +body: | + bb.0: + $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr18, $vgpr19, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr19, $vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec +... +... +# 8 pass source +# GCN-LABEL: name: xdl_mfma_8pass_write_vgpr_xdl_mfma_read_overlap_srcc +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 0 +# GCN-NEXT: V_MFMA +name: xdl_mfma_8pass_write_vgpr_xdl_mfma_read_overlap_srcc +body: | + bb.0: + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr0_vgpr1, killed $vgpr2_vgpr3, 1065353216, 0, 0, 0, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr18_vgpr19, killed $vgpr20_vgpr21, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec +... + +... +# 8 pass source +# GCN-LABEL: name: xdl_mfma_8pass_write_vgpr_xdl_mfma_read_overlap_srca +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 2 +# GCN-NEXT: V_MFMA +name: xdl_mfma_8pass_write_vgpr_xdl_mfma_read_overlap_srca +body: | + bb.0: + renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr0_vgpr1, killed $vgpr2_vgpr3, 1065353216, 0, 0, 0, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr2_vgpr3, killed $vgpr36_vgpr37, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec +... + +... +# 8 pass source +# GCN-LABEL: name: xdl_mfma_8pass_write_vgpr_xdl_mfma_read_overlap_srcb +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 2 +# GCN-NEXT: V_MFMA +name: xdl_mfma_8pass_write_vgpr_xdl_mfma_read_overlap_srcb +body: | + bb.0: + renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr0_vgpr1, killed $vgpr2_vgpr3, 1065353216, 0, 0, 0, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr36_vgpr37, killed $vgpr2_vgpr3, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec +... + +... +# 16 pass source +# GCN-LABEL: name: xdl_16pass_write_vgpr_xdl_mfma_read_overlap_srcc +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 0 +# GCN-NEXT: V_MFMA +name: xdl_16pass_write_vgpr_xdl_mfma_read_overlap_srcc +body: | + bb.0: + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr126_vgpr127, $vgpr128_vgpr129, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr126_vgpr127, $vgpr128_vgpr129, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec + +... + +... +# 16 pass source +# GCN-LABEL: name: xdl_16pass_write_vgpr_xdl_mfma_read_overlap_srca +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 2 +# GCN-NEXT: V_MFMA +name: xdl_16pass_write_vgpr_xdl_mfma_read_overlap_srca +body: | + bb.0: + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr126_vgpr127, $vgpr128_vgpr129, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr2_vgpr3, $vgpr128_vgpr129, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, 0, 0, 0, implicit $mode, implicit $exec + + +... + +... +# 16 pass source +# GCN-LABEL: name: xdl_16pass_write_vgpr_xdl_mfma_read_overlap_srcb +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 2 +# GCN-NEXT: V_MFMA +name: xdl_16pass_write_vgpr_xdl_mfma_read_overlap_srcb +body: | + bb.0: + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr126_vgpr127, $vgpr128_vgpr129, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr128_vgpr129, $vgpr2_vgpr3, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, 0, 0, 0, implicit $mode, implicit $exec + +... + +... +# 2 pass source +# GCN-LABEL: name: xdl_mfma_2pass_write_agpr_smfmac_read_overlap_srcc +# GCN: V_MFMA +# GCN-NEXT: S_NOP 2 +# GCN-NEXT: V_SMFMAC_ +name: xdl_mfma_2pass_write_agpr_smfmac_read_overlap_srcc +body: | + bb.0: + + $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X4F16_e64 $vgpr4_vgpr5, $vgpr6_vgpr7, $agpr0_agpr1_agpr2_agpr3, 1, 2, 3, implicit $mode, implicit $exec + $agpr2_agpr3_agpr4_agpr5 = V_SMFMAC_F32_16X16X32_F16_e64 $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5, $vgpr32, 0, 0, $agpr2_agpr3_agpr4_agpr5, implicit $mode, implicit $exec + +... + +... +# GCN-LABEL: name: xdl_4pass_mfma_write_agpr_smfmac_read_overlap_srcc +# GCN: V_MFMA +# GCN-NEXT: S_NOP 4 +# GCN-NEXT: V_SMFMAC_ +name: xdl_4pass_mfma_write_agpr_smfmac_read_overlap_srcc +body: | + bb.0: + $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_16X16X32I8_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + $agpr2_agpr3_agpr4_agpr5 = V_SMFMAC_F32_16X16X32_F16_e64 $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5, $vgpr32, 0, 0, $agpr2_agpr3_agpr4_agpr5, implicit $mode, implicit $exec + +... + +... +# GCN-LABEL: name: xdl_8pass_mfma_write_agpr_smfmac_read_overlap_srcc +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 0 +# GCN-NEXT: V_SMFMAC_ +name: xdl_8pass_mfma_write_agpr_smfmac_read_overlap_srcc +body: | + bb.0: + renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 killed $vgpr0_vgpr1, killed $vgpr2_vgpr3, 1065353216, 0, 0, 0, implicit $mode, implicit $exec + $agpr2_agpr3_agpr4_agpr5 = V_SMFMAC_F32_16X16X32_F16_e64 $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5, $vgpr32, 0, 0, $agpr2_agpr3_agpr4_agpr5, implicit $mode, implicit $exec +... + +... +# GCN-LABEL: name: xdl_16pass_mfma_write_agpr_smfmac_read_overlap_srcc +# GCN: V_MFMA +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: S_NOP 0 +# GCN-NEXT: V_SMFMAC_ +name: xdl_16pass_mfma_write_agpr_smfmac_read_overlap_srcc +body: | + bb.0: + $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = V_MFMA_F32_32X32X4F16_e64 $vgpr126_vgpr127, $vgpr128_vgpr129, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec + $agpr2_agpr3_agpr4_agpr5 = V_SMFMAC_F32_16X16X32_F16_e64 $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5, $vgpr32, 0, 0, $agpr2_agpr3_agpr4_agpr5, implicit $mode, implicit $exec +... -- GitLab From e09761944cea4aeafadd055b9510ef9f0e9a7338 Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Tue, 12 Mar 2024 18:04:14 +0100 Subject: [PATCH 0022/1407] [libc++] Improves UB handling in ios_base destructor. (#76525) Destroying an ios_base object before it is properly initialized is undefined behavior. Unlike typical C++ classes the initialization is not done in the constructor, but in a dedicated init function. Due to virtual inheritance of the basic_ios object in ostream and friends this undefined behaviour can be triggered when inheriting from classes that can throw in their constructor and inheriting from ostream. Use the __loc_ member of ios_base as sentinel to detect whether the object has or has not been initialized. Addresses https://github.com/llvm/llvm-project/issues/57964 --- libcxx/include/ios | 12 ++- libcxx/src/ios.cpp | 4 + .../ios.base.cons/dtor.uninitialized.pass.cpp | 80 +++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 libcxx/test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp diff --git a/libcxx/include/ios b/libcxx/include/ios index 4b1306fc2ad8..00c1d5c2d4bc 100644 --- a/libcxx/include/ios +++ b/libcxx/include/ios @@ -359,7 +359,13 @@ public: } protected: - _LIBCPP_HIDE_FROM_ABI ios_base() { // purposefully does no initialization + _LIBCPP_HIDE_FROM_ABI ios_base() : __loc_(nullptr) { + // Purposefully does no initialization + // + // Except for the locale, this is a sentinel to avoid destroying + // an uninitialized object. See + // test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp + // for the details. } void init(void* __sb); @@ -571,7 +577,9 @@ public: _LIBCPP_HIDE_FROM_ABI char_type widen(char __c) const; protected: - _LIBCPP_HIDE_FROM_ABI basic_ios() { // purposefully does no initialization + _LIBCPP_HIDE_FROM_ABI basic_ios() { + // purposefully does no initialization + // since the destructor does nothing this does not have ios_base issues. } _LIBCPP_HIDE_FROM_ABI void init(basic_streambuf* __sb); diff --git a/libcxx/src/ios.cpp b/libcxx/src/ios.cpp index d58827fa1255..a727855c4655 100644 --- a/libcxx/src/ios.cpp +++ b/libcxx/src/ios.cpp @@ -195,6 +195,10 @@ void ios_base::register_callback(event_callback fn, int index) { } ios_base::~ios_base() { + // Avoid UB when not properly initialized. See ios_base::ios_base for + // more information. + if (!__loc_) + return; __call_callbacks(erase_event); locale& loc_storage = *reinterpret_cast(&__loc_); loc_storage.~locale(); diff --git a/libcxx/test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp b/libcxx/test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp new file mode 100644 index 000000000000..ea42203d0544 --- /dev/null +++ b/libcxx/test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp @@ -0,0 +1,80 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: no-exceptions + +// The fix for issue 57964 requires an updated dylib due to explicit +// instantiations. That means Apple backdeployment targets remain broken. +// UNSUPPORTED: using-built-library-before-llvm-19 + +// + +// class ios_base + +// ~ios_base() +// +// Destroying a constructed ios_base object that has not been +// initialized by basic_ios::init is undefined behaviour. This can +// happen in practice, make sure the undefined behaviour is handled +// gracefully. +// +// +// [ios.base.cons]/1 +// +// ios_base(); +// Effects: Each ios_base member has an indeterminate value after construction. +// The object's members shall be initialized by calling basic_ios::init before +// the object's first use or before it is destroyed, whichever comes first; +// otherwise the behavior is undefined. +// +// [basic.ios.cons]/2 +// +// basic_ios(); +// Effects: Leaves its member objects uninitialized. The object shall be +// initialized by calling basic_ios::init before its first use or before it is +// destroyed, whichever comes first; otherwise the behavior is undefined. +// +// ostream and friends have a basic_ios virtual base. +// [class.base.init]/13 +// In a non-delegating constructor, initialization proceeds in the +// following order: +// - First, and only for the constructor of the most derived class +// ([intro.object]), virtual base classes are initialized ... +// +// So in this example +// struct Foo : AlwaysThrows, std::ostream { +// Foo() : AlwaysThrows{}, std::ostream{nullptr} {} +// }; +// +// Here +// - the ios_base object is constructed +// - the AlwaysThrows object is constructed and throws an exception +// - the AlwaysThrows object is destrodyed +// - the ios_base object is destroyed +// +// The ios_base object is destroyed before it has been initialized and runs +// into undefined behavior. By using __loc_ as a sentinel we can avoid +// accessing uninitialized memory in the destructor. + +#include + +struct AlwaysThrows { + AlwaysThrows() { throw 1; } +}; + +struct Foo : AlwaysThrows, std::ostream { + Foo() : AlwaysThrows(), std::ostream(nullptr) {} +}; + +int main(int, char**) { + try { + Foo foo; + } catch (...) { + }; + return 0; +} -- GitLab From 683a9ac803a56f6dda9b783a6e2d6d92a5d0626c Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Tue, 12 Mar 2024 17:10:16 +0000 Subject: [PATCH 0023/1407] [X86] combineVectorPack - use APInt::truncSSat for PACKSS constant folding. NFC. Unfortunately PACKUS can't use APInt::truncUSat --- llvm/lib/Target/X86/X86ISelLowering.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index b4d0421c14c0..72b45d462dfe 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -47630,16 +47630,12 @@ static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG, // PACKSS: Truncate signed value with signed saturation. // Source values less than dst minint are saturated to minint. // Source values greater than dst maxint are saturated to maxint. - if (Val.isSignedIntN(DstBitsPerElt)) - Val = Val.trunc(DstBitsPerElt); - else if (Val.isNegative()) - Val = APInt::getSignedMinValue(DstBitsPerElt); - else - Val = APInt::getSignedMaxValue(DstBitsPerElt); + Val = Val.truncSSat(DstBitsPerElt); } else { // PACKUS: Truncate signed value with unsigned saturation. // Source values less than zero are saturated to zero. // Source values greater than dst maxuint are saturated to maxuint. + // NOTE: This is different from APInt::truncUSat. if (Val.isIntN(DstBitsPerElt)) Val = Val.trunc(DstBitsPerElt); else if (Val.isNegative()) -- GitLab From 7bee91fadf8db90f71b458aaff4de0efa7dc23a0 Mon Sep 17 00:00:00 2001 From: "Diego A. Estrada Rivera" Date: Tue, 12 Mar 2024 13:21:31 -0400 Subject: [PATCH 0024/1407] [analyzer][NFC] Turn NodeBuilderContext into a class (#84638) From issue #73088. I changed `NodeBuilderContext` into a class. Additionally, there were some other mentions of the former being a struct which I also changed into a class. This is my first time working with an issue so I will be open to hearing any advice or changes that need to be done. --- .../clang/StaticAnalyzer/Core/CheckerManager.h | 2 +- .../StaticAnalyzer/Core/PathSensitive/CoreEngine.h | 12 +++++++++--- .../StaticAnalyzer/Core/PathSensitive/ExprEngine.h | 2 +- clang/lib/StaticAnalyzer/Core/CoreEngine.cpp | 6 +++--- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h b/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h index a45ba1bc573e..ad25d18f2807 100644 --- a/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h +++ b/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h @@ -49,7 +49,7 @@ class ExplodedNodeSet; class ExprEngine; struct EvalCallOptions; class MemRegion; -struct NodeBuilderContext; +class NodeBuilderContext; class ObjCMethodCall; class RegionAndSymbolInvalidationTraits; class SVal; diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h index 8e392421fef9..0ef353bf9731 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h @@ -59,7 +59,7 @@ class CoreEngine { friend class ExprEngine; friend class IndirectGotoNodeBuilder; friend class NodeBuilder; - friend struct NodeBuilderContext; + friend class NodeBuilderContext; friend class SwitchNodeBuilder; public: @@ -193,12 +193,12 @@ public: DataTag::Factory &getDataTags() { return DataTags; } }; -// TODO: Turn into a class. -struct NodeBuilderContext { +class NodeBuilderContext { const CoreEngine &Eng; const CFGBlock *Block; const LocationContext *LC; +public: NodeBuilderContext(const CoreEngine &E, const CFGBlock *B, const LocationContext *L) : Eng(E), Block(B), LC(L) { @@ -208,9 +208,15 @@ struct NodeBuilderContext { NodeBuilderContext(const CoreEngine &E, const CFGBlock *B, ExplodedNode *N) : NodeBuilderContext(E, B, N->getLocationContext()) {} + /// Return the CoreEngine associated with this builder. + const CoreEngine &getEngine() const { return Eng; } + /// Return the CFGBlock associated with this builder. const CFGBlock *getBlock() const { return Block; } + /// Return the location context associated with this builder. + const LocationContext *getLocationContext() const { return LC; } + /// Returns the number of times the current basic block has been /// visited on the exploded graph path. unsigned blockCount() const { diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h index f7894fb83ce6..859c1497d7e6 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h @@ -85,7 +85,7 @@ class ExplodedNodeSet; class ExplodedNode; class IndirectGotoNodeBuilder; class MemRegion; -struct NodeBuilderContext; +class NodeBuilderContext; class NodeBuilderWithSinks; class ProgramState; class ProgramStateManager; diff --git a/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp b/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp index 141d0cb320bf..8605fa149e4f 100644 --- a/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp +++ b/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp @@ -625,8 +625,8 @@ ExplodedNode* NodeBuilder::generateNodeImpl(const ProgramPoint &Loc, bool MarkAsSink) { HasGeneratedNodes = true; bool IsNew; - ExplodedNode *N = C.Eng.G.getNode(Loc, State, MarkAsSink, &IsNew); - N->addPredecessor(FromN, C.Eng.G); + ExplodedNode *N = C.getEngine().G.getNode(Loc, State, MarkAsSink, &IsNew); + N->addPredecessor(FromN, C.getEngine().G); Frontier.erase(FromN); if (!IsNew) @@ -655,7 +655,7 @@ ExplodedNode *BranchNodeBuilder::generateNode(ProgramStateRef State, if (!isFeasible(branch)) return nullptr; - ProgramPoint Loc = BlockEdge(C.Block, branch ? DstT:DstF, + ProgramPoint Loc = BlockEdge(C.getBlock(), branch ? DstT : DstF, NodePred->getLocationContext()); ExplodedNode *Succ = generateNodeImpl(Loc, State, NodePred); return Succ; -- GitLab From 93503aafcdc66837ecf220243aaa530c05c35895 Mon Sep 17 00:00:00 2001 From: Hiroshi Yamauchi <56735936+hjyamauchi@users.noreply.github.com> Date: Tue, 12 Mar 2024 10:26:44 -0700 Subject: [PATCH 0025/1407] Fix MSVC build issues (#84362) MSVC fails when there is ambiguity (multiple options) around implicit type conversion operators. Make ConstString's conversion operator to string_view explicit to avoid ambiguity with one to StringRef and remove an unused local variable that MSVC also fails on. --- lldb/include/lldb/Utility/ConstString.h | 4 ++-- lldb/source/Core/Mangled.cpp | 12 +++++++----- lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp | 1 - 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lldb/include/lldb/Utility/ConstString.h b/lldb/include/lldb/Utility/ConstString.h index 470a554ca048..f7f7ec7605eb 100644 --- a/lldb/include/lldb/Utility/ConstString.h +++ b/lldb/include/lldb/Utility/ConstString.h @@ -168,8 +168,8 @@ public: // Implicitly convert \class ConstString instances to \class StringRef. operator llvm::StringRef() const { return GetStringRef(); } - // Implicitly convert \class ConstString instances to \class std::string_view. - operator std::string_view() const { + // Explicitly convert \class ConstString instances to \class std::string_view. + explicit operator std::string_view() const { return std::string_view(m_string, GetLength()); } diff --git a/lldb/source/Core/Mangled.cpp b/lldb/source/Core/Mangled.cpp index 23ae3913093f..b167c51fdce2 100644 --- a/lldb/source/Core/Mangled.cpp +++ b/lldb/source/Core/Mangled.cpp @@ -125,7 +125,7 @@ void Mangled::SetValue(ConstString name) { } // Local helpers for different demangling implementations. -static char *GetMSVCDemangledStr(std::string_view M) { +static char *GetMSVCDemangledStr(llvm::StringRef M) { char *demangled_cstr = llvm::microsoftDemangle( M, nullptr, nullptr, llvm::MSDemangleFlags( @@ -169,27 +169,29 @@ static char *GetItaniumDemangledStr(const char *M) { return demangled_cstr; } -static char *GetRustV0DemangledStr(std::string_view M) { +static char *GetRustV0DemangledStr(llvm::StringRef M) { char *demangled_cstr = llvm::rustDemangle(M); if (Log *log = GetLog(LLDBLog::Demangle)) { if (demangled_cstr && demangled_cstr[0]) LLDB_LOG(log, "demangled rustv0: {0} -> \"{1}\"", M, demangled_cstr); else - LLDB_LOG(log, "demangled rustv0: {0} -> error: failed to demangle", M); + LLDB_LOG(log, "demangled rustv0: {0} -> error: failed to demangle", + static_cast(M)); } return demangled_cstr; } -static char *GetDLangDemangledStr(std::string_view M) { +static char *GetDLangDemangledStr(llvm::StringRef M) { char *demangled_cstr = llvm::dlangDemangle(M); if (Log *log = GetLog(LLDBLog::Demangle)) { if (demangled_cstr && demangled_cstr[0]) LLDB_LOG(log, "demangled dlang: {0} -> \"{1}\"", M, demangled_cstr); else - LLDB_LOG(log, "demangled dlang: {0} -> error: failed to demangle", M); + LLDB_LOG(log, "demangled dlang: {0} -> error: failed to demangle", + static_cast(M)); } return demangled_cstr; diff --git a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp index 6a2ea8c4a41b..f237dd63ab1c 100644 --- a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp +++ b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp @@ -527,7 +527,6 @@ TEST_F(SymbolFilePDBTests, TestTypedefs) { SymbolFilePDB *symfile = static_cast(module->GetSymbolFile()); llvm::pdb::IPDBSession &session = symfile->GetPDBSession(); - TypeMap results; const char *TypedefsToCheck[] = {"ClassTypedef", "NSClassTypedef", "FuncPointerTypedef", -- GitLab From c4e517f59c086eafe2eb61d23197820f05be799c Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Tue, 12 Mar 2024 10:30:39 -0700 Subject: [PATCH 0026/1407] [AMDGPU] Adding the amdgpu_num_work_groups function attribute (#79035) A new function attribute named amdgpu_num_work_groups is added. This attribute, which consists of three integers, allows programmers to let the compiler know the number of workgroups to be launched in each of the three dimensions and do optimizations based on that information. --------- Co-authored-by: Jun Wang --- clang/docs/ReleaseNotes.rst | 6 + clang/include/clang/Basic/Attr.td | 7 + clang/include/clang/Basic/AttrDocs.td | 27 ++++ clang/include/clang/Sema/Sema.h | 10 ++ clang/lib/CodeGen/Targets/AMDGPU.cpp | 23 +++ clang/lib/Sema/SemaDeclAttr.cpp | 62 ++++++++ .../lib/Sema/SemaTemplateInstantiateDecl.cpp | 29 ++++ clang/test/CodeGenCUDA/amdgpu-kernel-attrs.cu | 35 +++++ clang/test/CodeGenOpenCL/amdgpu-attrs.cl | 47 +++++++ ...a-attribute-supported-attributes-list.test | 1 + clang/test/SemaCUDA/amdgpu-attrs.cu | 132 ++++++++++++++++++ llvm/docs/AMDGPUUsage.rst | 10 ++ .../AMDGPU/AMDGPUHSAMetadataStreamer.cpp | 8 ++ llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp | 7 +- llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h | 3 + .../Target/AMDGPU/SIMachineFunctionInfo.cpp | 2 + .../lib/Target/AMDGPU/SIMachineFunctionInfo.h | 10 ++ .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 37 +++++ llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 18 +++ .../AMDGPU/attr-amdgpu-num-workgroups.ll | 84 +++++++++++ .../attr-amdgpu-num-workgroups_error_check.ll | 71 ++++++++++ 21 files changed, 628 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-workgroups.ll create mode 100644 llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-workgroups_error_check.ll diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 2842b63197ff..e14c92eae0af 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -194,6 +194,12 @@ Removed Compiler Flags Attribute Changes in Clang -------------------------- +- Introduced a new function attribute ``__attribute__((amdgpu_max_num_work_groups(x, y, z)))`` or + ``[[clang::amdgpu_max_num_work_groups(x, y, z)]]`` for the AMDGPU target. This attribute can be + attached to HIP or OpenCL kernel function definitions to provide an optimization hint. The parameters + ``x``, ``y``, and ``z`` specify the maximum number of workgroups for the respective dimensions, + and each must be a positive integer when provided. The parameter ``x`` is required, while ``y`` and + ``z`` are optional with default value of 1. Improvements to Clang's diagnostics ----------------------------------- diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index 080340669b60..63efd85dcd4e 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -2054,6 +2054,13 @@ def AMDGPUNumVGPR : InheritableAttr { let Subjects = SubjectList<[Function], ErrorDiag, "kernel functions">; } +def AMDGPUMaxNumWorkGroups : InheritableAttr { + let Spellings = [Clang<"amdgpu_max_num_work_groups", 0>]; + let Args = [ExprArgument<"MaxNumWorkGroupsX">, ExprArgument<"MaxNumWorkGroupsY", 1>, ExprArgument<"MaxNumWorkGroupsZ", 1>]; + let Documentation = [AMDGPUMaxNumWorkGroupsDocs]; + let Subjects = SubjectList<[Function], ErrorDiag, "kernel functions">; +} + def AMDGPUKernelCall : DeclOrTypeAttr { let Spellings = [Clang<"amdgpu_kernel">]; let Documentation = [Undocumented]; diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td index 2c07cd09b0d5..d61f96ade557 100644 --- a/clang/include/clang/Basic/AttrDocs.td +++ b/clang/include/clang/Basic/AttrDocs.td @@ -2741,6 +2741,33 @@ An error will be given if: }]; } +def AMDGPUMaxNumWorkGroupsDocs : Documentation { + let Category = DocCatAMDGPUAttributes; + let Content = [{ +This attribute specifies the max number of work groups when the kernel +is dispatched. + +Clang supports the +``__attribute__((amdgpu_max_num_work_groups(, , )))`` or +``[[clang::amdgpu_max_num_work_groups(, , )]]`` attribute for the +AMDGPU target. This attribute may be attached to HIP or OpenCL kernel function +definitions and is an optimization hint. + +The ```` parameter specifies the maximum number of work groups in the x dimension. +Similarly ```` and ```` are for the y and z dimensions respectively. +Each of the three values must be greater than 0 when provided. The ```` parameter +is required, while ```` and ```` are optional with default value of 1. + +If specified, the AMDGPU target backend might be able to produce better machine +code. + +An error will be given if: + - Specified values violate subtarget specifications; + - Specified values are not compatible with values provided through other + attributes. + }]; +} + def DocCatCallingConvs : DocumentationCategory<"Calling Conventions"> { let Content = [{ Clang supports several different calling conventions, depending on the target diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 267c79cc057c..b226851f0303 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -3911,6 +3911,16 @@ public: void addAMDGPUWavesPerEUAttr(Decl *D, const AttributeCommonInfo &CI, Expr *Min, Expr *Max); + /// Create an AMDGPUMaxNumWorkGroupsAttr attribute. + AMDGPUMaxNumWorkGroupsAttr * + CreateAMDGPUMaxNumWorkGroupsAttr(const AttributeCommonInfo &CI, Expr *XExpr, + Expr *YExpr, Expr *ZExpr); + + /// addAMDGPUMaxNumWorkGroupsAttr - Adds an amdgpu_max_num_work_groups + /// attribute to a particular declaration. + void addAMDGPUMaxNumWorkGroupsAttr(Decl *D, const AttributeCommonInfo &CI, + Expr *XExpr, Expr *YExpr, Expr *ZExpr); + DLLImportAttr *mergeDLLImportAttr(Decl *D, const AttributeCommonInfo &CI); DLLExportAttr *mergeDLLExportAttr(Decl *D, const AttributeCommonInfo &CI); MSInheritanceAttr *mergeMSInheritanceAttr(Decl *D, diff --git a/clang/lib/CodeGen/Targets/AMDGPU.cpp b/clang/lib/CodeGen/Targets/AMDGPU.cpp index 03ac6b78598f..44e86c0b40f6 100644 --- a/clang/lib/CodeGen/Targets/AMDGPU.cpp +++ b/clang/lib/CodeGen/Targets/AMDGPU.cpp @@ -356,6 +356,29 @@ void AMDGPUTargetCodeGenInfo::setFunctionDeclAttributes( if (NumVGPR != 0) F->addFnAttr("amdgpu-num-vgpr", llvm::utostr(NumVGPR)); } + + if (const auto *Attr = FD->getAttr()) { + uint32_t X = Attr->getMaxNumWorkGroupsX() + ->EvaluateKnownConstInt(M.getContext()) + .getExtValue(); + // Y and Z dimensions default to 1 if not specified + uint32_t Y = Attr->getMaxNumWorkGroupsY() + ? Attr->getMaxNumWorkGroupsY() + ->EvaluateKnownConstInt(M.getContext()) + .getExtValue() + : 1; + uint32_t Z = Attr->getMaxNumWorkGroupsZ() + ? Attr->getMaxNumWorkGroupsZ() + ->EvaluateKnownConstInt(M.getContext()) + .getExtValue() + : 1; + + llvm::SmallString<32> AttrVal; + llvm::raw_svector_ostream OS(AttrVal); + OS << X << ',' << Y << ',' << Z; + + F->addFnAttr("amdgpu-max-num-workgroups", AttrVal.str()); + } } /// Emits control constants used to change per-architecture behaviour in the diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index c00120b59d39..e3da3e606435 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -8079,6 +8079,65 @@ static void handleAMDGPUNumVGPRAttr(Sema &S, Decl *D, const ParsedAttr &AL) { D->addAttr(::new (S.Context) AMDGPUNumVGPRAttr(S.Context, AL, NumVGPR)); } +static bool +checkAMDGPUMaxNumWorkGroupsArguments(Sema &S, Expr *XExpr, Expr *YExpr, + Expr *ZExpr, + const AMDGPUMaxNumWorkGroupsAttr &Attr) { + if (S.DiagnoseUnexpandedParameterPack(XExpr) || + (YExpr && S.DiagnoseUnexpandedParameterPack(YExpr)) || + (ZExpr && S.DiagnoseUnexpandedParameterPack(ZExpr))) + return true; + + // Accept template arguments for now as they depend on something else. + // We'll get to check them when they eventually get instantiated. + if (XExpr->isValueDependent() || (YExpr && YExpr->isValueDependent()) || + (ZExpr && ZExpr->isValueDependent())) + return false; + + uint32_t NumWG = 0; + Expr *Exprs[3] = {XExpr, YExpr, ZExpr}; + for (int i = 0; i < 3; i++) { + if (Exprs[i]) { + if (!checkUInt32Argument(S, Attr, Exprs[i], NumWG, i, + /*StrictlyUnsigned=*/true)) + return true; + if (NumWG == 0) { + S.Diag(Attr.getLoc(), diag::err_attribute_argument_is_zero) + << &Attr << Exprs[i]->getSourceRange(); + return true; + } + } + } + + return false; +} + +AMDGPUMaxNumWorkGroupsAttr * +Sema::CreateAMDGPUMaxNumWorkGroupsAttr(const AttributeCommonInfo &CI, + Expr *XExpr, Expr *YExpr, Expr *ZExpr) { + AMDGPUMaxNumWorkGroupsAttr TmpAttr(Context, CI, XExpr, YExpr, ZExpr); + + if (checkAMDGPUMaxNumWorkGroupsArguments(*this, XExpr, YExpr, ZExpr, TmpAttr)) + return nullptr; + + return ::new (Context) + AMDGPUMaxNumWorkGroupsAttr(Context, CI, XExpr, YExpr, ZExpr); +} + +void Sema::addAMDGPUMaxNumWorkGroupsAttr(Decl *D, const AttributeCommonInfo &CI, + Expr *XExpr, Expr *YExpr, + Expr *ZExpr) { + if (auto *Attr = CreateAMDGPUMaxNumWorkGroupsAttr(CI, XExpr, YExpr, ZExpr)) + D->addAttr(Attr); +} + +static void handleAMDGPUMaxNumWorkGroupsAttr(Sema &S, Decl *D, + const ParsedAttr &AL) { + Expr *YExpr = (AL.getNumArgs() > 1) ? AL.getArgAsExpr(1) : nullptr; + Expr *ZExpr = (AL.getNumArgs() > 2) ? AL.getArgAsExpr(2) : nullptr; + S.addAMDGPUMaxNumWorkGroupsAttr(D, AL, AL.getArgAsExpr(0), YExpr, ZExpr); +} + static void handleX86ForceAlignArgPointerAttr(Sema &S, Decl *D, const ParsedAttr &AL) { // If we try to apply it to a function pointer, don't warn, but don't @@ -9183,6 +9242,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL, case ParsedAttr::AT_AMDGPUNumVGPR: handleAMDGPUNumVGPRAttr(S, D, AL); break; + case ParsedAttr::AT_AMDGPUMaxNumWorkGroups: + handleAMDGPUMaxNumWorkGroupsAttr(S, D, AL); + break; case ParsedAttr::AT_AVRSignal: handleAVRSignalAttr(S, D, AL); break; diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp index 20c2c93ac9c7..8ef8bfdf2a7b 100644 --- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp +++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp @@ -607,6 +607,29 @@ static void instantiateDependentAMDGPUWavesPerEUAttr( S.addAMDGPUWavesPerEUAttr(New, Attr, MinExpr, MaxExpr); } +static void instantiateDependentAMDGPUMaxNumWorkGroupsAttr( + Sema &S, const MultiLevelTemplateArgumentList &TemplateArgs, + const AMDGPUMaxNumWorkGroupsAttr &Attr, Decl *New) { + EnterExpressionEvaluationContext Unevaluated( + S, Sema::ExpressionEvaluationContext::ConstantEvaluated); + + ExprResult ResultX = S.SubstExpr(Attr.getMaxNumWorkGroupsX(), TemplateArgs); + if (!ResultX.isUsable()) + return; + ExprResult ResultY = S.SubstExpr(Attr.getMaxNumWorkGroupsY(), TemplateArgs); + if (!ResultY.isUsable()) + return; + ExprResult ResultZ = S.SubstExpr(Attr.getMaxNumWorkGroupsZ(), TemplateArgs); + if (!ResultZ.isUsable()) + return; + + Expr *XExpr = ResultX.getAs(); + Expr *YExpr = ResultY.getAs(); + Expr *ZExpr = ResultZ.getAs(); + + S.addAMDGPUMaxNumWorkGroupsAttr(New, Attr, XExpr, YExpr, ZExpr); +} + // This doesn't take any template parameters, but we have a custom action that // needs to happen when the kernel itself is instantiated. We need to run the // ItaniumMangler to mark the names required to name this kernel. @@ -792,6 +815,12 @@ void Sema::InstantiateAttrs(const MultiLevelTemplateArgumentList &TemplateArgs, *AMDGPUFlatWorkGroupSize, New); } + if (const auto *AMDGPUMaxNumWorkGroups = + dyn_cast(TmplAttr)) { + instantiateDependentAMDGPUMaxNumWorkGroupsAttr( + *this, TemplateArgs, *AMDGPUMaxNumWorkGroups, New); + } + if (const auto *ParamAttr = dyn_cast(TmplAttr)) { instantiateDependentHLSLParamModifierAttr(*this, TemplateArgs, ParamAttr, New); diff --git a/clang/test/CodeGenCUDA/amdgpu-kernel-attrs.cu b/clang/test/CodeGenCUDA/amdgpu-kernel-attrs.cu index a1642421af2c..11a133fd1351 100644 --- a/clang/test/CodeGenCUDA/amdgpu-kernel-attrs.cu +++ b/clang/test/CodeGenCUDA/amdgpu-kernel-attrs.cu @@ -40,12 +40,45 @@ __attribute__((amdgpu_num_vgpr(64))) // expected-no-diagnostics __global__ void num_vgpr_64() { // CHECK: define{{.*}} amdgpu_kernel void @_Z11num_vgpr_64v() [[NUM_VGPR_64:#[0-9]+]] } +__attribute__((amdgpu_max_num_work_groups(32, 4, 2))) // expected-no-diagnostics +__global__ void max_num_work_groups_32_4_2() { +// CHECK: define{{.*}} amdgpu_kernel void @_Z26max_num_work_groups_32_4_2v() [[MAX_NUM_WORK_GROUPS_32_4_2:#[0-9]+]] +} +__attribute__((amdgpu_max_num_work_groups(32))) // expected-no-diagnostics +__global__ void max_num_work_groups_32() { +// CHECK: define{{.*}} amdgpu_kernel void @_Z22max_num_work_groups_32v() [[MAX_NUM_WORK_GROUPS_32_1_1:#[0-9]+]] +} +__attribute__((amdgpu_max_num_work_groups(32,1))) // expected-no-diagnostics +__global__ void max_num_work_groups_32_1() { +// CHECK: define{{.*}} amdgpu_kernel void @_Z24max_num_work_groups_32_1v() [[MAX_NUM_WORK_GROUPS_32_1_1:#[0-9]+]] +} + + + +template +__attribute__((amdgpu_max_num_work_groups(a, 4, 2))) +__global__ void template_a_4_2_max_num_work_groups() {} +template __global__ void template_a_4_2_max_num_work_groups<32>(); +// CHECK: define{{.*}} amdgpu_kernel void @_Z34template_a_4_2_max_num_work_groupsILj32EEvv() [[MAX_NUM_WORK_GROUPS_32_4_2:#[0-9]+]] + +template +__attribute__((amdgpu_max_num_work_groups(32, a, 2))) +__global__ void template_32_a_2_max_num_work_groups() {} +template __global__ void template_32_a_2_max_num_work_groups<4>(); +// CHECK: define{{.*}} amdgpu_kernel void @_Z35template_32_a_2_max_num_work_groupsILj4EEvv() [[MAX_NUM_WORK_GROUPS_32_4_2:#[0-9]+]] + +template +__attribute__((amdgpu_max_num_work_groups(32, 4, a))) +__global__ void template_32_4_a_max_num_work_groups() {} +template __global__ void template_32_4_a_max_num_work_groups<2>(); +// CHECK: define{{.*}} amdgpu_kernel void @_Z35template_32_4_a_max_num_work_groupsILj2EEvv() [[MAX_NUM_WORK_GROUPS_32_4_2:#[0-9]+]] // Make sure this is silently accepted on other targets. // NAMD-NOT: "amdgpu-flat-work-group-size" // NAMD-NOT: "amdgpu-waves-per-eu" // NAMD-NOT: "amdgpu-num-vgpr" // NAMD-NOT: "amdgpu-num-sgpr" +// NAMD-NOT: "amdgpu-max-num-work-groups" // DEFAULT-DAG: attributes [[FLAT_WORK_GROUP_SIZE_DEFAULT]] = {{.*}}"amdgpu-flat-work-group-size"="1,1024"{{.*}}"uniform-work-group-size"="true" // MAX1024-DAG: attributes [[FLAT_WORK_GROUP_SIZE_DEFAULT]] = {{.*}}"amdgpu-flat-work-group-size"="1,1024" @@ -53,5 +86,7 @@ __global__ void num_vgpr_64() { // CHECK-DAG: attributes [[WAVES_PER_EU_2]] = {{.*}}"amdgpu-waves-per-eu"="2" // CHECK-DAG: attributes [[NUM_SGPR_32]] = {{.*}}"amdgpu-num-sgpr"="32" // CHECK-DAG: attributes [[NUM_VGPR_64]] = {{.*}}"amdgpu-num-vgpr"="64" +// CHECK-DAG: attributes [[MAX_NUM_WORK_GROUPS_32_4_2]] = {{.*}}"amdgpu-max-num-workgroups"="32,4,2" +// CHECK-DAG: attributes [[MAX_NUM_WORK_GROUPS_32_1_1]] = {{.*}}"amdgpu-max-num-workgroups"="32,1,1" // NOUB-NOT: "uniform-work-group-size"="true" diff --git a/clang/test/CodeGenOpenCL/amdgpu-attrs.cl b/clang/test/CodeGenOpenCL/amdgpu-attrs.cl index b0dfc97b53b2..5648bc13458e 100644 --- a/clang/test/CodeGenOpenCL/amdgpu-attrs.cl +++ b/clang/test/CodeGenOpenCL/amdgpu-attrs.cl @@ -139,6 +139,46 @@ kernel void reqd_work_group_size_32_2_1_flat_work_group_size_16_128() { // CHECK: define{{.*}} amdgpu_kernel void @reqd_work_group_size_32_2_1_flat_work_group_size_16_128() [[FLAT_WORK_GROUP_SIZE_16_128:#[0-9]+]] } +__attribute__((amdgpu_max_num_work_groups(1, 1, 1))) // expected-no-diagnostics +kernel void max_num_work_groups_1_1_1() { +// CHECK: define{{.*}} amdgpu_kernel void @max_num_work_groups_1_1_1() [[MAX_NUM_WORK_GROUPS_1_1_1:#[0-9]+]] +} + +__attribute__((amdgpu_max_num_work_groups(32, 1, 1))) // expected-no-diagnostics +kernel void max_num_work_groups_32_1_1() { +// CHECK: define{{.*}} amdgpu_kernel void @max_num_work_groups_32_1_1() [[MAX_NUM_WORK_GROUPS_32_1_1:#[0-9]+]] +} + +__attribute__((amdgpu_max_num_work_groups(32, 8, 1))) // expected-no-diagnostics +kernel void max_num_work_groups_32_8_1() { +// CHECK: define{{.*}} amdgpu_kernel void @max_num_work_groups_32_8_1() [[MAX_NUM_WORK_GROUPS_32_8_1:#[0-9]+]] +} + +__attribute__((amdgpu_max_num_work_groups(1, 1, 32))) // expected-no-diagnostics +kernel void max_num_work_groups_1_1_32() { +// CHECK: define{{.*}} amdgpu_kernel void @max_num_work_groups_1_1_32() [[MAX_NUM_WORK_GROUPS_1_1_32:#[0-9]+]] +} + +__attribute__((amdgpu_max_num_work_groups(1, 8, 32))) // expected-no-diagnostics +kernel void max_num_work_groups_1_8_32() { +// CHECK: define{{.*}} amdgpu_kernel void @max_num_work_groups_1_8_32() [[MAX_NUM_WORK_GROUPS_1_8_32:#[0-9]+]] +} + +__attribute__((amdgpu_max_num_work_groups(4, 8, 32))) // expected-no-diagnostics +kernel void max_num_work_groups_4_8_32() { +// CHECK: define{{.*}} amdgpu_kernel void @max_num_work_groups_4_8_32() [[MAX_NUM_WORK_GROUPS_4_8_32:#[0-9]+]] +} + +__attribute__((amdgpu_max_num_work_groups(32))) // expected-no-diagnostics +kernel void max_num_work_groups_32() { +// CHECK: define{{.*}} amdgpu_kernel void @max_num_work_groups_32() [[MAX_NUM_WORK_GROUPS_32_1_1:#[0-9]+]] +} + +__attribute__((amdgpu_max_num_work_groups(32,1))) // expected-no-diagnostics +kernel void max_num_work_groups_32_1() { +// CHECK: define{{.*}} amdgpu_kernel void @max_num_work_groups_32_1() [[MAX_NUM_WORK_GROUPS_32_1_1:#[0-9]+]] +} + void a_function() { // CHECK: define{{.*}} void @a_function() [[A_FUNCTION:#[0-9]+]] } @@ -189,5 +229,12 @@ kernel void default_kernel() { // CHECK-DAG: attributes [[FLAT_WORK_GROUP_SIZE_32_64_WAVES_PER_EU_2_NUM_SGPR_32_NUM_VGPR_64]] = {{.*}} "amdgpu-flat-work-group-size"="32,64" "amdgpu-num-sgpr"="32" "amdgpu-num-vgpr"="64" "amdgpu-waves-per-eu"="2" // CHECK-DAG: attributes [[FLAT_WORK_GROUP_SIZE_32_64_WAVES_PER_EU_2_4_NUM_SGPR_32_NUM_VGPR_64]] = {{.*}} "amdgpu-flat-work-group-size"="32,64" "amdgpu-num-sgpr"="32" "amdgpu-num-vgpr"="64" "amdgpu-waves-per-eu"="2,4" +// CHECK-DAG: attributes [[MAX_NUM_WORK_GROUPS_1_1_1]] = {{.*}} "amdgpu-max-num-workgroups"="1,1,1" +// CHECK-DAG: attributes [[MAX_NUM_WORK_GROUPS_32_1_1]] = {{.*}} "amdgpu-max-num-workgroups"="32,1,1" +// CHECK-DAG: attributes [[MAX_NUM_WORK_GROUPS_32_8_1]] = {{.*}} "amdgpu-max-num-workgroups"="32,8,1" +// CHECK-DAG: attributes [[MAX_NUM_WORK_GROUPS_1_1_32]] = {{.*}} "amdgpu-max-num-workgroups"="1,1,32" +// CHECK-DAG: attributes [[MAX_NUM_WORK_GROUPS_1_8_32]] = {{.*}} "amdgpu-max-num-workgroups"="1,8,32" +// CHECK-DAG: attributes [[MAX_NUM_WORK_GROUPS_4_8_32]] = {{.*}} "amdgpu-max-num-workgroups"="4,8,32" + // CHECK-DAG: attributes [[A_FUNCTION]] = {{.*}} // CHECK-DAG: attributes [[DEFAULT_KERNEL_ATTRS]] = {{.*}} "amdgpu-flat-work-group-size"="1,256" diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test index ec84ebdc6abe..318bfb2df2a7 100644 --- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test +++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test @@ -4,6 +4,7 @@ // CHECK: #pragma clang attribute supports the following attributes: // CHECK-NEXT: AMDGPUFlatWorkGroupSize (SubjectMatchRule_function) +// CHECK-NEXT: AMDGPUMaxNumWorkGroups (SubjectMatchRule_function) // CHECK-NEXT: AMDGPUNumSGPR (SubjectMatchRule_function) // CHECK-NEXT: AMDGPUNumVGPR (SubjectMatchRule_function) // CHECK-NEXT: AMDGPUWavesPerEU (SubjectMatchRule_function) diff --git a/clang/test/SemaCUDA/amdgpu-attrs.cu b/clang/test/SemaCUDA/amdgpu-attrs.cu index 4811ef796c66..e04b32d121bc 100644 --- a/clang/test/SemaCUDA/amdgpu-attrs.cu +++ b/clang/test/SemaCUDA/amdgpu-attrs.cu @@ -63,6 +63,16 @@ __global__ void flat_work_group_size_32_64_waves_per_eu_2_num_sgpr_32_num_vgpr_6 __attribute__((amdgpu_flat_work_group_size(32, 64), amdgpu_waves_per_eu(2, 4), amdgpu_num_sgpr(32), amdgpu_num_vgpr(64))) __global__ void flat_work_group_size_32_64_waves_per_eu_2_4_num_sgpr_32_num_vgpr_64() {} +__attribute__((amdgpu_max_num_work_groups(32, 1, 1))) +__global__ void max_num_work_groups_32_1_1() {} + +__attribute__((amdgpu_max_num_work_groups(32, 1, 1), amdgpu_flat_work_group_size(32, 64))) +__global__ void max_num_work_groups_32_1_1_flat_work_group_size_32_64() {} + +__attribute__((amdgpu_max_num_work_groups(32, 1, 1), amdgpu_flat_work_group_size(32, 64), amdgpu_waves_per_eu(2, 4), amdgpu_num_sgpr(32), amdgpu_num_vgpr(64))) +__global__ void max_num_work_groups_32_1_1_flat_work_group_size_32_64_waves_per_eu_2_4_num_sgpr_32_num_vgpr_64() {} + + // expected-error@+2{{attribute 'reqd_work_group_size' can only be applied to an OpenCL kernel function}} __attribute__((reqd_work_group_size(32, 64, 64))) __global__ void reqd_work_group_size_32_64_64() {} @@ -194,3 +204,125 @@ __global__ void non_cexpr_waves_per_eu_2() {} // expected-error@+1{{'amdgpu_waves_per_eu' attribute requires parameter 1 to be an integer constant}} __attribute__((amdgpu_waves_per_eu(2, ipow2(2)))) __global__ void non_cexpr_waves_per_eu_2_4() {} + +__attribute__((amdgpu_max_num_work_groups(32))) +__global__ void max_num_work_groups_32() {} + +__attribute__((amdgpu_max_num_work_groups(32, 1))) +__global__ void max_num_work_groups_32_1() {} + +// expected-error@+1{{'amdgpu_max_num_work_groups' attribute takes no more than 3 arguments}} +__attribute__((amdgpu_max_num_work_groups(32, 1, 1, 1))) +__global__ void max_num_work_groups_32_1_1_1() {} + +// expected-error@+1{{'amdgpu_max_num_work_groups' attribute takes at least 1 argument}} +__attribute__((amdgpu_max_num_work_groups())) +__global__ void max_num_work_groups_no_arg() {} + +// expected-error@+1{{expected expression}} +__attribute__((amdgpu_max_num_work_groups(,1,1))) +__global__ void max_num_work_groups_empty_1_1() {} + +// expected-error@+1{{expected expression}} +__attribute__((amdgpu_max_num_work_groups(32,,1))) +__global__ void max_num_work_groups_32_empty_1() {} + +// expected-error@+1{{'amdgpu_max_num_work_groups' attribute requires parameter 0 to be an integer constant}} +__attribute__((amdgpu_max_num_work_groups(ipow2(5), 1, 1))) +__global__ void max_num_work_groups_32_1_1_non_int_arg0() {} + +// expected-error@+1{{'amdgpu_max_num_work_groups' attribute requires parameter 1 to be an integer constant}} +__attribute__((amdgpu_max_num_work_groups(32, "1", 1))) +__global__ void max_num_work_groups_32_1_1_non_int_arg1() {} + +// expected-error@+1{{'amdgpu_max_num_work_groups' attribute requires a non-negative integral compile time constant expression}} +__attribute__((amdgpu_max_num_work_groups(-32, 1, 1))) +__global__ void max_num_work_groups_32_1_1_neg_int_arg0() {} + +// expected-error@+1{{'amdgpu_max_num_work_groups' attribute requires a non-negative integral compile time constant expression}} +__attribute__((amdgpu_max_num_work_groups(32, -1, 1))) +__global__ void max_num_work_groups_32_1_1_neg_int_arg1() {} + +// expected-error@+1{{'amdgpu_max_num_work_groups' attribute requires a non-negative integral compile time constant expression}} +__attribute__((amdgpu_max_num_work_groups(32, 1, -1))) +__global__ void max_num_work_groups_32_1_1_neg_int_arg2() {} + +// expected-error@+1{{'amdgpu_max_num_work_groups' attribute must be greater than 0}} +__attribute__((amdgpu_max_num_work_groups(0, 1, 1))) +__global__ void max_num_work_groups_0_1_1() {} + +// expected-error@+1{{'amdgpu_max_num_work_groups' attribute must be greater than 0}} +__attribute__((amdgpu_max_num_work_groups(32, 0, 1))) +__global__ void max_num_work_groups_32_0_1() {} + +// expected-error@+1{{'amdgpu_max_num_work_groups' attribute must be greater than 0}} +__attribute__((amdgpu_max_num_work_groups(32, 1, 0))) +__global__ void max_num_work_groups_32_1_0() {} + +__attribute__((amdgpu_max_num_work_groups(4294967295))) +__global__ void max_num_work_groups_max_unsigned_int() {} + +// expected-error@+1{{integer constant expression evaluates to value 4294967296 that cannot be represented in a 32-bit unsigned integer type}} +__attribute__((amdgpu_max_num_work_groups(4294967296))) +__global__ void max_num_work_groups_max_unsigned_int_plus1() {} + +// expected-error@+1{{integer constant expression evaluates to value 10000000000 that cannot be represented in a 32-bit unsigned integer type}} +__attribute__((amdgpu_max_num_work_groups(10000000000))) +__global__ void max_num_work_groups_too_large() {} + +int num_wg_x = 32; +int num_wg_y = 1; +int num_wg_z = 1; +// expected-error@+1{{'amdgpu_max_num_work_groups' attribute requires parameter 0 to be an integer constant}} +__attribute__((amdgpu_max_num_work_groups(num_wg_x, 1, 1))) +__global__ void max_num_work_groups_32_1_1_non_const_arg0() {} + +// expected-error@+1{{'amdgpu_max_num_work_groups' attribute requires parameter 1 to be an integer constant}} +__attribute__((amdgpu_max_num_work_groups(32, num_wg_y, 1))) +__global__ void max_num_work_groups_32_1_1_non_const_arg1() {} + +// expected-error@+1{{'amdgpu_max_num_work_groups' attribute requires parameter 2 to be an integer constant}} +__attribute__((amdgpu_max_num_work_groups(32, 1, num_wg_z))) +__global__ void max_num_work_groups_32_1_1_non_const_arg2() {} + +const int c_num_wg_x = 32; +__attribute__((amdgpu_max_num_work_groups(c_num_wg_x, 1, 1))) +__global__ void max_num_work_groups_32_1_1_const_arg0() {} + +template +__attribute__((amdgpu_max_num_work_groups(a, 1, 1))) +__global__ void template_a_1_1_max_num_work_groups() {} +template __global__ void template_a_1_1_max_num_work_groups<32>(); + +template +__attribute__((amdgpu_max_num_work_groups(32, a, 1))) +__global__ void template_32_a_1_max_num_work_groups() {} +template __global__ void template_32_a_1_max_num_work_groups<1>(); + +template +__attribute__((amdgpu_max_num_work_groups(32, 1, a))) +__global__ void template_32_1_a_max_num_work_groups() {} +template __global__ void template_32_1_a_max_num_work_groups<1>(); + +// expected-error@+3{{'amdgpu_max_num_work_groups' attribute must be greater than 0}} +// expected-note@+4{{in instantiation of}} +template +__attribute__((amdgpu_max_num_work_groups(b, 1, 1))) +__global__ void template_b_1_1_max_num_work_groups() {} +template __global__ void template_b_1_1_max_num_work_groups<0>(); + +// expected-error@+3{{'amdgpu_max_num_work_groups' attribute must be greater than 0}} +// expected-note@+4{{in instantiation of}} +template +__attribute__((amdgpu_max_num_work_groups(32, b, 1))) +__global__ void template_32_b_1_max_num_work_groups() {} +template __global__ void template_32_b_1_max_num_work_groups<0>(); + +// expected-error@+3{{'amdgpu_max_num_work_groups' attribute must be greater than 0}} +// expected-note@+4{{in instantiation of}} +template +__attribute__((amdgpu_max_num_work_groups(32, 1, b))) +__global__ void template_32_1_b_max_num_work_groups() {} +template __global__ void template_32_1_b_max_num_work_groups<0>(); + + diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index f5f37d9e8a3b..99d7a482710f 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -1442,6 +1442,11 @@ The AMDGPU backend supports the following LLVM IR attributes. the frame. This is an internal detail of how LDS variables are lowered, language front ends should not set this attribute. + "amdgpu-max-num-workgroups"="x,y,z" Specify the maximum number of work groups for the kernel dispatch in the + X, Y, and Z dimensions. Generated by the ``amdgpu_max_num_work_groups`` + CLANG attribute [CLANG-ATTR]_. Clang only emits this attribute when all + the three numbers are >= 1. + ======================================= ========================================================== Calling Conventions @@ -3917,6 +3922,11 @@ same *vendor-name*. If omitted, "normal" is assumed. + ".max_num_work_groups_{x,y,z}" integer The max number of + launched work-groups + in the X, Y, and Z + dimensions. Each number + must be >=1. =================================== ============== ========= ================================ .. diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp index c20fdd51607a..9e288ab50e17 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp @@ -494,6 +494,14 @@ MetadataStreamerMsgPackV4::getHSAKernelProps(const MachineFunction &MF, Kern[".max_flat_workgroup_size"] = Kern.getDocument()->getNode(MFI.getMaxFlatWorkGroupSize()); + unsigned NumWGX = MFI.getMaxNumWorkGroupsX(); + unsigned NumWGY = MFI.getMaxNumWorkGroupsY(); + unsigned NumWGZ = MFI.getMaxNumWorkGroupsZ(); + if (NumWGX != 0 && NumWGY != 0 && NumWGZ != 0) { + Kern[".max_num_workgroups_x"] = Kern.getDocument()->getNode(NumWGX); + Kern[".max_num_workgroups_y"] = Kern.getDocument()->getNode(NumWGY); + Kern[".max_num_workgroups_z"] = Kern.getDocument()->getNode(NumWGZ); + } Kern[".sgpr_spill_count"] = Kern.getDocument()->getNode(MFI.getNumSpilledSGPRs()); Kern[".vgpr_spill_count"] = diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp index bcc7dedf3229..fa77b94fc22d 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp @@ -432,7 +432,7 @@ std::pair AMDGPUSubtarget::getEffectiveWavesPerEU( std::pair Default(1, getMaxWavesPerEU()); // If minimum/maximum flat work group sizes were explicitly requested using - // "amdgpu-flat-work-group-size" attribute, then set default minimum/maximum + // "amdgpu-flat-workgroup-size" attribute, then set default minimum/maximum // number of waves per execution unit to values implied by requested // minimum/maximum flat work group sizes. unsigned MinImpliedByFlatWorkGroupSize = @@ -1108,3 +1108,8 @@ void GCNUserSGPRUsageInfo::allocKernargPreloadSGPRs(unsigned NumSGPRs) { unsigned GCNUserSGPRUsageInfo::getNumFreeUserSGPRs() { return AMDGPU::getMaxNumUserSGPRs(ST) - NumUsedUserSGPRs; } + +SmallVector +AMDGPUSubtarget::getMaxNumWorkGroups(const Function &F) const { + return AMDGPU::getIntegerVecAttribute(F, "amdgpu-max-num-workgroups", 3); +} diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h index b72697973be7..e2d8b5d1ce97 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h @@ -288,6 +288,9 @@ public: /// 2) dimension. unsigned getMaxWorkitemID(const Function &Kernel, unsigned Dimension) const; + /// Return the number of work groups for the function. + SmallVector getMaxNumWorkGroups(const Function &F) const; + /// Return true if only a single workitem can be active in a wave. bool isSingleLaneExecution(const Function &Kernel) const; diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp index 52d6fe6c7ba5..2569f40fec0e 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp @@ -46,6 +46,8 @@ SIMachineFunctionInfo::SIMachineFunctionInfo(const Function &F, const GCNSubtarget &ST = *static_cast(STI); FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F); WavesPerEU = ST.getWavesPerEU(F); + MaxNumWorkGroups = ST.getMaxNumWorkGroups(F); + assert(MaxNumWorkGroups.size() == 3); Occupancy = ST.computeOccupancy(F, getLDSSize()); CallingConv::ID CC = F.getCallingConv(); diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h index 0336ec4985ea..7d0c1ba8448e 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h @@ -426,6 +426,9 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunction, const AMDGPUGWSResourcePseudoSourceValue GWSResourcePSV; + // Default/requested number of work groups for the function. + SmallVector MaxNumWorkGroups = {0, 0, 0}; + private: unsigned NumUserSGPRs = 0; unsigned NumSystemSGPRs = 0; @@ -1072,6 +1075,13 @@ public: // \returns true if a function needs or may need AGPRs. bool usesAGPRs(const MachineFunction &MF) const; + + /// \returns Default/requested number of work groups for this function. + SmallVector getMaxNumWorkGroups() const { return MaxNumWorkGroups; } + + unsigned getMaxNumWorkGroupsX() const { return MaxNumWorkGroups[0]; } + unsigned getMaxNumWorkGroupsY() const { return MaxNumWorkGroups[1]; } + unsigned getMaxNumWorkGroupsZ() const { return MaxNumWorkGroups[2]; } }; } // end namespace llvm diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index edb0e50da289..aa47dccf2dd2 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -11,6 +11,7 @@ #include "AMDGPUAsmUtils.h" #include "AMDKernelCodeT.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Constants.h" @@ -1298,6 +1299,42 @@ getIntegerPairAttribute(const Function &F, StringRef Name, return Ints; } +SmallVector getIntegerVecAttribute(const Function &F, StringRef Name, + unsigned Size) { + assert(Size > 2); + SmallVector Default(Size, 0); + + Attribute A = F.getFnAttribute(Name); + if (!A.isStringAttribute()) + return Default; + + SmallVector Vals(Size, 0); + + LLVMContext &Ctx = F.getContext(); + + StringRef S = A.getValueAsString(); + unsigned i = 0; + for (; !S.empty() && i < Size; i++) { + std::pair Strs = S.split(','); + unsigned IntVal; + if (Strs.first.trim().getAsInteger(0, IntVal)) { + Ctx.emitError("can't parse integer attribute " + Strs.first + " in " + + Name); + return Default; + } + Vals[i] = IntVal; + S = Strs.second; + } + + if (!S.empty() || i < Size) { + Ctx.emitError("attribute " + Name + + " has incorrect number of integers; expected " + + llvm::utostr(Size)); + return Default; + } + return Vals; +} + unsigned getVmcntBitMask(const IsaVersion &Version) { return (1 << (getVmcntBitWidthLo(Version.Major) + getVmcntBitWidthHi(Version.Major))) - diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index d7ea2a3eff4b..f8521cba077c 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -863,6 +863,14 @@ bool isReadOnlySegment(const GlobalValue *GV); /// target triple \p TT, false otherwise. bool shouldEmitConstantsToTextSection(const Triple &TT); +/// \returns Integer value requested using \p F's \p Name attribute. +/// +/// \returns \p Default if attribute is not present. +/// +/// \returns \p Default and emits error if requested value cannot be converted +/// to integer. +int getIntegerAttribute(const Function &F, StringRef Name, int Default); + /// \returns A pair of integer values requested using \p F's \p Name attribute /// in "first[,second]" format ("second" is optional unless \p OnlyFirstRequired /// is false). @@ -877,6 +885,16 @@ getIntegerPairAttribute(const Function &F, StringRef Name, std::pair Default, bool OnlyFirstRequired = false); +/// \returns Generate a vector of integer values requested using \p F's \p Name +/// attribute. +/// +/// \returns true if exactly Size (>2) number of integers are found in the +/// attribute. +/// +/// \returns false if any error occurs. +SmallVector getIntegerVecAttribute(const Function &F, StringRef Name, + unsigned Size); + /// Represents the counter values to wait for in an s_waitcnt instruction. /// /// Large values (including the maximum possible integer) can be used to diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-workgroups.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-workgroups.ll new file mode 100644 index 000000000000..bc58222076ac --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-workgroups.ll @@ -0,0 +1,84 @@ +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck %s + +; Attribute not specified. +; CHECK-LABEL: {{^}}empty_no_attribute: +define amdgpu_kernel void @empty_no_attribute() { +entry: + ret void +} + +; Ignore if number of work groups for x dimension is 0. +; CHECK-LABEL: {{^}}empty_max_num_workgroups_x0: +define amdgpu_kernel void @empty_max_num_workgroups_x0() #0 { +entry: + ret void +} +attributes #0 = {"amdgpu-max-num-workgroups"="0,2,3"} + +; Ignore if number of work groups for y dimension is 0. +; CHECK-LABEL: {{^}}empty_max_num_workgroups_y0: +define amdgpu_kernel void @empty_max_num_workgroups_y0() #1 { +entry: + ret void +} +attributes #1 = {"amdgpu-max-num-workgroups"="1,0,3"} + +; Ignore if number of work groups for z dimension is 0. +; CHECK-LABEL: {{^}}empty_max_num_workgroups_z0: +define amdgpu_kernel void @empty_max_num_workgroups_z0() #2 { +entry: + ret void +} +attributes #2 = {"amdgpu-max-num-workgroups"="1,2,0"} + +; CHECK-LABEL: {{^}}empty_max_num_workgroups_1_2_3: +define amdgpu_kernel void @empty_max_num_workgroups_1_2_3() #3 { +entry: + ret void +} +attributes #3 = {"amdgpu-max-num-workgroups"="1,2,3"} + +; CHECK-LABEL: {{^}}empty_max_num_workgroups_1024_1024_1024: +define amdgpu_kernel void @empty_max_num_workgroups_1024_1024_1024() #4 { +entry: + ret void +} +attributes #4 = {"amdgpu-max-num-workgroups"="1024,1024,1024"} + + +; CHECK: .amdgpu_metadata +; CHECK: - .args: +; CHECK: .max_flat_workgroup_size: 1024 +; CHECK-NEXT: .name: empty_no_attribute +; CHECK-NEXT: .private_segment_fixed_size: 0 + +; CHECK: - .args: +; CHECK: .max_flat_workgroup_size: 1024 +; CHECK-NEXT: .name: empty_max_num_workgroups_x0 +; CHECK-NEXT: .private_segment_fixed_size: 0 + +; CHECK: - .args: +; CHECK: .max_flat_workgroup_size: 1024 +; CHECK-NEXT: .name: empty_max_num_workgroups_y0 +; CHECK-NEXT: .private_segment_fixed_size: 0 + +; CHECK: - .args: +; CHECK: .max_flat_workgroup_size: 1024 +; CHECK-NEXT: .name: empty_max_num_workgroups_z0 +; CHECK-NEXT: .private_segment_fixed_size: 0 + +; CHECK: - .args: +; CHECK: .max_flat_workgroup_size: 1024 +; CHECK-NEXT: .max_num_workgroups_x: 1 +; CHECK-NEXT: .max_num_workgroups_y: 2 +; CHECK-NEXT: .max_num_workgroups_z: 3 +; CHECK-NEXT: .name: empty_max_num_workgroups_1_2_3 +; CHECK-NEXT: .private_segment_fixed_size: 0 + +; CHECK: - .args: +; CHECK: .max_flat_workgroup_size: 1024 +; CHECK-NEXT: .max_num_workgroups_x: 1024 +; CHECK-NEXT: .max_num_workgroups_y: 1024 +; CHECK-NEXT: .max_num_workgroups_z: 1024 +; CHECK-NEXT: .name: empty_max_num_workgroups_1024_1024_1024 +; CHECK-NEXT: .private_segment_fixed_size: 0 diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-workgroups_error_check.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-workgroups_error_check.ll new file mode 100644 index 000000000000..6d86d2d7c1a3 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-workgroups_error_check.ll @@ -0,0 +1,71 @@ +; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s 2>&1 | FileCheck --check-prefix=ERROR %s + +; ERROR: error: can't parse integer attribute -1 in amdgpu-max-num-workgroups +define amdgpu_kernel void @empty_max_num_workgroups_neg_num1() #21 { +entry: + ret void +} +attributes #21 = {"amdgpu-max-num-workgroups"="-1,2,3"} + +; ERROR: error: can't parse integer attribute -2 in amdgpu-max-num-workgroups +define amdgpu_kernel void @empty_max_num_workgroups_neg_num2() #22 { +entry: + ret void +} +attributes #22 = {"amdgpu-max-num-workgroups"="1,-2,3"} + +; ERROR: error: can't parse integer attribute -3 in amdgpu-max-num-workgroups +define amdgpu_kernel void @empty_max_num_workgroups_neg_num3() #23 { +entry: + ret void +} +attributes #23 = {"amdgpu-max-num-workgroups"="1,2,-3"} + +; ERROR: error: can't parse integer attribute 1.0 in amdgpu-max-num-workgroups +define amdgpu_kernel void @empty_max_num_workgroups_non_int1() #31 { +entry: + ret void +} +attributes #31 = {"amdgpu-max-num-workgroups"="1.0,2,3"} + +; ERROR: error: can't parse integer attribute 2.0 in amdgpu-max-num-workgroups +define amdgpu_kernel void @empty_max_num_workgroups_non_int2() #32 { +entry: + ret void +} +attributes #32 = {"amdgpu-max-num-workgroups"="1,2.0,3"} + +; ERROR: error: can't parse integer attribute 3.0 in amdgpu-max-num-workgroups +define amdgpu_kernel void @empty_max_num_workgroups_non_int3() #33 { +entry: + ret void +} +attributes #33 = {"amdgpu-max-num-workgroups"="1,2,3.0"} + +; ERROR: error: can't parse integer attribute 10000000000 in amdgpu-max-num-workgroups +define amdgpu_kernel void @empty_max_num_workgroups_too_large() #41 { +entry: + ret void +} +attributes #41 = {"amdgpu-max-num-workgroups"="10000000000,2,3"} + +; ERROR: error: attribute amdgpu-max-num-workgroups has incorrect number of integers; expected 3 +define amdgpu_kernel void @empty_max_num_workgroups_1_arg() #51 { +entry: + ret void +} +attributes #51 = {"amdgpu-max-num-workgroups"="1"} + +; ERROR: error: attribute amdgpu-max-num-workgroups has incorrect number of integers; expected 3 +define amdgpu_kernel void @empty_max_num_workgroups_2_args() #52 { +entry: + ret void +} +attributes #52 = {"amdgpu-max-num-workgroups"="1,2"} + +; ERROR: error: attribute amdgpu-max-num-workgroups has incorrect number of integers; expected 3 +define amdgpu_kernel void @empty_max_num_workgroups_4_args() #53 { +entry: + ret void +} +attributes #53 = {"amdgpu-max-num-workgroups"="1,2,3,4"} -- GitLab From c1af6ab505a83bfb4fc8752591ad333190bc9389 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Tue, 12 Mar 2024 17:29:30 +0000 Subject: [PATCH 0027/1407] [X86] getFauxShuffleMask - recognise CONCAT(SUB0, SUB1) style patterns Handles the INSERT_SUBVECTOR(INSERT_SUBVECTOR(UNDEF,SUB0,0),SUB1,N) pattern Currently limited to v8i64/v8f64 cases as only AVX512 has decent cross lane 2-input shuffles, the plan is to relax this as I deal with some regressions --- llvm/lib/Target/X86/X86ISelLowering.cpp | 17 ++ .../vector-interleaved-store-i16-stride-7.ll | 10 +- .../vector-interleaved-store-i16-stride-8.ll | 28 +- .../vector-interleaved-store-i32-stride-7.ll | 20 +- .../vector-interleaved-store-i32-stride-8.ll | 280 +++++++++--------- .../vector-interleaved-store-i64-stride-4.ll | 120 ++++---- 6 files changed, 233 insertions(+), 242 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 72b45d462dfe..2b5e3c0379a1 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -5858,6 +5858,23 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts, Ops.push_back(SubBCSrc); return true; } + // Handle CONCAT(SUB0, SUB1). + // Limit this to vXi64 512-bit vector cases to make the most of AVX512 + // cross lane shuffles. + if (Depth > 0 && InsertIdx == NumSubElts && NumElts == (2 * NumSubElts) && + NumBitsPerElt == 64 && NumSizeInBits == 512 && + Src.getOpcode() == ISD::INSERT_SUBVECTOR && + Src.getOperand(0).isUndef() && + Src.getOperand(1).getValueType() == SubVT && + Src.getConstantOperandVal(2) == 0) { + for (int i = 0; i != (int)NumSubElts; ++i) + Mask.push_back(i); + for (int i = 0; i != (int)NumSubElts; ++i) + Mask.push_back(i + NumElts); + Ops.push_back(Src.getOperand(1)); + Ops.push_back(Sub); + return true; + } // Handle INSERT_SUBVECTOR(SRC0, SHUFFLE(SRC1)). SmallVector SubMask; SmallVector SubInputs; diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll index 79cc8e49f1fd..9e70aef86885 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll @@ -821,9 +821,8 @@ define void @store_i16_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0] ; AVX512BW-FCP-NEXT: vpermi2q %ymm3, %ymm0, %ymm1 -; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm0 -; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,16,20,24,1,5,9,13,17,21,25,2,6,10,14,18,22,26,3,7,11,15,19,23,27,0,0,0,0] -; AVX512BW-FCP-NEXT: vpermw %zmm0, %zmm1, %zmm0 +; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm0 = [0,4,8,12,32,36,40,1,5,9,13,33,37,41,2,6,10,14,34,38,42,3,7,11,15,35,39,43,0,0,0,0] +; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm2, %zmm0 ; AVX512BW-FCP-NEXT: vextracti32x4 $2, %zmm0, 32(%rax) ; AVX512BW-FCP-NEXT: vextracti32x4 $3, %zmm0, %xmm1 ; AVX512BW-FCP-NEXT: vmovq %xmm1, 48(%rax) @@ -873,9 +872,8 @@ define void @store_i16_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0] ; AVX512DQ-BW-FCP-NEXT: vpermi2q %ymm3, %ymm0, %ymm1 -; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm0 -; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,16,20,24,1,5,9,13,17,21,25,2,6,10,14,18,22,26,3,7,11,15,19,23,27,0,0,0,0] -; AVX512DQ-BW-FCP-NEXT: vpermw %zmm0, %zmm1, %zmm0 +; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm0 = [0,4,8,12,32,36,40,1,5,9,13,33,37,41,2,6,10,14,34,38,42,3,7,11,15,35,39,43,0,0,0,0] +; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm2, %zmm0 ; AVX512DQ-BW-FCP-NEXT: vextracti32x4 $2, %zmm0, 32(%rax) ; AVX512DQ-BW-FCP-NEXT: vextracti32x4 $3, %zmm0, %xmm1 ; AVX512DQ-BW-FCP-NEXT: vmovq %xmm1, 48(%rax) diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll index 194b715b6594..32825f291e98 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll @@ -762,10 +762,9 @@ define void @store_i16_stride8_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0] ; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 ; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 -; AVX512BW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,16,20,24,28,1,5,9,13,17,21,25,29,2,6,10,14,18,22,26,30,3,7,11,15,19,23,27,31] -; AVX512BW-NEXT: vpermw %zmm0, %zmm1, %zmm0 -; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rax) +; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,32,36,40,44,1,5,9,13,33,37,41,45,2,6,10,14,34,38,42,46,3,7,11,15,35,39,43,47] +; AVX512BW-NEXT: vpermi2w %zmm2, %zmm0, %zmm1 +; AVX512BW-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; @@ -788,10 +787,9 @@ define void @store_i16_stride8_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0] ; AVX512BW-FCP-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 ; AVX512BW-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 -; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,16,20,24,28,1,5,9,13,17,21,25,29,2,6,10,14,18,22,26,30,3,7,11,15,19,23,27,31] -; AVX512BW-FCP-NEXT: vpermw %zmm0, %zmm1, %zmm0 -; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, (%rax) +; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,32,36,40,44,1,5,9,13,33,37,41,45,2,6,10,14,34,38,42,46,3,7,11,15,35,39,43,47] +; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm0, %zmm1 +; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512BW-FCP-NEXT: vzeroupper ; AVX512BW-FCP-NEXT: retq ; @@ -814,10 +812,9 @@ define void @store_i16_stride8_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0] ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 -; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,16,20,24,28,1,5,9,13,17,21,25,29,2,6,10,14,18,22,26,30,3,7,11,15,19,23,27,31] -; AVX512DQ-BW-NEXT: vpermw %zmm0, %zmm1, %zmm0 -; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, (%rax) +; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,32,36,40,44,1,5,9,13,33,37,41,45,2,6,10,14,34,38,42,46,3,7,11,15,35,39,43,47] +; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm0, %zmm1 +; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512DQ-BW-NEXT: vzeroupper ; AVX512DQ-BW-NEXT: retq ; @@ -840,10 +837,9 @@ define void @store_i16_stride8_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0] ; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 ; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 -; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,16,20,24,28,1,5,9,13,17,21,25,29,2,6,10,14,18,22,26,30,3,7,11,15,19,23,27,31] -; AVX512DQ-BW-FCP-NEXT: vpermw %zmm0, %zmm1, %zmm0 -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, (%rax) +; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,32,36,40,44,1,5,9,13,33,37,41,45,2,6,10,14,34,38,42,46,3,7,11,15,35,39,43,47] +; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm0, %zmm1 +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512DQ-BW-FCP-NEXT: vzeroupper ; AVX512DQ-BW-FCP-NEXT: retq %in.vec0 = load <4 x i16>, ptr %in.vecptr0, align 64 diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll index 837d990596a5..45a76599d3e9 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll @@ -227,9 +227,8 @@ define void @store_i32_stride7_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0] ; AVX512-FCP-NEXT: vpermi2q %ymm3, %ymm0, %ymm1 -; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm0 -; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,2,4,6,8,10,12,1,3,5,7,9,11,13,0,0] -; AVX512-FCP-NEXT: vpermd %zmm0, %zmm1, %zmm0 +; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm0 = [0,2,4,6,16,18,20,1,3,5,7,17,19,21,0,0] +; AVX512-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm0 ; AVX512-FCP-NEXT: vextracti32x4 $2, %zmm0, 32(%rax) ; AVX512-FCP-NEXT: vextracti32x4 $3, %zmm0, %xmm1 ; AVX512-FCP-NEXT: vmovq %xmm1, 48(%rax) @@ -279,9 +278,8 @@ define void @store_i32_stride7_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0] ; AVX512DQ-FCP-NEXT: vpermi2q %ymm3, %ymm0, %ymm1 -; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm0 -; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,2,4,6,8,10,12,1,3,5,7,9,11,13,0,0] -; AVX512DQ-FCP-NEXT: vpermd %zmm0, %zmm1, %zmm0 +; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm0 = [0,2,4,6,16,18,20,1,3,5,7,17,19,21,0,0] +; AVX512DQ-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm0 ; AVX512DQ-FCP-NEXT: vextracti32x4 $2, %zmm0, 32(%rax) ; AVX512DQ-FCP-NEXT: vextracti32x4 $3, %zmm0, %xmm1 ; AVX512DQ-FCP-NEXT: vmovq %xmm1, 48(%rax) @@ -331,9 +329,8 @@ define void @store_i32_stride7_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0] ; AVX512BW-FCP-NEXT: vpermi2q %ymm3, %ymm0, %ymm1 -; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm0 -; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,2,4,6,8,10,12,1,3,5,7,9,11,13,0,0] -; AVX512BW-FCP-NEXT: vpermd %zmm0, %zmm1, %zmm0 +; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm0 = [0,2,4,6,16,18,20,1,3,5,7,17,19,21,0,0] +; AVX512BW-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm0 ; AVX512BW-FCP-NEXT: vextracti32x4 $2, %zmm0, 32(%rax) ; AVX512BW-FCP-NEXT: vextracti32x4 $3, %zmm0, %xmm1 ; AVX512BW-FCP-NEXT: vmovq %xmm1, 48(%rax) @@ -383,9 +380,8 @@ define void @store_i32_stride7_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0] ; AVX512DQ-BW-FCP-NEXT: vpermi2q %ymm3, %ymm0, %ymm1 -; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm0 -; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,2,4,6,8,10,12,1,3,5,7,9,11,13,0,0] -; AVX512DQ-BW-FCP-NEXT: vpermd %zmm0, %zmm1, %zmm0 +; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm0 = [0,2,4,6,16,18,20,1,3,5,7,17,19,21,0,0] +; AVX512DQ-BW-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm0 ; AVX512DQ-BW-FCP-NEXT: vextracti32x4 $2, %zmm0, 32(%rax) ; AVX512DQ-BW-FCP-NEXT: vextracti32x4 $3, %zmm0, %xmm1 ; AVX512DQ-BW-FCP-NEXT: vmovq %xmm1, 48(%rax) diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll index 955927eb7691..265f6daeb200 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll @@ -160,24 +160,23 @@ define void @store_i32_stride8_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11 -; AVX512-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero -; AVX512-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512-NEXT: vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] -; AVX512-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512-NEXT: vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX512-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512-NEXT: vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0] -; AVX512-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512-NEXT: vmovsd {{.*#+}} xmm4 = mem[0],zero -; AVX512-NEXT: vmovlhps {{.*#+}} xmm3 = xmm4[0],xmm3[0] -; AVX512-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2 -; AVX512-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX512-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15] -; AVX512-NEXT: vpermps %zmm0, %zmm1, %zmm0 -; AVX512-NEXT: vmovaps %zmm0, (%rax) +; AVX512-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX512-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero +; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0] +; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 +; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,2,4,6,16,18,20,22,1,3,5,7,17,19,21,23] +; AVX512-NEXT: vpermi2d %zmm2, %zmm0, %zmm1 +; AVX512-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq ; @@ -186,24 +185,23 @@ define void @store_i32_stride8_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11 -; AVX512-FCP-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero -; AVX512-FCP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512-FCP-NEXT: vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] -; AVX512-FCP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512-FCP-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512-FCP-NEXT: vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX512-FCP-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512-FCP-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0] -; AVX512-FCP-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512-FCP-NEXT: vmovsd {{.*#+}} xmm4 = mem[0],zero -; AVX512-FCP-NEXT: vmovlhps {{.*#+}} xmm3 = xmm4[0],xmm3[0] -; AVX512-FCP-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2 -; AVX512-FCP-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX512-FCP-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512-FCP-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15] -; AVX512-FCP-NEXT: vpermps %zmm0, %zmm1, %zmm0 -; AVX512-FCP-NEXT: vmovaps %zmm0, (%rax) +; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero +; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0] +; AVX512-FCP-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 +; AVX512-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,2,4,6,16,18,20,22,1,3,5,7,17,19,21,23] +; AVX512-FCP-NEXT: vpermi2d %zmm2, %zmm0, %zmm1 +; AVX512-FCP-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512-FCP-NEXT: vzeroupper ; AVX512-FCP-NEXT: retq ; @@ -212,24 +210,23 @@ define void @store_i32_stride8_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %r10 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %r11 -; AVX512DQ-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero -; AVX512DQ-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512DQ-NEXT: vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] -; AVX512DQ-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512DQ-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512DQ-NEXT: vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX512DQ-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512DQ-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512DQ-NEXT: vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0] -; AVX512DQ-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512DQ-NEXT: vmovsd {{.*#+}} xmm4 = mem[0],zero -; AVX512DQ-NEXT: vmovlhps {{.*#+}} xmm3 = xmm4[0],xmm3[0] -; AVX512DQ-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2 -; AVX512DQ-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX512DQ-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512DQ-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15] -; AVX512DQ-NEXT: vpermps %zmm0, %zmm1, %zmm0 -; AVX512DQ-NEXT: vmovaps %zmm0, (%rax) +; AVX512DQ-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX512DQ-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512DQ-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512DQ-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512DQ-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512DQ-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512DQ-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512DQ-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero +; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0] +; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 +; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,2,4,6,16,18,20,22,1,3,5,7,17,19,21,23] +; AVX512DQ-NEXT: vpermi2d %zmm2, %zmm0, %zmm1 +; AVX512DQ-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512DQ-NEXT: vzeroupper ; AVX512DQ-NEXT: retq ; @@ -238,24 +235,23 @@ define void @store_i32_stride8_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11 -; AVX512DQ-FCP-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero -; AVX512DQ-FCP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512DQ-FCP-NEXT: vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] -; AVX512DQ-FCP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512DQ-FCP-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512DQ-FCP-NEXT: vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX512DQ-FCP-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512DQ-FCP-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512DQ-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0] -; AVX512DQ-FCP-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512DQ-FCP-NEXT: vmovsd {{.*#+}} xmm4 = mem[0],zero -; AVX512DQ-FCP-NEXT: vmovlhps {{.*#+}} xmm3 = xmm4[0],xmm3[0] -; AVX512DQ-FCP-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2 -; AVX512DQ-FCP-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX512DQ-FCP-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512DQ-FCP-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15] -; AVX512DQ-FCP-NEXT: vpermps %zmm0, %zmm1, %zmm0 -; AVX512DQ-FCP-NEXT: vmovaps %zmm0, (%rax) +; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero +; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0] +; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 +; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,2,4,6,16,18,20,22,1,3,5,7,17,19,21,23] +; AVX512DQ-FCP-NEXT: vpermi2d %zmm2, %zmm0, %zmm1 +; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512DQ-FCP-NEXT: vzeroupper ; AVX512DQ-FCP-NEXT: retq ; @@ -264,24 +260,23 @@ define void @store_i32_stride8_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r10 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r11 -; AVX512BW-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero -; AVX512BW-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512BW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] -; AVX512BW-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512BW-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512BW-NEXT: vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX512BW-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512BW-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512BW-NEXT: vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0] -; AVX512BW-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512BW-NEXT: vmovsd {{.*#+}} xmm4 = mem[0],zero -; AVX512BW-NEXT: vmovlhps {{.*#+}} xmm3 = xmm4[0],xmm3[0] -; AVX512BW-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2 -; AVX512BW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX512BW-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512BW-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15] -; AVX512BW-NEXT: vpermps %zmm0, %zmm1, %zmm0 -; AVX512BW-NEXT: vmovaps %zmm0, (%rax) +; AVX512BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX512BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512BW-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512BW-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512BW-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512BW-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512BW-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero +; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0] +; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 +; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,2,4,6,16,18,20,22,1,3,5,7,17,19,21,23] +; AVX512BW-NEXT: vpermi2d %zmm2, %zmm0, %zmm1 +; AVX512BW-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; @@ -290,24 +285,23 @@ define void @store_i32_stride8_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11 -; AVX512BW-FCP-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero -; AVX512BW-FCP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512BW-FCP-NEXT: vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] -; AVX512BW-FCP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512BW-FCP-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512BW-FCP-NEXT: vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX512BW-FCP-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512BW-FCP-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512BW-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0] -; AVX512BW-FCP-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512BW-FCP-NEXT: vmovsd {{.*#+}} xmm4 = mem[0],zero -; AVX512BW-FCP-NEXT: vmovlhps {{.*#+}} xmm3 = xmm4[0],xmm3[0] -; AVX512BW-FCP-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2 -; AVX512BW-FCP-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX512BW-FCP-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512BW-FCP-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15] -; AVX512BW-FCP-NEXT: vpermps %zmm0, %zmm1, %zmm0 -; AVX512BW-FCP-NEXT: vmovaps %zmm0, (%rax) +; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero +; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0] +; AVX512BW-FCP-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 +; AVX512BW-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,2,4,6,16,18,20,22,1,3,5,7,17,19,21,23] +; AVX512BW-FCP-NEXT: vpermi2d %zmm2, %zmm0, %zmm1 +; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512BW-FCP-NEXT: vzeroupper ; AVX512BW-FCP-NEXT: retq ; @@ -316,24 +310,23 @@ define void @store_i32_stride8_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r10 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r11 -; AVX512DQ-BW-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero -; AVX512DQ-BW-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512DQ-BW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] -; AVX512DQ-BW-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512DQ-BW-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512DQ-BW-NEXT: vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX512DQ-BW-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512DQ-BW-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512DQ-BW-NEXT: vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0] -; AVX512DQ-BW-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512DQ-BW-NEXT: vmovsd {{.*#+}} xmm4 = mem[0],zero -; AVX512DQ-BW-NEXT: vmovlhps {{.*#+}} xmm3 = xmm4[0],xmm3[0] -; AVX512DQ-BW-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2 -; AVX512DQ-BW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX512DQ-BW-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512DQ-BW-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15] -; AVX512DQ-BW-NEXT: vpermps %zmm0, %zmm1, %zmm0 -; AVX512DQ-BW-NEXT: vmovaps %zmm0, (%rax) +; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero +; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0] +; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 +; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,2,4,6,16,18,20,22,1,3,5,7,17,19,21,23] +; AVX512DQ-BW-NEXT: vpermi2d %zmm2, %zmm0, %zmm1 +; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512DQ-BW-NEXT: vzeroupper ; AVX512DQ-BW-NEXT: retq ; @@ -342,24 +335,23 @@ define void @store_i32_stride8_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11 -; AVX512DQ-BW-FCP-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero -; AVX512DQ-BW-FCP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512DQ-BW-FCP-NEXT: vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] -; AVX512DQ-BW-FCP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; AVX512DQ-BW-FCP-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512DQ-BW-FCP-NEXT: vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX512DQ-BW-FCP-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; AVX512DQ-BW-FCP-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512DQ-BW-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0] -; AVX512DQ-BW-FCP-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero -; AVX512DQ-BW-FCP-NEXT: vmovsd {{.*#+}} xmm4 = mem[0],zero -; AVX512DQ-BW-FCP-NEXT: vmovlhps {{.*#+}} xmm3 = xmm4[0],xmm3[0] -; AVX512DQ-BW-FCP-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2 -; AVX512DQ-BW-FCP-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX512DQ-BW-FCP-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512DQ-BW-FCP-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15] -; AVX512DQ-BW-FCP-NEXT: vpermps %zmm0, %zmm1, %zmm0 -; AVX512DQ-BW-FCP-NEXT: vmovaps %zmm0, (%rax) +; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero +; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero +; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero +; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0] +; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 +; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,2,4,6,16,18,20,22,1,3,5,7,17,19,21,23] +; AVX512DQ-BW-FCP-NEXT: vpermi2d %zmm2, %zmm0, %zmm1 +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512DQ-BW-FCP-NEXT: vzeroupper ; AVX512DQ-BW-FCP-NEXT: retq %in.vec0 = load <2 x i32>, ptr %in.vecptr0, align 64 diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-4.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-4.ll index 38623c6ce0cb..ded7c002c873 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-4.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-4.ll @@ -94,105 +94,97 @@ define void @store_i64_stride4_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; ; AVX512-LABEL: store_i64_stride4_vf2: ; AVX512: # %bb.0: -; AVX512-NEXT: vmovaps (%rdi), %xmm0 -; AVX512-NEXT: vmovaps (%rdx), %xmm1 -; AVX512-NEXT: vinsertf128 $1, (%rcx), %ymm1, %ymm1 -; AVX512-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0 -; AVX512-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,1,3,5,7] -; AVX512-NEXT: vpermpd %zmm0, %zmm1, %zmm0 -; AVX512-NEXT: vmovaps %zmm0, (%r8) +; AVX512-NEXT: vmovdqa (%rdi), %xmm0 +; AVX512-NEXT: vmovdqa (%rdx), %xmm1 +; AVX512-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 +; AVX512-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 +; AVX512-NEXT: vpmovsxbq {{.*#+}} zmm2 = [0,2,8,10,1,3,9,11] +; AVX512-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 +; AVX512-NEXT: vmovdqa64 %zmm2, (%r8) ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq ; ; AVX512-FCP-LABEL: store_i64_stride4_vf2: ; AVX512-FCP: # %bb.0: -; AVX512-FCP-NEXT: vmovaps (%rdi), %xmm0 -; AVX512-FCP-NEXT: vmovaps (%rdx), %xmm1 -; AVX512-FCP-NEXT: vinsertf128 $1, (%rcx), %ymm1, %ymm1 -; AVX512-FCP-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0 -; AVX512-FCP-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512-FCP-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,1,3,5,7] -; AVX512-FCP-NEXT: vpermpd %zmm0, %zmm1, %zmm0 -; AVX512-FCP-NEXT: vmovaps %zmm0, (%r8) +; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0 +; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm1 +; AVX512-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 +; AVX512-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 +; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm2 = [0,2,8,10,1,3,9,11] +; AVX512-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 +; AVX512-FCP-NEXT: vmovdqa64 %zmm2, (%r8) ; AVX512-FCP-NEXT: vzeroupper ; AVX512-FCP-NEXT: retq ; ; AVX512DQ-LABEL: store_i64_stride4_vf2: ; AVX512DQ: # %bb.0: -; AVX512DQ-NEXT: vmovaps (%rdi), %xmm0 -; AVX512DQ-NEXT: vmovaps (%rdx), %xmm1 -; AVX512DQ-NEXT: vinsertf128 $1, (%rcx), %ymm1, %ymm1 -; AVX512DQ-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0 -; AVX512DQ-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512DQ-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,1,3,5,7] -; AVX512DQ-NEXT: vpermpd %zmm0, %zmm1, %zmm0 -; AVX512DQ-NEXT: vmovaps %zmm0, (%r8) +; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0 +; AVX512DQ-NEXT: vmovdqa (%rdx), %xmm1 +; AVX512DQ-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 +; AVX512DQ-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 +; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm2 = [0,2,8,10,1,3,9,11] +; AVX512DQ-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 +; AVX512DQ-NEXT: vmovdqa64 %zmm2, (%r8) ; AVX512DQ-NEXT: vzeroupper ; AVX512DQ-NEXT: retq ; ; AVX512DQ-FCP-LABEL: store_i64_stride4_vf2: ; AVX512DQ-FCP: # %bb.0: -; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %xmm0 -; AVX512DQ-FCP-NEXT: vmovaps (%rdx), %xmm1 -; AVX512DQ-FCP-NEXT: vinsertf128 $1, (%rcx), %ymm1, %ymm1 -; AVX512DQ-FCP-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0 -; AVX512DQ-FCP-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512DQ-FCP-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,1,3,5,7] -; AVX512DQ-FCP-NEXT: vpermpd %zmm0, %zmm1, %zmm0 -; AVX512DQ-FCP-NEXT: vmovaps %zmm0, (%r8) +; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0 +; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm1 +; AVX512DQ-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 +; AVX512DQ-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 +; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm2 = [0,2,8,10,1,3,9,11] +; AVX512DQ-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 +; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, (%r8) ; AVX512DQ-FCP-NEXT: vzeroupper ; AVX512DQ-FCP-NEXT: retq ; ; AVX512BW-LABEL: store_i64_stride4_vf2: ; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vmovaps (%rdi), %xmm0 -; AVX512BW-NEXT: vmovaps (%rdx), %xmm1 -; AVX512BW-NEXT: vinsertf128 $1, (%rcx), %ymm1, %ymm1 -; AVX512BW-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0 -; AVX512BW-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512BW-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,1,3,5,7] -; AVX512BW-NEXT: vpermpd %zmm0, %zmm1, %zmm0 -; AVX512BW-NEXT: vmovaps %zmm0, (%r8) +; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0 +; AVX512BW-NEXT: vmovdqa (%rdx), %xmm1 +; AVX512BW-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 +; AVX512BW-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 +; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm2 = [0,2,8,10,1,3,9,11] +; AVX512BW-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 +; AVX512BW-NEXT: vmovdqa64 %zmm2, (%r8) ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; ; AVX512BW-FCP-LABEL: store_i64_stride4_vf2: ; AVX512BW-FCP: # %bb.0: -; AVX512BW-FCP-NEXT: vmovaps (%rdi), %xmm0 -; AVX512BW-FCP-NEXT: vmovaps (%rdx), %xmm1 -; AVX512BW-FCP-NEXT: vinsertf128 $1, (%rcx), %ymm1, %ymm1 -; AVX512BW-FCP-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0 -; AVX512BW-FCP-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512BW-FCP-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,1,3,5,7] -; AVX512BW-FCP-NEXT: vpermpd %zmm0, %zmm1, %zmm0 -; AVX512BW-FCP-NEXT: vmovaps %zmm0, (%r8) +; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm0 +; AVX512BW-FCP-NEXT: vmovdqa (%rdx), %xmm1 +; AVX512BW-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 +; AVX512BW-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 +; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm2 = [0,2,8,10,1,3,9,11] +; AVX512BW-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 +; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, (%r8) ; AVX512BW-FCP-NEXT: vzeroupper ; AVX512BW-FCP-NEXT: retq ; ; AVX512DQ-BW-LABEL: store_i64_stride4_vf2: ; AVX512DQ-BW: # %bb.0: -; AVX512DQ-BW-NEXT: vmovaps (%rdi), %xmm0 -; AVX512DQ-BW-NEXT: vmovaps (%rdx), %xmm1 -; AVX512DQ-BW-NEXT: vinsertf128 $1, (%rcx), %ymm1, %ymm1 -; AVX512DQ-BW-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0 -; AVX512DQ-BW-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512DQ-BW-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,1,3,5,7] -; AVX512DQ-BW-NEXT: vpermpd %zmm0, %zmm1, %zmm0 -; AVX512DQ-BW-NEXT: vmovaps %zmm0, (%r8) +; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm0 +; AVX512DQ-BW-NEXT: vmovdqa (%rdx), %xmm1 +; AVX512DQ-BW-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 +; AVX512DQ-BW-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 +; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} zmm2 = [0,2,8,10,1,3,9,11] +; AVX512DQ-BW-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 +; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, (%r8) ; AVX512DQ-BW-NEXT: vzeroupper ; AVX512DQ-BW-NEXT: retq ; ; AVX512DQ-BW-FCP-LABEL: store_i64_stride4_vf2: ; AVX512DQ-BW-FCP: # %bb.0: -; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %xmm0 -; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdx), %xmm1 -; AVX512DQ-BW-FCP-NEXT: vinsertf128 $1, (%rcx), %ymm1, %ymm1 -; AVX512DQ-BW-FCP-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0 -; AVX512DQ-BW-FCP-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512DQ-BW-FCP-NEXT: vmovaps {{.*#+}} zmm1 = [0,2,4,6,1,3,5,7] -; AVX512DQ-BW-FCP-NEXT: vpermpd %zmm0, %zmm1, %zmm0 -; AVX512DQ-BW-FCP-NEXT: vmovaps %zmm0, (%r8) +; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm0 +; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdx), %xmm1 +; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 +; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 +; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm2 = [0,2,8,10,1,3,9,11] +; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, (%r8) ; AVX512DQ-BW-FCP-NEXT: vzeroupper ; AVX512DQ-BW-FCP-NEXT: retq %in.vec0 = load <2 x i64>, ptr %in.vecptr0, align 64 -- GitLab From f1ca2a09671e4d4acc2bea362b39268ed7883b6d Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Tue, 12 Mar 2024 10:56:14 -0700 Subject: [PATCH 0028/1407] [ELF] Add --compress-section to compress matched non-SHF_ALLOC sections --compress-sections =[none|zlib|zstd] is similar to --compress-debug-sections but applies to broader sections without the SHF_ALLOC flag. lld will report an error if a SHF_ALLOC section is matched. An interesting use case is to compress `.strtab`/`.symtab`, which consume a significant portion of the file size (15.1% for a release build of Clang). An older revision is available at https://reviews.llvm.org/D154641 . This patch focuses on non-allocated sections for safety. Moving `maybeCompress` as D154641 does not handle STT_SECTION symbols for `-r --compress-debug-sections=zlib` (see `relocatable-section-symbol.s` from #66804). Since different output sections may use different compression algorithms, we need CompressedData::type to generalize config->compressDebugSections. GNU ld feature request: https://sourceware.org/bugzilla/show_bug.cgi?id=27452 Link: https://discourse.llvm.org/t/rfc-compress-arbitrary-sections-with-ld-lld-compress-sections/71674 Pull Request: https://github.com/llvm/llvm-project/pull/84855 --- lld/ELF/Config.h | 4 +- lld/ELF/Driver.cpp | 24 ++++- lld/ELF/Options.td | 4 + lld/ELF/OutputSections.cpp | 43 ++++++--- lld/ELF/OutputSections.h | 8 +- lld/docs/ReleaseNotes.rst | 4 + lld/docs/ld.lld.1 | 4 + lld/test/ELF/compress-sections-err.s | 3 + lld/test/ELF/compress-sections-special.s | 31 +++++++ lld/test/ELF/compress-sections.s | 91 +++++++++++++++++++ lld/test/ELF/linkerscript/compress-sections.s | 62 +++++++++++++ 11 files changed, 259 insertions(+), 19 deletions(-) create mode 100644 lld/test/ELF/compress-sections-special.s create mode 100644 lld/test/ELF/compress-sections.s create mode 100644 lld/test/ELF/linkerscript/compress-sections.s diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h index 691ebfc07432..9ae01eb90fa4 100644 --- a/lld/ELF/Config.h +++ b/lld/ELF/Config.h @@ -222,7 +222,9 @@ struct Config { CGProfileSortKind callGraphProfileSort; bool checkSections; bool checkDynamicRelocs; - llvm::DebugCompressionType compressDebugSections; + std::optional compressDebugSections; + llvm::SmallVector, 0> + compressSections; bool cref; llvm::SmallVector, 0> deadRelocInNonAlloc; diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp index de4b2e345ac9..2439d141fb66 100644 --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -1224,9 +1224,10 @@ static void readConfigs(opt::InputArgList &args) { config->checkSections = args.hasFlag(OPT_check_sections, OPT_no_check_sections, true); config->chroot = args.getLastArgValue(OPT_chroot); - config->compressDebugSections = getCompressionType( - args.getLastArgValue(OPT_compress_debug_sections, "none"), - "--compress-debug-sections"); + if (auto *arg = args.getLastArg(OPT_compress_debug_sections)) { + config->compressDebugSections = + getCompressionType(arg->getValue(), "--compress-debug-sections"); + } config->cref = args.hasArg(OPT_cref); config->optimizeBBJumps = args.hasFlag(OPT_optimize_bb_jumps, OPT_no_optimize_bb_jumps, false); @@ -1516,6 +1517,23 @@ static void readConfigs(opt::InputArgList &args) { } } + for (opt::Arg *arg : args.filtered(OPT_compress_sections)) { + SmallVector fields; + StringRef(arg->getValue()).split(fields, '='); + if (fields.size() != 2 || fields[1].empty()) { + error(arg->getSpelling() + + ": parse error, not 'section-glob=[none|zlib|zstd]'"); + continue; + } + auto type = getCompressionType(fields[1], arg->getSpelling()); + if (Expected pat = GlobPattern::create(fields[0])) { + config->compressSections.emplace_back(std::move(*pat), type); + } else { + error(arg->getSpelling() + ": " + toString(pat.takeError())); + continue; + } + } + for (opt::Arg *arg : args.filtered(OPT_z)) { std::pair option = StringRef(arg->getValue()).split('='); diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td index c10a73e2d9c3..3819b86238ea 100644 --- a/lld/ELF/Options.td +++ b/lld/ELF/Options.td @@ -67,6 +67,10 @@ defm compress_debug_sections: Eq<"compress-debug-sections", "Compress DWARF debug sections">, MetaVarName<"[none,zlib,zstd]">; +defm compress_sections: EEq<"compress-sections", + "Compress non-SHF_ALLOC output sections matching ">, + MetaVarName<"=[none|zlib|zstd]">; + defm defsym: Eq<"defsym", "Define a symbol alias">, MetaVarName<"=">; defm optimize_bb_jumps: BB<"optimize-bb-jumps", diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp index ee9374186787..55e6a14f103e 100644 --- a/lld/ELF/OutputSections.cpp +++ b/lld/ELF/OutputSections.cpp @@ -326,17 +326,30 @@ static SmallVector deflateShard(ArrayRef in, int level, } #endif -// Compress section contents if this section contains debug info. +// Compress certain non-SHF_ALLOC sections: +// +// * (if --compress-debug-sections is specified) non-empty .debug_* sections +// * (if --compress-sections is specified) matched sections template void OutputSection::maybeCompress() { using Elf_Chdr = typename ELFT::Chdr; (void)sizeof(Elf_Chdr); - // Compress only DWARF debug sections. - if (config->compressDebugSections == DebugCompressionType::None || - (flags & SHF_ALLOC) || !name.starts_with(".debug_") || size == 0) + DebugCompressionType ctype = DebugCompressionType::None; + for (auto &[glob, t] : config->compressSections) + if (glob.match(name)) + ctype = t; + if (!(flags & SHF_ALLOC) && config->compressDebugSections && + name.starts_with(".debug_") && size) + ctype = *config->compressDebugSections; + if (ctype == DebugCompressionType::None) + return; + if (flags & SHF_ALLOC) { + errorOrWarn("--compress-sections: section '" + name + + "' with the SHF_ALLOC flag cannot be compressed"); return; + } - llvm::TimeTraceScope timeScope("Compress debug sections"); + llvm::TimeTraceScope timeScope("Compress sections"); compressed.uncompressedSize = size; auto buf = std::make_unique(size); // Write uncompressed data to a temporary zero-initialized buffer. @@ -344,14 +357,21 @@ template void OutputSection::maybeCompress() { parallel::TaskGroup tg; writeTo(buf.get(), tg); } + // The generic ABI specifies "The sh_size and sh_addralign fields of the + // section header for a compressed section reflect the requirements of the + // compressed section." However, 1-byte alignment has been wildly accepted + // and utilized for a long time. Removing alignment padding is particularly + // useful when there are many compressed output sections. + addralign = 1; #if LLVM_ENABLE_ZSTD // Use ZSTD's streaming compression API which permits parallel workers working // on the stream. See http://facebook.github.io/zstd/zstd_manual.html // "Streaming compression - HowTo". - if (config->compressDebugSections == DebugCompressionType::Zstd) { + if (ctype == DebugCompressionType::Zstd) { // Allocate a buffer of half of the input size, and grow it by 1.5x if // insufficient. + compressed.type = ELFCOMPRESS_ZSTD; compressed.shards = std::make_unique[]>(1); SmallVector &out = compressed.shards[0]; out.resize_for_overwrite(std::max(size / 2, 32)); @@ -424,6 +444,7 @@ template void OutputSection::maybeCompress() { } size += 4; // checksum + compressed.type = ELFCOMPRESS_ZLIB; compressed.shards = std::move(shardsOut); compressed.numShards = numShards; compressed.checksum = checksum; @@ -450,20 +471,18 @@ void OutputSection::writeTo(uint8_t *buf, parallel::TaskGroup &tg) { if (type == SHT_NOBITS) return; - // If --compress-debug-section is specified and if this is a debug section, - // we've already compressed section contents. If that's the case, - // just write it down. + // If the section is compressed due to + // --compress-debug-section/--compress-sections, the content is already known. if (compressed.shards) { auto *chdr = reinterpret_cast(buf); + chdr->ch_type = compressed.type; chdr->ch_size = compressed.uncompressedSize; chdr->ch_addralign = addralign; buf += sizeof(*chdr); - if (config->compressDebugSections == DebugCompressionType::Zstd) { - chdr->ch_type = ELFCOMPRESS_ZSTD; + if (compressed.type == ELFCOMPRESS_ZSTD) { memcpy(buf, compressed.shards[0].data(), compressed.shards[0].size()); return; } - chdr->ch_type = ELFCOMPRESS_ZLIB; // Compute shard offsets. auto offsets = std::make_unique(compressed.numShards); diff --git a/lld/ELF/OutputSections.h b/lld/ELF/OutputSections.h index c7931471a6ed..421a0181feb5 100644 --- a/lld/ELF/OutputSections.h +++ b/lld/ELF/OutputSections.h @@ -23,6 +23,7 @@ struct PhdrEntry; struct CompressedData { std::unique_ptr[]> shards; + uint32_t type = 0; uint32_t numShards = 0; uint32_t checksum = 0; uint64_t uncompressedSize; @@ -116,12 +117,13 @@ public: void sortInitFini(); void sortCtorsDtors(); + // Used for implementation of --compress-debug-sections and + // --compress-sections. + CompressedData compressed; + private: SmallVector storage; - // Used for implementation of --compress-debug-sections option. - CompressedData compressed; - std::array getFiller(); }; diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst index 6f60efd87c97..97ed06048910 100644 --- a/lld/docs/ReleaseNotes.rst +++ b/lld/docs/ReleaseNotes.rst @@ -26,6 +26,10 @@ Non-comprehensive list of changes in this release ELF Improvements ---------------- +* ``--compress-sections =[none|zlib|zstd]`` is added to compress + matched output sections without the ``SHF_ALLOC`` flag. + (`#84855 `_) + Breaking changes ---------------- diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1 index e4d39e47f5c5..e759776c8d55 100644 --- a/lld/docs/ld.lld.1 +++ b/lld/docs/ld.lld.1 @@ -164,6 +164,10 @@ to set the compression level to 6. The compression level is 5. .El .Pp +.It Fl -compress-sections Ns = Ns Ar section-glob=[none|zlib|zstd] +Compress output sections that match the glob and do not have the SHF_ALLOC flag. +This is like a generalized +.Cm --compress-debug-sections. .It Fl -cref Output cross reference table. If .Fl Map diff --git a/lld/test/ELF/compress-sections-err.s b/lld/test/ELF/compress-sections-err.s index 097803807083..1b46aea12e9c 100644 --- a/lld/test/ELF/compress-sections-err.s +++ b/lld/test/ELF/compress-sections-err.s @@ -5,8 +5,11 @@ # RUN: ld.lld %t.o --compress-debug-sections=zlib --compress-debug-sections=none -o /dev/null 2>&1 | count 0 # RUN: not ld.lld %t.o --compress-debug-sections=zlib -o /dev/null 2>&1 | \ # RUN: FileCheck %s --implicit-check-not=error: +# RUN: not ld.lld %t.o --compress-sections=foo=zlib -o /dev/null 2>&1 | \ +# RUN: FileCheck %s --check-prefix=CHECK2 --implicit-check-not=error: # CHECK: error: --compress-debug-sections: LLVM was not built with LLVM_ENABLE_ZLIB or did not find zlib at build time +# CHECK2: error: --compress-sections: LLVM was not built with LLVM_ENABLE_ZLIB or did not find zlib at build time .globl _start _start: diff --git a/lld/test/ELF/compress-sections-special.s b/lld/test/ELF/compress-sections-special.s new file mode 100644 index 000000000000..80c61fe626a4 --- /dev/null +++ b/lld/test/ELF/compress-sections-special.s @@ -0,0 +1,31 @@ +# REQUIRES: x86, zlib + +# RUN: rm -rf %t && mkdir %t && cd %t +# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o a.o +# RUN: ld.lld -pie a.o --compress-sections .strtab=zlib --compress-sections .symtab=zlib -o out +# RUN: llvm-readelf -Ss -x .strtab out 2>&1 | FileCheck %s + +# CHECK: nonalloc0 PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00 0 0 1 +# CHECK: .symtab SYMTAB 0000000000000000 [[#%x,]] [[#%x,]] 18 C 12 3 1 +# CHECK-NEXT: .shstrtab STRTAB 0000000000000000 [[#%x,]] [[#%x,]] 00 0 0 1 +# CHECK-NEXT: .strtab STRTAB 0000000000000000 [[#%x,]] [[#%x,]] 00 C 0 0 1 + +## TODO Add compressed SHT_STRTAB/SHT_SYMTAB support to llvm-readelf +# CHECK: warning: {{.*}}: unable to get the string table for the SHT_SYMTAB section: SHT_STRTAB string table section + +# CHECK: Hex dump of section '.strtab': +# CHECK-NEXT: 01000000 00000000 1a000000 00000000 +# CHECK-NEXT: 01000000 00000000 {{.*}} + +# RUN: not ld.lld -shared a.o --compress-sections .dynstr=zlib 2>&1 | FileCheck %s --check-prefix=ERR-ALLOC +# ERR-ALLOC: error: --compress-sections: section '.dynstr' with the SHF_ALLOC flag cannot be compressed + +.globl _start, g0, g1 +_start: +l0: +g0: +g1: + +.section nonalloc0,"" +.quad .text+1 +.quad .text+2 diff --git a/lld/test/ELF/compress-sections.s b/lld/test/ELF/compress-sections.s new file mode 100644 index 000000000000..59b5408c9624 --- /dev/null +++ b/lld/test/ELF/compress-sections.s @@ -0,0 +1,91 @@ +# REQUIRES: x86, zlib, zstd + +# RUN: rm -rf %t && mkdir %t && cd %t +# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o a.o +# RUN: ld.lld -pie a.o -o out --compress-sections '*0=zlib' --compress-sections '*0=none' --compress-sections 'nomatch=none' +# RUN: llvm-readelf -SrsX out | FileCheck %s --check-prefix=CHECK1 + +# CHECK1: Name Type Address Off Size ES Flg Lk Inf Al +# CHECK1: foo0 PROGBITS [[#%x,FOO0:]] [[#%x,]] [[#%x,]] 00 A 0 0 8 +# CHECK1-NEXT: foo1 PROGBITS [[#%x,FOO1:]] [[#%x,]] [[#%x,]] 00 A 0 0 8 +# CHECK1-NEXT: .text PROGBITS [[#%x,TEXT:]] [[#%x,]] [[#%x,]] 00 AX 0 0 4 +# CHECK1: nonalloc0 PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00 0 0 8 +# CHECK1-NEXT: nonalloc1 PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00 0 0 8 +# CHECK1-NEXT: .debug_str PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 01 MS 0 0 1 + +# CHECK1: 0000000000000010 0 NOTYPE LOCAL DEFAULT [[#]] (nonalloc0) sym0 +# CHECK1: 0000000000000008 0 NOTYPE LOCAL DEFAULT [[#]] (nonalloc1) sym1 + +# RUN: ld.lld -pie a.o --compress-sections '*c0=zlib' --compress-sections .debug_str=zstd -o out2 +# RUN: llvm-readelf -SrsX -x nonalloc0 -x .debug_str out2 | FileCheck %s --check-prefix=CHECK2 + +# CHECK2: Name Type Address Off Size ES Flg Lk Inf Al +# CHECK2: foo0 PROGBITS [[#%x,FOO0:]] [[#%x,]] [[#%x,]] 00 A 0 0 8 +# CHECK2-NEXT: foo1 PROGBITS [[#%x,FOO1:]] [[#%x,]] [[#%x,]] 00 A 0 0 8 +# CHECK2-NEXT: .text PROGBITS [[#%x,TEXT:]] [[#%x,]] [[#%x,]] 00 AX 0 0 4 +# CHECK2: nonalloc0 PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00 C 0 0 1 +# CHECK2-NEXT: nonalloc1 PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00 0 0 8 +# CHECK2-NEXT: .debug_str PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 01 MSC 0 0 1 + +# CHECK2: 0000000000000010 0 NOTYPE LOCAL DEFAULT [[#]] (nonalloc0) sym0 +# CHECK2: 0000000000000008 0 NOTYPE LOCAL DEFAULT [[#]] (nonalloc1) sym1 + +# CHECK2: Hex dump of section 'nonalloc0': +## zlib with ch_size=0x10 +# CHECK2-NEXT: 01000000 00000000 10000000 00000000 +# CHECK2-NEXT: 01000000 00000000 {{.*}} +# CHECK2: Hex dump of section '.debug_str': +## zstd with ch_size=0x38 +# CHECK2-NEXT: 02000000 00000000 38000000 00000000 +# CHECK2-NEXT: 01000000 00000000 {{.*}} + +## --compress-debug-sections=none takes precedence. +# RUN: ld.lld a.o --compress-debug-sections=none --compress-sections .debug_str=zstd -o out3 +# RUN: llvm-readelf -S out3 | FileCheck %s --check-prefix=CHECK3 + +# CHECK3: .debug_str PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 01 MS 0 0 1 + +# RUN: not ld.lld a.o --compress-sections '*0=zlib' 2>&1 | \ +# RUN: FileCheck %s --check-prefix=ERR-ALLOC --implicit-check-not=error: +# ERR-ALLOC: error: --compress-sections: section 'foo0' with the SHF_ALLOC flag cannot be compressed + +# RUN: not ld.lld --compress-sections=foo a.o 2>&1 | \ +# RUN: FileCheck %s --check-prefix=ERR1 --implicit-check-not=error: +# ERR1: error: --compress-sections: parse error, not 'section-glob=[none|zlib|zstd]' + +# RUN: not ld.lld --compress-sections 'a[=zlib' a.o 2>&1 | \ +# RUN: FileCheck %s --check-prefix=ERR2 --implicit-check-not=error: +# ERR2: error: --compress-sections: invalid glob pattern, unmatched '[' + +# RUN: not ld.lld a.o --compress-sections='.debug*=zlib-gabi' --compress-sections='.debug*=' 2>&1 | \ +# RUN: FileCheck -check-prefix=ERR3 %s +# ERR3: unknown --compress-sections value: zlib-gabi +# ERR3-NEXT: --compress-sections: parse error, not 'section-glob=[none|zlib|zstd]' + +.globl _start +_start: + ret + +.section foo0,"a" +.balign 8 +.quad .text-. +.quad .text-. +.section foo1,"a" +.balign 8 +.quad .text-. +.quad .text-. +.section nonalloc0,"" +.balign 8 +.quad .text+1 +.quad .text+2 +sym0: +.section nonalloc1,"" +.balign 8 +.quad 42 +sym1: + +.section .debug_str,"MS",@progbits,1 +.Linfo_string0: + .asciz "AAAAAAAAAAAAAAAAAAAAAAAAAAA" +.Linfo_string1: + .asciz "BBBBBBBBBBBBBBBBBBBBBBBBBBB" diff --git a/lld/test/ELF/linkerscript/compress-sections.s b/lld/test/ELF/linkerscript/compress-sections.s new file mode 100644 index 000000000000..9b4574a1778c --- /dev/null +++ b/lld/test/ELF/linkerscript/compress-sections.s @@ -0,0 +1,62 @@ +# REQUIRES: x86, zlib + +# RUN: rm -rf %t && split-file %s %t && cd %t +# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o +# RUN: ld.lld -T a.lds a.o --compress-sections nonalloc=zlib --compress-sections str=zlib -o out +# RUN: llvm-readelf -SsXz -p str out | FileCheck %s + +# CHECK: Name Type Address Off Size ES Flg Lk Inf Al +# CHECK: nonalloc PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00 C 0 0 1 +# CHECK-NEXT: str PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 01 MSC 0 0 1 + +# CHECK: 0000000000000000 0 NOTYPE GLOBAL DEFAULT [[#]] (nonalloc) nonalloc_start +# CHECK: 0000000000000023 0 NOTYPE GLOBAL DEFAULT [[#]] (nonalloc) nonalloc_end +# CHECK: String dump of section 'str': +# CHECK-NEXT: [ 0] AAA +# CHECK-NEXT: [ 4] BBB + +## TODO The uncompressed size of 'nonalloc' is dependent on linker script +## commands, which is not handled. We should report an error. +# RUN: ld.lld -T b.lds a.o --compress-sections nonalloc=zlib + +#--- a.s +.globl _start +_start: + ret + +.section nonalloc0,"" +.balign 8 +.quad .text +.quad .text +.section nonalloc1,"" +.balign 8 +.quad 42 + +.section str,"MS",@progbits,1 + .asciz "AAA" + .asciz "BBB" + +#--- a.lds +SECTIONS { + .text : { *(.text) } + c = SIZEOF(.text); + b = c+1; + a = b+1; + nonalloc : { + nonalloc_start = .; +## In general, using data commands is error-prone. This case is correct, though. + *(nonalloc*) QUAD(SIZEOF(.text)) + . += a; + nonalloc_end = .; + } + str : { *(str) } +} + +#--- b.lds +SECTIONS { + nonalloc : { *(nonalloc*) . += a; } + .text : { *(.text) } + a = b+1; + b = c+1; + c = SIZEOF(.text); +} -- GitLab From 536e0ebaaa842471ae91bbda4f8cc1821690861e Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Tue, 12 Mar 2024 11:06:23 -0700 Subject: [PATCH 0029/1407] Remove XFAIL from tests passing on green dragon --- .../debuginfo-tests/llgdb-tests/static-member-2.cpp | 1 - .../debuginfo-tests/llgdb-tests/static-member.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/cross-project-tests/debuginfo-tests/llgdb-tests/static-member-2.cpp b/cross-project-tests/debuginfo-tests/llgdb-tests/static-member-2.cpp index 3f11ae018fc8..5b6647c0631c 100644 --- a/cross-project-tests/debuginfo-tests/llgdb-tests/static-member-2.cpp +++ b/cross-project-tests/debuginfo-tests/llgdb-tests/static-member-2.cpp @@ -2,7 +2,6 @@ // RUN: %clangxx %target_itanium_abi_host_triple %t -o %t.out // RUN: %test_debuginfo %s %t.out // XFAIL: gdb-clang-incompatibility -// XFAIL: system-darwin // DEBUGGER: delete breakpoints // DEBUGGER: break static-member.cpp:33 diff --git a/cross-project-tests/debuginfo-tests/llgdb-tests/static-member.cpp b/cross-project-tests/debuginfo-tests/llgdb-tests/static-member.cpp index 57316dfd6404..29dd84dc8325 100644 --- a/cross-project-tests/debuginfo-tests/llgdb-tests/static-member.cpp +++ b/cross-project-tests/debuginfo-tests/llgdb-tests/static-member.cpp @@ -2,7 +2,6 @@ // RUN: %clangxx %target_itanium_abi_host_triple %t -o %t.out // RUN: %test_debuginfo %s %t.out // XFAIL: !system-darwin && gdb-clang-incompatibility -// XFAIL: system-darwin // DEBUGGER: delete breakpoints // DEBUGGER: break static-member.cpp:33 // DEBUGGER: r -- GitLab From 42ecccfe346daf342b6c46a6a471ba5ed99b1139 Mon Sep 17 00:00:00 2001 From: amilendra Date: Tue, 12 Mar 2024 18:17:13 +0000 Subject: [PATCH 0030/1407] [libcxx] Fix incorrect type in the has-1024-bit-atomics feature test (#84904) --- libcxx/utils/libcxx/test/features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py index 4fd8798b794a..872bff372b3d 100644 --- a/libcxx/utils/libcxx/test/features.py +++ b/libcxx/utils/libcxx/test/features.py @@ -176,7 +176,7 @@ DEFAULT_FEATURES = [ cfg, """ #include - struct Large { int storage[1024/8]; }; + struct Large { char storage[1024/8]; }; std::atomic x; int main(int, char**) { (void)x.load(); (void)x.is_lock_free(); return 0; } """, -- GitLab From a843f26a77dee7900891b6748d43cf8d4e423bfe Mon Sep 17 00:00:00 2001 From: David Blaikie Date: Tue, 12 Mar 2024 11:25:12 -0700 Subject: [PATCH 0031/1407] [NFC] SLVectorizer comparator refactoring that preserves behavior (#84966) Spinning off from #79321 / 35f4592 - looked like the comparator could be simplified & made more clear/less risk of leaving hidden bugs. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 87 +++++++++---------- 1 file changed, 43 insertions(+), 44 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 7b99c3ac8c55..6ef46e3c5258 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -16614,36 +16614,11 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) { if (Opcodes1.size() > Opcodes2.size()) return false; for (int I = 0, E = Opcodes1.size(); I < E; ++I) { - // Undefs are compatible with any other value. - if (isa(Opcodes1[I]) || isa(Opcodes2[I])) { - if (isa(Opcodes1[I]) && isa(Opcodes2[I])) - continue; - if (isa(Opcodes1[I])) { - assert(isa(Opcodes2[I]) && "Expected 2nd undef value"); - return true; - } - if (isa(Opcodes2[I])) { - assert(isa(Opcodes1[I]) && "Expected 1st undef value"); - return false; - } - if (isa(Opcodes1[I]) && !isa(Opcodes1[I])) { - assert(isa(Opcodes2[I]) && "Expected 2nd undef value"); - return true; - } - if (isa(Opcodes2[I]) && !isa(Opcodes2[I])) { - assert(isa(Opcodes1[I]) && "Expected 1st undef value"); - return false; - } - if (!isa(Opcodes2[I])) { - assert(isa(Opcodes1[I]) && "Expected 1st undef value"); - return false; - } - assert(!isa(Opcodes1[I]) && isa(Opcodes2[I]) && - "Expected 1st non-undef and 2nd undef value"); - return true; - } - if (auto *I1 = dyn_cast(Opcodes1[I])) - if (auto *I2 = dyn_cast(Opcodes2[I])) { + { + // Instructions come first. + auto *I1 = dyn_cast(Opcodes1[I]); + auto *I2 = dyn_cast(Opcodes2[I]); + if (I1 && I2) { DomTreeNodeBase *NodeI1 = DT->getNode(I1->getParent()); DomTreeNodeBase *NodeI2 = DT->getNode(I2->getParent()); if (!NodeI1) @@ -16660,20 +16635,44 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) { continue; return I1->getOpcode() < I2->getOpcode(); } - if (isa(Opcodes1[I]) && isa(Opcodes2[I])) - continue; - if (isa(Opcodes1[I]) && !isa(Opcodes2[I])) - return true; - if (!isa(Opcodes1[I]) && isa(Opcodes2[I])) - return false; - if (isa(Opcodes1[I]) && !isa(Opcodes2[I])) - return true; - if (!isa(Opcodes1[I]) && isa(Opcodes2[I])) - return false; - if (Opcodes1[I]->getValueID() < Opcodes2[I]->getValueID()) - return true; - if (Opcodes1[I]->getValueID() > Opcodes2[I]->getValueID()) - return false; + if (I1) + return true; + if (I2) + return false; + } + { + // Non-undef constants come next. + bool C1 = isa(Opcodes1[I]) && !isa(Opcodes1[I]); + bool C2 = isa(Opcodes2[I]) && !isa(Opcodes2[I]); + if (C1 && C2) + continue; + if (C1) + return true; + if (C2) + return false; + } + bool U1 = isa(Opcodes1[I]); + bool U2 = isa(Opcodes2[I]); + { + // Non-constant non-instructions come next. + if (!U1 && !U2) { + auto ValID1 = Opcodes1[I]->getValueID(); + auto ValID2 = Opcodes2[I]->getValueID(); + if (ValID1 == ValID2) + continue; + if (ValID1 < ValID2) + return true; + if (ValID1 > ValID2) + return false; + } + if (!U1) + return true; + if (!U2) + return false; + } + // Undefs come last. + assert(U1 && U2); + continue; } return false; }; -- GitLab From 377da51546b2f514c3e90fca351004a2cf3f1eed Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Mon, 11 Mar 2024 22:52:20 -0500 Subject: [PATCH 0032/1407] [InstCombine] Add test for detecting `(x ^ -x)` as a ~Mask; NFC --- .../InstCombine/icmp-and-lowbit-mask.ll | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll b/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll index 640a95b05616..903d70685ab5 100644 --- a/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll +++ b/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll @@ -453,6 +453,25 @@ define i1 @src_is_notmask_shl(i8 %x_in, i8 %y, i1 %cond) { ret i1 %r } +define i1 @src_is_notmask_x_xor_neg_x(i8 %x_in, i8 %y, i1 %cond) { +; CHECK-LABEL: @src_is_notmask_x_xor_neg_x( +; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 +; CHECK-NEXT: [[NEG_Y:%.*]] = sub i8 0, [[Y:%.*]] +; CHECK-NEXT: [[NOTMASK0:%.*]] = xor i8 [[NEG_Y]], [[Y]] +; CHECK-NEXT: [[NOTMASK:%.*]] = select i1 [[COND:%.*]], i8 [[NOTMASK0]], i8 -8 +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X]], [[NOTMASK]] +; CHECK-NEXT: [[R:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: ret i1 [[R]] +; + %x = xor i8 %x_in, 123 + %neg_y = sub i8 0, %y + %nmask0 = xor i8 %y, %neg_y + %notmask = select i1 %cond, i8 %nmask0, i8 -8 + %and = and i8 %x, %notmask + %r = icmp eq i8 %and, 0 + ret i1 %r +} + define i1 @src_is_notmask_shl_fail_multiuse_invert(i8 %x_in, i8 %y, i1 %cond) { ; CHECK-LABEL: @src_is_notmask_shl_fail_multiuse_invert( ; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 122 -- GitLab From 5ca325e49cedee2aa5e80581ba95dcab56292c32 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Mon, 11 Mar 2024 22:52:29 -0500 Subject: [PATCH 0033/1407] [InstCombine] Detect `(x ^ -x)` as a ~Mask Proof: https://alive2.llvm.org/ce/z/TAFmPw This is a lemma for clearing up some of the regressions that #84688 causes. Closes #84868 --- llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp | 7 +++++-- llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll | 7 +++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp index e71f3e113b96..0dce0077bf15 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -4113,9 +4113,12 @@ static bool isMaskOrZero(const Value *V, bool Not, const SimplifyQuery &Q, if (match(V, m_Not(m_Value(X)))) return isMaskOrZero(X, !Not, Q, Depth); + // (X ^ -X) is a ~Mask + if (Not) + return match(V, m_c_Xor(m_Value(X), m_Neg(m_Deferred(X)))); // (X ^ (X - 1)) is a Mask - return !Not && - match(V, m_c_Xor(m_Value(X), m_Add(m_Deferred(X), m_AllOnes()))); + else + return match(V, m_c_Xor(m_Value(X), m_Add(m_Deferred(X), m_AllOnes()))); case Instruction::Select: // c ? Mask0 : Mask1 is a Mask. return isMaskOrZero(I->getOperand(1), Not, Q, Depth) && diff --git a/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll b/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll index 903d70685ab5..070609228958 100644 --- a/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll +++ b/llvm/test/Transforms/InstCombine/icmp-and-lowbit-mask.ll @@ -456,11 +456,10 @@ define i1 @src_is_notmask_shl(i8 %x_in, i8 %y, i1 %cond) { define i1 @src_is_notmask_x_xor_neg_x(i8 %x_in, i8 %y, i1 %cond) { ; CHECK-LABEL: @src_is_notmask_x_xor_neg_x( ; CHECK-NEXT: [[X:%.*]] = xor i8 [[X_IN:%.*]], 123 -; CHECK-NEXT: [[NEG_Y:%.*]] = sub i8 0, [[Y:%.*]] +; CHECK-NEXT: [[NEG_Y:%.*]] = add i8 [[Y:%.*]], -1 ; CHECK-NEXT: [[NOTMASK0:%.*]] = xor i8 [[NEG_Y]], [[Y]] -; CHECK-NEXT: [[NOTMASK:%.*]] = select i1 [[COND:%.*]], i8 [[NOTMASK0]], i8 -8 -; CHECK-NEXT: [[AND:%.*]] = and i8 [[X]], [[NOTMASK]] -; CHECK-NEXT: [[R:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[COND:%.*]], i8 [[NOTMASK0]], i8 7 +; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[X]], [[TMP3]] ; CHECK-NEXT: ret i1 [[R]] ; %x = xor i8 %x_in, 123 -- GitLab From 9ac03158987802706110ef465c4b6a7553cc4b86 Mon Sep 17 00:00:00 2001 From: David Blaikie Date: Tue, 12 Mar 2024 18:28:30 +0000 Subject: [PATCH 0034/1407] Add comment to assert from a843f26 --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 6ef46e3c5258..b8b67609d755 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -16671,7 +16671,7 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) { return false; } // Undefs come last. - assert(U1 && U2); + assert(U1 && U2 && "The only thing left should be undef & undef."); continue; } return false; -- GitLab From f5334f5da5166ac26fb24ca44669b3d425f0d131 Mon Sep 17 00:00:00 2001 From: Jonathan Peyton Date: Tue, 12 Mar 2024 11:36:19 -0700 Subject: [PATCH 0035/1407] [OpenMP] Add debug checks for divide by zero (#83300) --- openmp/runtime/src/kmp_affinity.cpp | 2 ++ openmp/runtime/src/kmp_sched.cpp | 23 ++++++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp index b79b57eafd6a..c3ee4de75a23 100644 --- a/openmp/runtime/src/kmp_affinity.cpp +++ b/openmp/runtime/src/kmp_affinity.cpp @@ -1777,6 +1777,8 @@ static bool __kmp_affinity_create_hwloc_map(kmp_i18n_id_t *const msg_id) { __kmp_nThreadsPerCore = __kmp_hwloc_get_nobjs_under_obj(o, HWLOC_OBJ_PU); else __kmp_nThreadsPerCore = 1; // no CORE found + if (__kmp_nThreadsPerCore == 0) + __kmp_nThreadsPerCore = 1; __kmp_ncores = __kmp_xproc / __kmp_nThreadsPerCore; if (nCoresPerPkg == 0) nCoresPerPkg = 1; // to prevent possible division by 0 diff --git a/openmp/runtime/src/kmp_sched.cpp b/openmp/runtime/src/kmp_sched.cpp index 53182bef5873..4d764e441f28 100644 --- a/openmp/runtime/src/kmp_sched.cpp +++ b/openmp/runtime/src/kmp_sched.cpp @@ -52,6 +52,7 @@ char const *traits_t::spec = "ld"; } else if (i > 0) { \ t = (u - l) / i + 1; \ } else { \ + KMP_DEBUG_ASSERT(i != 0); \ t = (l - u) / (-i) + 1; \ } \ KMP_COUNT_VALUE(stat, t); \ @@ -284,6 +285,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid, // upper-lower can exceed the limit of signed type trip_count = (UT)(*pupper - *plower) / incr + 1; } else { + KMP_DEBUG_ASSERT(incr != 0); trip_count = (UT)(*plower - *pupper) / (-incr) + 1; } @@ -318,6 +320,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid, if (plastiter != NULL) *plastiter = (tid == trip_count - 1); } else { + KMP_DEBUG_ASSERT(nth != 0); if (__kmp_static == kmp_sch_static_balanced) { UT small_chunk = trip_count / nth; UT extras = trip_count % nth; @@ -358,6 +361,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid, case kmp_sch_static_chunked: { ST span; UT nchunks; + KMP_DEBUG_ASSERT(chunk != 0); if (chunk < 1) chunk = 1; else if ((UT)chunk > trip_count) @@ -383,6 +387,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid, } case kmp_sch_static_balanced_chunked: { T old_upper = *pupper; + KMP_DEBUG_ASSERT(nth != 0); // round up to make sure the chunk is enough to cover all iterations UT span = (trip_count + nth - 1) / nth; @@ -398,8 +403,10 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid, } else if (*pupper < old_upper) *pupper = old_upper; - if (plastiter != NULL) + if (plastiter != NULL) { + KMP_DEBUG_ASSERT(chunk != 0); *plastiter = (tid == ((trip_count - 1) / (UT)chunk)); + } break; } default: @@ -417,6 +424,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid, // Calculate chunk in case it was not specified; it is specified for // kmp_sch_static_chunked if (schedtype == kmp_sch_static) { + KMP_DEBUG_ASSERT(nth != 0); cur_chunk = trip_count / nth + ((trip_count % nth) ? 1 : 0); } // 0 - "static" schedule @@ -547,6 +555,7 @@ static void __kmp_dist_for_static_init(ident_t *loc, kmp_int32 gtid, // upper-lower can exceed the limit of signed type trip_count = (UT)(*pupper - *plower) / incr + 1; } else { + KMP_DEBUG_ASSERT(incr != 0); trip_count = (UT)(*plower - *pupper) / (-incr) + 1; } @@ -568,6 +577,7 @@ static void __kmp_dist_for_static_init(ident_t *loc, kmp_int32 gtid, *plastiter = (tid == 0 && team_id == trip_count - 1); } else { // Get the team's chunk first (each team gets at most one chunk) + KMP_DEBUG_ASSERT(nteams != 0); if (__kmp_static == kmp_sch_static_balanced) { UT chunkD = trip_count / nteams; UT extras = trip_count % nteams; @@ -619,6 +629,7 @@ static void __kmp_dist_for_static_init(ident_t *loc, kmp_int32 gtid, // upper-lower can exceed the limit of signed type trip_count = (UT)(*pupperDist - *plower) / incr + 1; } else { + KMP_DEBUG_ASSERT(incr != 0); trip_count = (UT)(*plower - *pupperDist) / (-incr) + 1; } KMP_DEBUG_ASSERT(trip_count); @@ -637,6 +648,7 @@ static void __kmp_dist_for_static_init(ident_t *loc, kmp_int32 gtid, if (*plastiter != 0 && !(tid == trip_count - 1)) *plastiter = 0; } else { + KMP_DEBUG_ASSERT(nth != 0); if (__kmp_static == kmp_sch_static_balanced) { UT chunkL = trip_count / nth; UT extras = trip_count % nth; @@ -684,9 +696,11 @@ static void __kmp_dist_for_static_init(ident_t *loc, kmp_int32 gtid, *pstride = span * nth; *plower = *plower + (span * tid); *pupper = *plower + span - incr; - if (plastiter != NULL) + if (plastiter != NULL) { + KMP_DEBUG_ASSERT(chunk != 0); if (*plastiter != 0 && !(tid == ((trip_count - 1) / (UT)chunk) % nth)) *plastiter = 0; + } break; } default: @@ -809,6 +823,7 @@ static void __kmp_team_static_init(ident_t *loc, kmp_int32 gtid, // upper-lower can exceed the limit of signed type trip_count = (UT)(upper - lower) / incr + 1; } else { + KMP_DEBUG_ASSERT(incr != 0); trip_count = (UT)(lower - upper) / (-incr) + 1; } if (chunk < 1) @@ -817,8 +832,10 @@ static void __kmp_team_static_init(ident_t *loc, kmp_int32 gtid, *p_st = span * nteams; *p_lb = lower + (span * team_id); *p_ub = *p_lb + span - incr; - if (p_last != NULL) + if (p_last != NULL) { + KMP_DEBUG_ASSERT(chunk != 0); *p_last = (team_id == ((trip_count - 1) / (UT)chunk) % nteams); + } // Correct upper bound if needed if (incr > 0) { if (*p_ub < *p_lb) // overflow? -- GitLab From 3303be63fc2ac196568b03f58c146655e19183f6 Mon Sep 17 00:00:00 2001 From: Jonathan Peyton Date: Tue, 12 Mar 2024 11:36:43 -0700 Subject: [PATCH 0036/1407] [OpenMP] Make sure mask is set to nullptr (#83299) --- openmp/runtime/src/kmp.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h index 48d7124e56c5..de758d37269d 100644 --- a/openmp/runtime/src/kmp.h +++ b/openmp/runtime/src/kmp.h @@ -825,7 +825,7 @@ class kmp_affinity_raii_t { public: kmp_affinity_raii_t(const kmp_affin_mask_t *new_mask = nullptr) - : restored(false) { + : mask(nullptr), restored(false) { if (KMP_AFFINITY_CAPABLE()) { KMP_CPU_ALLOC(mask); KMP_ASSERT(mask != NULL); @@ -835,7 +835,7 @@ public: } } void restore() { - if (!restored && KMP_AFFINITY_CAPABLE()) { + if (mask && KMP_AFFINITY_CAPABLE() && !restored) { __kmp_set_system_affinity(mask, /*abort_on_error=*/true); KMP_CPU_FREE(mask); } -- GitLab From 6272500e0b1456a87256f0c8659fdc86cfe3ef9a Mon Sep 17 00:00:00 2001 From: Jonathan Peyton Date: Tue, 12 Mar 2024 11:37:01 -0700 Subject: [PATCH 0037/1407] [OpenMP] Remove unused logical/physical CPUID information (#83298) --- openmp/runtime/src/kmp.h | 2 - openmp/runtime/src/kmp_utility.cpp | 68 +----------------------------- 2 files changed, 1 insertion(+), 69 deletions(-) diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h index de758d37269d..569a1ab9b477 100644 --- a/openmp/runtime/src/kmp.h +++ b/openmp/runtime/src/kmp.h @@ -1392,8 +1392,6 @@ typedef struct kmp_cpuinfo { int stepping; // CPUID(1).EAX[3:0] ( Stepping ) kmp_cpuinfo_flags_t flags; int apic_id; - int physical_id; - int logical_id; kmp_uint64 frequency; // Nominal CPU frequency in Hz. char name[3 * sizeof(kmp_cpuid_t)]; // CPUID(0x80000002,0x80000003,0x80000004) } kmp_cpuinfo_t; diff --git a/openmp/runtime/src/kmp_utility.cpp b/openmp/runtime/src/kmp_utility.cpp index f901eaca92f4..bfa450c9ced2 100644 --- a/openmp/runtime/src/kmp_utility.cpp +++ b/openmp/runtime/src/kmp_utility.cpp @@ -28,68 +28,6 @@ static const char *unknown = "unknown"; static int trace_level = 5; #endif -/* LOG_ID_BITS = ( 1 + floor( log_2( max( log_per_phy - 1, 1 )))) - * APIC_ID = (PHY_ID << LOG_ID_BITS) | LOG_ID - * PHY_ID = APIC_ID >> LOG_ID_BITS - */ -int __kmp_get_physical_id(int log_per_phy, int apic_id) { - int index_lsb, index_msb, temp; - - if (log_per_phy > 1) { - index_lsb = 0; - index_msb = 31; - - temp = log_per_phy; - while ((temp & 1) == 0) { - temp >>= 1; - index_lsb++; - } - - temp = log_per_phy; - while ((temp & 0x80000000) == 0) { - temp <<= 1; - index_msb--; - } - - /* If >1 bits were set in log_per_phy, choose next higher power of 2 */ - if (index_lsb != index_msb) - index_msb++; - - return ((int)(apic_id >> index_msb)); - } - - return apic_id; -} - -/* - * LOG_ID_BITS = ( 1 + floor( log_2( max( log_per_phy - 1, 1 )))) - * APIC_ID = (PHY_ID << LOG_ID_BITS) | LOG_ID - * LOG_ID = APIC_ID & (( 1 << LOG_ID_BITS ) - 1 ) - */ -int __kmp_get_logical_id(int log_per_phy, int apic_id) { - unsigned current_bit; - int bits_seen; - - if (log_per_phy <= 1) - return (0); - - bits_seen = 0; - - for (current_bit = 1; log_per_phy != 0; current_bit <<= 1) { - if (log_per_phy & current_bit) { - log_per_phy &= ~current_bit; - bits_seen++; - } - } - - /* If exactly 1 bit was set in log_per_phy, choose next lower power of 2 */ - if (bits_seen == 1) { - current_bit >>= 1; - } - - return ((int)((current_bit - 1) & apic_id)); -} - static kmp_uint64 __kmp_parse_frequency( // R: Frequency in Hz. char const *frequency // I: Float number and unit: MHz, GHz, or TGz. ) { @@ -122,7 +60,6 @@ static kmp_uint64 __kmp_parse_frequency( // R: Frequency in Hz. void __kmp_query_cpuid(kmp_cpuinfo_t *p) { struct kmp_cpuid buf; int max_arg; - int log_per_phy; #ifdef KMP_DEBUG int cflush_size; #endif @@ -227,11 +164,8 @@ void __kmp_query_cpuid(kmp_cpuinfo_t *p) { if ((buf.edx >> 28) & 1) { /* Bits 23-16: Logical Processors per Physical Processor (1 for P4) */ - log_per_phy = data[2]; p->apic_id = data[3]; /* Bits 31-24: Processor Initial APIC ID (X) */ - KA_TRACE(trace_level, (" HT(%d TPUs)", log_per_phy)); - p->physical_id = __kmp_get_physical_id(log_per_phy, p->apic_id); - p->logical_id = __kmp_get_logical_id(log_per_phy, p->apic_id); + KA_TRACE(trace_level, (" HT(%d TPUs)", data[2])); } #ifdef KMP_DEBUG if ((buf.edx >> 29) & 1) { -- GitLab From 7c83d1bd612783634aae33baf8765ecfdcc5cd0d Mon Sep 17 00:00:00 2001 From: Han-Chung Wang Date: Tue, 12 Mar 2024 11:46:05 -0700 Subject: [PATCH 0038/1407] [mlir][vector] Use inferRankReducedResultType for subview type inference. (#84395) Fixes https://github.com/openxla/iree/issues/16475 --- .../Vector/Transforms/VectorTransforms.cpp | 48 ++++--------------- ...tor-transfer-collapse-inner-most-dims.mlir | 35 ++++++++------ 2 files changed, 28 insertions(+), 55 deletions(-) diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp index a2d4e2166331..6f6b6dcdad20 100644 --- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp @@ -1255,42 +1255,6 @@ getTransferFoldableInnerUnitDims(MemRefType srcType, VectorType vectorType) { return result; } -/// Returns a MemRef type that drops inner `dimsToDrop` dimensions from -/// `srcType`. E.g., if `srcType` is memref<512x16x1x1xf32> and `dimsToDrop` is -/// two, it returns memref<512x16x16> type. -static MemRefType getMemRefTypeWithDroppingInnerDims(OpBuilder &builder, - MemRefType srcType, - size_t dimsToDrop) { - MemRefLayoutAttrInterface layout = srcType.getLayout(); - if (isa(layout) && layout.isIdentity()) { - return MemRefType::get(srcType.getShape().drop_back(dimsToDrop), - srcType.getElementType(), nullptr, - srcType.getMemorySpace()); - } - MemRefLayoutAttrInterface updatedLayout; - if (auto strided = dyn_cast(layout)) { - auto strides = llvm::to_vector(strided.getStrides().drop_back(dimsToDrop)); - updatedLayout = StridedLayoutAttr::get(strided.getContext(), - strided.getOffset(), strides); - return MemRefType::get(srcType.getShape().drop_back(dimsToDrop), - srcType.getElementType(), updatedLayout, - srcType.getMemorySpace()); - } - - // Non-strided layout case. - AffineMap map = srcType.getLayout().getAffineMap(); - int numSymbols = map.getNumSymbols(); - for (size_t i = 0; i < dimsToDrop; ++i) { - int dim = srcType.getRank() - i - 1; - map = map.replace(builder.getAffineDimExpr(dim), - builder.getAffineConstantExpr(0), map.getNumDims() - 1, - numSymbols); - } - return MemRefType::get(srcType.getShape().drop_back(dimsToDrop), - srcType.getElementType(), updatedLayout, - srcType.getMemorySpace()); -} - /// Drop inner most contiguous unit dimensions from transfer_read operand. class DropInnerMostUnitDimsTransferRead : public OpRewritePattern { @@ -1337,8 +1301,10 @@ class DropInnerMostUnitDimsTransferRead rewriter.getIndexAttr(0)); SmallVector strides(srcType.getRank(), rewriter.getIndexAttr(1)); - MemRefType resultMemrefType = - getMemRefTypeWithDroppingInnerDims(rewriter, srcType, dimsToDrop); + auto resultMemrefType = + cast(memref::SubViewOp::inferRankReducedResultType( + srcType.getShape().drop_back(dimsToDrop), srcType, offsets, sizes, + strides)); ArrayAttr inBoundsAttr = readOp.getInBounds() ? rewriter.getArrayAttr( @@ -1421,8 +1387,10 @@ class DropInnerMostUnitDimsTransferWrite rewriter.getIndexAttr(0)); SmallVector strides(srcType.getRank(), rewriter.getIndexAttr(1)); - MemRefType resultMemrefType = - getMemRefTypeWithDroppingInnerDims(rewriter, srcType, dimsToDrop); + auto resultMemrefType = + cast(memref::SubViewOp::inferRankReducedResultType( + srcType.getShape().drop_back(dimsToDrop), srcType, offsets, sizes, + strides)); ArrayAttr inBoundsAttr = writeOp.getInBounds() ? rewriter.getArrayAttr( diff --git a/mlir/test/Dialect/Vector/vector-transfer-collapse-inner-most-dims.mlir b/mlir/test/Dialect/Vector/vector-transfer-collapse-inner-most-dims.mlir index 3984f17f9e8c..477755b66c02 100644 --- a/mlir/test/Dialect/Vector/vector-transfer-collapse-inner-most-dims.mlir +++ b/mlir/test/Dialect/Vector/vector-transfer-collapse-inner-most-dims.mlir @@ -16,22 +16,27 @@ func.func @contiguous_inner_most_view(%in: memref<1x1x8x1xf32, strided<[3072, 8, // ----- -func.func @contiguous_outer_dyn_inner_most_view(%in: memref>) -> vector<1x8x1xf32>{ +func.func @contiguous_outer_dyn_inner_most_view(%a: index, %b: index, %memref: memref) -> vector<8x1xf32> { %c0 = arith.constant 0 : index - %cst = arith.constant 0.0 : f32 - %0 = vector.transfer_read %in[%c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true]} : memref>, vector<1x8x1xf32> - return %0 : vector<1x8x1xf32> + %pad = arith.constant 0.0 : f32 + %v = vector.transfer_read %memref[%a, %b, %c0, %c0], %pad {in_bounds = [true, true]} : memref, vector<8x1xf32> + return %v : vector<8x1xf32> } -// CHECK: func @contiguous_outer_dyn_inner_most_view( +// CHECK: func.func @contiguous_outer_dyn_inner_most_view( +// CHECK-SAME: %[[IDX0:[a-zA-Z0-9]+]] +// CHECK-SAME: %[[IDX1:[a-zA-Z0-9]+]] // CHECK-SAME: %[[SRC:[a-zA-Z0-9]+]] -// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index -// CHECK-DAG: %[[D0:.+]] = memref.dim %[[SRC]], %[[C0]] -// CHECK: %[[SRC_0:.+]] = memref.subview %[[SRC]][0, 0, 0, 0] [%[[D0]], 1, 8, 1] [1, 1, 1, 1] -// CHECK-SAME: memref> to memref> -// CHECK: %[[VEC:.+]] = vector.transfer_read %[[SRC_0]] -// CHECK-SAME: memref>, vector<1x8xf32> -// CHECK: %[[RESULT:.+]] = vector.shape_cast %[[VEC]] -// CHECK: return %[[RESULT]] +// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index +// CHECK-DAG: %[[C1:.+]] = arith.constant 1 : index +// CHECK-DAG: %[[PAD:.+]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[D0:.+]] = memref.dim %[[SRC]], %[[C0]] +// CHECK: %[[D1:.+]] = memref.dim %[[SRC]], %[[C1]] +// CHECK: %[[VIEW:.+]] = memref.subview %[[SRC]][0, 0, 0, 0] [%[[D0]], %[[D1]], 8, 1] [1, 1, 1, 1] +// CHECK-SAME: memref to memref> +// CHECK: %[[VEC:.+]] = vector.transfer_read %[[VIEW]] +// CHECK-SAME: memref>, vector<8xf32> +// CHECK: %[[RESULT:.+]] = vector.shape_cast %[[VEC]] +// CHECK: return %[[RESULT]] // ----- @@ -43,7 +48,7 @@ func.func @contiguous_inner_most_dim(%A: memref<16x1xf32>, %i:index, %j:index) - } // CHECK: func @contiguous_inner_most_dim(%[[SRC:.+]]: memref<16x1xf32>, %[[I:.+]]: index, %[[J:.+]]: index) -> vector<8x1xf32> // CHECK: %[[SRC_0:.+]] = memref.subview %[[SRC]] -// CHECK-SAME: memref<16x1xf32> to memref<16xf32> +// CHECK-SAME: memref<16x1xf32> to memref<16xf32, strided<[1]>> // CHECK: %[[V:.+]] = vector.transfer_read %[[SRC_0]] // CHECK: %[[RESULT]] = vector.shape_cast %[[V]] : vector<8xf32> to vector<8x1xf32> // CHECK: return %[[RESULT]] @@ -111,7 +116,7 @@ func.func @drop_two_inner_most_dim_for_transfer_write(%arg0: memref<1x512x16x1x1 // CHECK-SAME: %[[IDX:[a-zA-Z0-9]+]] // CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index // CHECK: %[[SUBVIEW:.+]] = memref.subview %[[DEST]] -// CHECK-SAME: memref<1x512x16x1x1xf32> to memref<1x512x16xf32> +// CHECK-SAME: memref<1x512x16x1x1xf32> to memref<1x512x16xf32, strided<[8192, 16, 1]>> // CHECK: %[[CAST:.+]] = vector.shape_cast %[[VEC]] : vector<1x16x16x1x1xf32> to vector<1x16x16xf32> // CHECK: vector.transfer_write %[[CAST]], %[[SUBVIEW]] // CHECK-SAME: [%[[C0]], %[[IDX]], %[[C0]]] -- GitLab From 45219702e77f6f834ea2dfe2a68b140e59f7e0e2 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Tue, 12 Mar 2024 19:07:37 +0000 Subject: [PATCH 0039/1407] [test][X86] Precommit test for large data threshold and i1 global --- llvm/test/CodeGen/X86/code-model-elf.ll | 181 ++++++++++++++++-------- 1 file changed, 121 insertions(+), 60 deletions(-) diff --git a/llvm/test/CodeGen/X86/code-model-elf.ll b/llvm/test/CodeGen/X86/code-model-elf.ll index 71a6a310906e..4e96d39d153f 100644 --- a/llvm/test/CodeGen/X86/code-model-elf.ll +++ b/llvm/test/CodeGen/X86/code-model-elf.ll @@ -54,6 +54,7 @@ target triple = "x86_64--linux" @extern_data = external global [10 x i32], align 16 @thread_data = external thread_local global i32, align 4 @unknown_size_data = dso_local global [0 x i32] zeroinitializer, align 16 +@bool = dso_local global i1 false @opaque = external dso_local global %t @forced_small_data = dso_local global [10 x i32] zeroinitializer, code_model "small", align 16 @forced_large_data = dso_local global [10 x i32] zeroinitializer, code_model "large", align 16 @@ -746,6 +747,66 @@ define dso_local i32 @load_unknown_size_data() #0 { ret i32 %rv } +define dso_local i1 @load_bool() #0 { +; SMALL-STATIC-LABEL: load_bool: +; SMALL-STATIC: # %bb.0: +; SMALL-STATIC-NEXT: movzbl bool(%rip), %eax +; SMALL-STATIC-NEXT: retq +; +; MEDIUM-STATIC-LABEL: load_bool: +; MEDIUM-STATIC: # %bb.0: +; MEDIUM-STATIC-NEXT: movabsq $bool, %rax +; MEDIUM-STATIC-NEXT: movzbl (%rax), %eax +; MEDIUM-STATIC-NEXT: retq +; +; LARGE-STATIC-LABEL: load_bool: +; LARGE-STATIC: # %bb.0: +; LARGE-STATIC-NEXT: movabsq $bool, %rax +; LARGE-STATIC-NEXT: movzbl (%rax), %eax +; LARGE-STATIC-NEXT: retq +; +; SMALL-PIC-LABEL: load_bool: +; SMALL-PIC: # %bb.0: +; SMALL-PIC-NEXT: movzbl bool(%rip), %eax +; SMALL-PIC-NEXT: retq +; +; MEDIUM-SMALL-DATA-PIC-LABEL: load_bool: +; MEDIUM-SMALL-DATA-PIC: # %bb.0: +; MEDIUM-SMALL-DATA-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax +; MEDIUM-SMALL-DATA-PIC-NEXT: movabsq $bool@GOTOFF, %rcx +; MEDIUM-SMALL-DATA-PIC-NEXT: movzbl (%rax,%rcx), %eax +; MEDIUM-SMALL-DATA-PIC-NEXT: retq +; +; MEDIUM-PIC-LABEL: load_bool: +; MEDIUM-PIC: # %bb.0: +; MEDIUM-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax +; MEDIUM-PIC-NEXT: movabsq $bool@GOTOFF, %rcx +; MEDIUM-PIC-NEXT: movzbl (%rax,%rcx), %eax +; MEDIUM-PIC-NEXT: retq +; +; LARGE-PIC-LABEL: load_bool: +; LARGE-PIC: # %bb.0: +; LARGE-PIC-NEXT: .L12$pb: +; LARGE-PIC-NEXT: leaq .L12$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L12$pb, %rcx +; LARGE-PIC-NEXT: addq %rax, %rcx +; LARGE-PIC-NEXT: movabsq $bool@GOTOFF, %rax +; LARGE-PIC-NEXT: movzbl (%rcx,%rax), %eax +; LARGE-PIC-NEXT: retq +; +; LARGE-SMALL-DATA-PIC-LABEL: load_bool: +; LARGE-SMALL-DATA-PIC: # %bb.0: +; LARGE-SMALL-DATA-PIC-NEXT: .L12$pb: +; LARGE-SMALL-DATA-PIC-NEXT: leaq .L12$pb(%rip), %rax +; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L12$pb, %rcx +; LARGE-SMALL-DATA-PIC-NEXT: addq %rax, %rcx +; LARGE-SMALL-DATA-PIC-NEXT: movabsq $bool@GOTOFF, %rax +; LARGE-SMALL-DATA-PIC-NEXT: movzbl (%rcx,%rax), %eax +; LARGE-SMALL-DATA-PIC-NEXT: retq + %rv = load i1, ptr @bool + ret i1 %rv +} + define dso_local ptr @lea_opaque() #0 { ; SMALL-STATIC-LABEL: lea_opaque: ; SMALL-STATIC: # %bb.0: @@ -783,9 +844,9 @@ define dso_local ptr @lea_opaque() #0 { ; ; LARGE-PIC-LABEL: lea_opaque: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L12$pb: -; LARGE-PIC-NEXT: leaq .L12$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L12$pb, %rcx +; LARGE-PIC-NEXT: .L13$pb: +; LARGE-PIC-NEXT: leaq .L13$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L13$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $opaque@GOTOFF, %rax ; LARGE-PIC-NEXT: addq %rcx, %rax @@ -793,9 +854,9 @@ define dso_local ptr @lea_opaque() #0 { ; ; LARGE-SMALL-DATA-PIC-LABEL: lea_opaque: ; LARGE-SMALL-DATA-PIC: # %bb.0: -; LARGE-SMALL-DATA-PIC-NEXT: .L12$pb: -; LARGE-SMALL-DATA-PIC-NEXT: leaq .L12$pb(%rip), %rax -; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L12$pb, %rcx +; LARGE-SMALL-DATA-PIC-NEXT: .L13$pb: +; LARGE-SMALL-DATA-PIC-NEXT: leaq .L13$pb(%rip), %rax +; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L13$pb, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: addq %rax, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: movabsq $opaque@GOTOFF, %rax ; LARGE-SMALL-DATA-PIC-NEXT: addq %rcx, %rax @@ -840,9 +901,9 @@ define dso_local ptr @lea_ehdr_start() #0 { ; ; LARGE-PIC-LABEL: lea_ehdr_start: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L13$pb: -; LARGE-PIC-NEXT: leaq .L13$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L13$pb, %rcx +; LARGE-PIC-NEXT: .L14$pb: +; LARGE-PIC-NEXT: leaq .L14$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L14$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $__ehdr_start@GOTOFF, %rax ; LARGE-PIC-NEXT: addq %rcx, %rax @@ -850,9 +911,9 @@ define dso_local ptr @lea_ehdr_start() #0 { ; ; LARGE-SMALL-DATA-PIC-LABEL: lea_ehdr_start: ; LARGE-SMALL-DATA-PIC: # %bb.0: -; LARGE-SMALL-DATA-PIC-NEXT: .L13$pb: -; LARGE-SMALL-DATA-PIC-NEXT: leaq .L13$pb(%rip), %rax -; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L13$pb, %rcx +; LARGE-SMALL-DATA-PIC-NEXT: .L14$pb: +; LARGE-SMALL-DATA-PIC-NEXT: leaq .L14$pb(%rip), %rax +; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L14$pb, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: addq %rax, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: movabsq $__ehdr_start@GOTOFF, %rax ; LARGE-SMALL-DATA-PIC-NEXT: addq %rcx, %rax @@ -897,9 +958,9 @@ define dso_local ptr @lea_start_foo() #0 { ; ; LARGE-PIC-LABEL: lea_start_foo: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L14$pb: -; LARGE-PIC-NEXT: leaq .L14$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L14$pb, %rcx +; LARGE-PIC-NEXT: .L15$pb: +; LARGE-PIC-NEXT: leaq .L15$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L15$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $__start_foo@GOTOFF, %rax ; LARGE-PIC-NEXT: addq %rcx, %rax @@ -907,9 +968,9 @@ define dso_local ptr @lea_start_foo() #0 { ; ; LARGE-SMALL-DATA-PIC-LABEL: lea_start_foo: ; LARGE-SMALL-DATA-PIC: # %bb.0: -; LARGE-SMALL-DATA-PIC-NEXT: .L14$pb: -; LARGE-SMALL-DATA-PIC-NEXT: leaq .L14$pb(%rip), %rax -; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L14$pb, %rcx +; LARGE-SMALL-DATA-PIC-NEXT: .L15$pb: +; LARGE-SMALL-DATA-PIC-NEXT: leaq .L15$pb(%rip), %rax +; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L15$pb, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: addq %rax, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: movabsq $__start_foo@GOTOFF, %rax ; LARGE-SMALL-DATA-PIC-NEXT: addq %rcx, %rax @@ -954,9 +1015,9 @@ define dso_local ptr @lea_stop_foo() #0 { ; ; LARGE-PIC-LABEL: lea_stop_foo: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L15$pb: -; LARGE-PIC-NEXT: leaq .L15$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L15$pb, %rcx +; LARGE-PIC-NEXT: .L16$pb: +; LARGE-PIC-NEXT: leaq .L16$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L16$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $__stop_foo@GOTOFF, %rax ; LARGE-PIC-NEXT: addq %rcx, %rax @@ -964,9 +1025,9 @@ define dso_local ptr @lea_stop_foo() #0 { ; ; LARGE-SMALL-DATA-PIC-LABEL: lea_stop_foo: ; LARGE-SMALL-DATA-PIC: # %bb.0: -; LARGE-SMALL-DATA-PIC-NEXT: .L15$pb: -; LARGE-SMALL-DATA-PIC-NEXT: leaq .L15$pb(%rip), %rax -; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L15$pb, %rcx +; LARGE-SMALL-DATA-PIC-NEXT: .L16$pb: +; LARGE-SMALL-DATA-PIC-NEXT: leaq .L16$pb(%rip), %rax +; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L16$pb, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: addq %rax, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: movabsq $__stop_foo@GOTOFF, %rax ; LARGE-SMALL-DATA-PIC-NEXT: addq %rcx, %rax @@ -1035,9 +1096,9 @@ define dso_local ptr @lea_static_fn() #0 { ; ; LARGE-PIC-LABEL: lea_static_fn: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L19$pb: -; LARGE-PIC-NEXT: leaq .L19$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L19$pb, %rcx +; LARGE-PIC-NEXT: .L20$pb: +; LARGE-PIC-NEXT: leaq .L20$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L20$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $static_fn@GOTOFF, %rax ; LARGE-PIC-NEXT: addq %rcx, %rax @@ -1045,9 +1106,9 @@ define dso_local ptr @lea_static_fn() #0 { ; ; LARGE-SMALL-DATA-PIC-LABEL: lea_static_fn: ; LARGE-SMALL-DATA-PIC: # %bb.0: -; LARGE-SMALL-DATA-PIC-NEXT: .L19$pb: -; LARGE-SMALL-DATA-PIC-NEXT: leaq .L19$pb(%rip), %rax -; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L19$pb, %rcx +; LARGE-SMALL-DATA-PIC-NEXT: .L20$pb: +; LARGE-SMALL-DATA-PIC-NEXT: leaq .L20$pb(%rip), %rax +; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L20$pb, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: addq %rax, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: movabsq $static_fn@GOTOFF, %rax ; LARGE-SMALL-DATA-PIC-NEXT: addq %rcx, %rax @@ -1088,9 +1149,9 @@ define dso_local ptr @lea_global_fn() #0 { ; ; LARGE-PIC-LABEL: lea_global_fn: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L20$pb: -; LARGE-PIC-NEXT: leaq .L20$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L20$pb, %rcx +; LARGE-PIC-NEXT: .L21$pb: +; LARGE-PIC-NEXT: leaq .L21$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L21$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $global_fn@GOTOFF, %rax ; LARGE-PIC-NEXT: addq %rcx, %rax @@ -1098,9 +1159,9 @@ define dso_local ptr @lea_global_fn() #0 { ; ; LARGE-SMALL-DATA-PIC-LABEL: lea_global_fn: ; LARGE-SMALL-DATA-PIC: # %bb.0: -; LARGE-SMALL-DATA-PIC-NEXT: .L20$pb: -; LARGE-SMALL-DATA-PIC-NEXT: leaq .L20$pb(%rip), %rax -; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L20$pb, %rcx +; LARGE-SMALL-DATA-PIC-NEXT: .L21$pb: +; LARGE-SMALL-DATA-PIC-NEXT: leaq .L21$pb(%rip), %rax +; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L21$pb, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: addq %rax, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: movabsq $global_fn@GOTOFF, %rax ; LARGE-SMALL-DATA-PIC-NEXT: addq %rcx, %rax @@ -1141,9 +1202,9 @@ define dso_local ptr @lea_extern_fn() #0 { ; ; LARGE-PIC-LABEL: lea_extern_fn: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L21$pb: -; LARGE-PIC-NEXT: leaq .L21$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L21$pb, %rcx +; LARGE-PIC-NEXT: .L22$pb: +; LARGE-PIC-NEXT: leaq .L22$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L22$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $extern_fn@GOT, %rax ; LARGE-PIC-NEXT: movq (%rcx,%rax), %rax @@ -1151,9 +1212,9 @@ define dso_local ptr @lea_extern_fn() #0 { ; ; LARGE-SMALL-DATA-PIC-LABEL: lea_extern_fn: ; LARGE-SMALL-DATA-PIC: # %bb.0: -; LARGE-SMALL-DATA-PIC-NEXT: .L21$pb: -; LARGE-SMALL-DATA-PIC-NEXT: leaq .L21$pb(%rip), %rax -; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L21$pb, %rcx +; LARGE-SMALL-DATA-PIC-NEXT: .L22$pb: +; LARGE-SMALL-DATA-PIC-NEXT: leaq .L22$pb(%rip), %rax +; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L22$pb, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: addq %rax, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: movabsq $extern_fn@GOT, %rax ; LARGE-SMALL-DATA-PIC-NEXT: movq (%rcx,%rax), %rax @@ -1194,9 +1255,9 @@ define dso_local ptr @lea_ifunc() #0 { ; ; LARGE-PIC-LABEL: lea_ifunc: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L22$pb: -; LARGE-PIC-NEXT: leaq .L22$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L22$pb, %rcx +; LARGE-PIC-NEXT: .L23$pb: +; LARGE-PIC-NEXT: leaq .L23$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L23$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $ifunc_func@GOT, %rax ; LARGE-PIC-NEXT: movq (%rcx,%rax), %rax @@ -1204,9 +1265,9 @@ define dso_local ptr @lea_ifunc() #0 { ; ; LARGE-SMALL-DATA-PIC-LABEL: lea_ifunc: ; LARGE-SMALL-DATA-PIC: # %bb.0: -; LARGE-SMALL-DATA-PIC-NEXT: .L22$pb: -; LARGE-SMALL-DATA-PIC-NEXT: leaq .L22$pb(%rip), %rax -; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L22$pb, %rcx +; LARGE-SMALL-DATA-PIC-NEXT: .L23$pb: +; LARGE-SMALL-DATA-PIC-NEXT: leaq .L23$pb(%rip), %rax +; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L23$pb, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: addq %rax, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: movabsq $ifunc_func@GOT, %rax ; LARGE-SMALL-DATA-PIC-NEXT: movq (%rcx,%rax), %rax @@ -1247,9 +1308,9 @@ define dso_local ptr @lea_dso_local_ifunc() #0 { ; ; LARGE-PIC-LABEL: lea_dso_local_ifunc: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L23$pb: -; LARGE-PIC-NEXT: leaq .L23$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L23$pb, %rcx +; LARGE-PIC-NEXT: .L24$pb: +; LARGE-PIC-NEXT: leaq .L24$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L24$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $dso_local_ifunc_func@GOTOFF, %rax ; LARGE-PIC-NEXT: addq %rcx, %rax @@ -1257,9 +1318,9 @@ define dso_local ptr @lea_dso_local_ifunc() #0 { ; ; LARGE-SMALL-DATA-PIC-LABEL: lea_dso_local_ifunc: ; LARGE-SMALL-DATA-PIC: # %bb.0: -; LARGE-SMALL-DATA-PIC-NEXT: .L23$pb: -; LARGE-SMALL-DATA-PIC-NEXT: leaq .L23$pb(%rip), %rax -; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L23$pb, %rcx +; LARGE-SMALL-DATA-PIC-NEXT: .L24$pb: +; LARGE-SMALL-DATA-PIC-NEXT: leaq .L24$pb(%rip), %rax +; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L24$pb, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: addq %rax, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: movabsq $dso_local_ifunc_func@GOTOFF, %rax ; LARGE-SMALL-DATA-PIC-NEXT: addq %rcx, %rax @@ -1334,9 +1395,9 @@ define dso_local float @load_constant_pool(float %x) #0 { ; ; LARGE-PIC-LABEL: load_constant_pool: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L25$pb: -; LARGE-PIC-NEXT: leaq .L25$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L25$pb, %rcx +; LARGE-PIC-NEXT: .L26$pb: +; LARGE-PIC-NEXT: leaq .L26$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L26$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq ${{\.?LCPI[0-9]+_[0-9]+}}@GOTOFF, %rax ; LARGE-PIC-NEXT: addss (%rcx,%rax), %xmm0 @@ -1344,9 +1405,9 @@ define dso_local float @load_constant_pool(float %x) #0 { ; ; LARGE-SMALL-DATA-PIC-LABEL: load_constant_pool: ; LARGE-SMALL-DATA-PIC: # %bb.0: -; LARGE-SMALL-DATA-PIC-NEXT: .L25$pb: -; LARGE-SMALL-DATA-PIC-NEXT: leaq .L25$pb(%rip), %rax -; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L25$pb, %rcx +; LARGE-SMALL-DATA-PIC-NEXT: .L26$pb: +; LARGE-SMALL-DATA-PIC-NEXT: leaq .L26$pb(%rip), %rax +; LARGE-SMALL-DATA-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L26$pb, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: addq %rax, %rcx ; LARGE-SMALL-DATA-PIC-NEXT: movabsq ${{\.?LCPI[0-9]+_[0-9]+}}@GOTOFF, %rax ; LARGE-SMALL-DATA-PIC-NEXT: addss (%rcx,%rax), %xmm0 -- GitLab From a38b7a432d3cbb093af9310eba5b4982dc0a0243 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Tue, 12 Mar 2024 12:37:17 -0700 Subject: [PATCH 0040/1407] [InstallAPI] Break up headers and add common header for TextAPI types (#84960) Before it gets too unwieldy, add a common header for all MachO types that are used across InstallAPI. Also, break up the types in `InstallAPI/Frontend`. This both avoids circular dependencies and is logically easier to maintain as more functionality gets added. --- clang/include/clang/InstallAPI/Context.h | 6 +- clang/include/clang/InstallAPI/Frontend.h | 94 --------------- .../clang/InstallAPI/FrontendRecords.h | 108 ++++++++++++++++++ clang/include/clang/InstallAPI/MachO.h | 40 +++++++ clang/lib/InstallAPI/Frontend.cpp | 1 + clang/lib/InstallAPI/Visitor.cpp | 2 +- .../clang-installapi/ClangInstallAPI.cpp | 6 +- clang/tools/clang-installapi/Options.h | 12 +- 8 files changed, 159 insertions(+), 110 deletions(-) create mode 100644 clang/include/clang/InstallAPI/FrontendRecords.h create mode 100644 clang/include/clang/InstallAPI/MachO.h diff --git a/clang/include/clang/InstallAPI/Context.h b/clang/include/clang/InstallAPI/Context.h index 4e9e90e5d2db..bdb576d7d85f 100644 --- a/clang/include/clang/InstallAPI/Context.h +++ b/clang/include/clang/InstallAPI/Context.h @@ -12,8 +12,8 @@ #include "clang/Basic/Diagnostic.h" #include "clang/Basic/FileManager.h" #include "clang/InstallAPI/HeaderFile.h" +#include "clang/InstallAPI/MachO.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/TextAPI/InterfaceFile.h" namespace clang { namespace installapi { @@ -25,7 +25,7 @@ class FrontendRecordsSlice; struct InstallAPIContext { /// Library attributes that are typically passed as linker inputs. - llvm::MachO::RecordsSlice::BinaryAttrs BA; + BinaryAttrs BA; /// All headers that represent a library. HeaderSeq InputHeaders; @@ -49,7 +49,7 @@ struct InstallAPIContext { llvm::StringRef OutputLoc{}; /// What encoding to write output as. - llvm::MachO::FileType FT = llvm::MachO::FileType::TBD_V5; + FileType FT = FileType::TBD_V5; /// Populate entries of headers that should be included for TextAPI /// generation. diff --git a/clang/include/clang/InstallAPI/Frontend.h b/clang/include/clang/InstallAPI/Frontend.h index cbc2b159ebd1..873cb50d60a5 100644 --- a/clang/include/clang/InstallAPI/Frontend.h +++ b/clang/include/clang/InstallAPI/Frontend.h @@ -25,100 +25,6 @@ namespace clang { namespace installapi { -using SymbolFlags = llvm::MachO::SymbolFlags; -using RecordLinkage = llvm::MachO::RecordLinkage; -using GlobalRecord = llvm::MachO::GlobalRecord; -using ObjCContainerRecord = llvm::MachO::ObjCContainerRecord; -using ObjCInterfaceRecord = llvm::MachO::ObjCInterfaceRecord; -using ObjCCategoryRecord = llvm::MachO::ObjCCategoryRecord; -using ObjCIVarRecord = llvm::MachO::ObjCIVarRecord; - -// Represents a collection of frontend records for a library that are tied to a -// darwin target triple. -class FrontendRecordsSlice : public llvm::MachO::RecordsSlice { -public: - FrontendRecordsSlice(const llvm::Triple &T) - : llvm::MachO::RecordsSlice({T}) {} - - /// Add non-ObjC global record with attributes from AST. - /// - /// \param Name The name of symbol. - /// \param Linkage The linkage of symbol. - /// \param GV The kind of global. - /// \param Avail The availability information tied to the active target - /// triple. - /// \param D The pointer to the declaration from traversing AST. - /// \param Access The intended access level of symbol. - /// \param Flags The flags that describe attributes of the symbol. - /// \param Inlined Whether declaration is inlined, only applicable to - /// functions. - /// \return The non-owning pointer to added record in slice. - GlobalRecord *addGlobal(StringRef Name, RecordLinkage Linkage, - GlobalRecord::Kind GV, - const clang::AvailabilityInfo Avail, const Decl *D, - const HeaderType Access, - SymbolFlags Flags = SymbolFlags::None, - bool Inlined = false); - - /// Add ObjC Class record with attributes from AST. - /// - /// \param Name The name of class, not symbol. - /// \param Linkage The linkage of symbol. - /// \param Avail The availability information tied to the active target - /// triple. - /// \param D The pointer to the declaration from traversing AST. - /// \param Access The intended access level of symbol. - /// \param IsEHType Whether declaration has an exception attribute. - /// \return The non-owning pointer to added record in slice. - ObjCInterfaceRecord *addObjCInterface(StringRef Name, RecordLinkage Linkage, - const clang::AvailabilityInfo Avail, - const Decl *D, HeaderType Access, - bool IsEHType); - - /// Add ObjC Category record with attributes from AST. - /// - /// \param ClassToExtend The name of class that is extended by category, not - /// symbol. - /// \param CategoryName The name of category, not symbol. - /// \param Avail The availability information tied - /// to the active target triple. - /// \param D The pointer to the declaration from traversing AST. - /// \param Access The intended access level of symbol. - /// \return The non-owning pointer to added record in slice. - ObjCCategoryRecord *addObjCCategory(StringRef ClassToExtend, - StringRef CategoryName, - const clang::AvailabilityInfo Avail, - const Decl *D, HeaderType Access); - - /// Add ObjC IVar record with attributes from AST. - /// - /// \param Container The owning pointer for instance variable. - /// \param Name The name of ivar, not symbol. - /// \param Linkage The linkage of symbol. - /// \param Avail The availability information tied to the active target - /// triple. - /// \param D The pointer to the declaration from traversing AST. - /// \param Access The intended access level of symbol. - /// \param AC The access control tied to the ivar declaration. - /// \return The non-owning pointer to added record in slice. - ObjCIVarRecord *addObjCIVar(ObjCContainerRecord *Container, - StringRef IvarName, RecordLinkage Linkage, - const clang::AvailabilityInfo Avail, - const Decl *D, HeaderType Access, - const clang::ObjCIvarDecl::AccessControl AC); - -private: - /// Frontend information captured about records. - struct FrontendAttrs { - const AvailabilityInfo Avail; - const Decl *D; - const HeaderType Access; - }; - - /// Mapping of records stored in slice to their frontend attributes. - llvm::DenseMap FrontendRecords; -}; - /// Create a buffer that contains all headers to scan /// for global symbols with. std::unique_ptr createInputBuffer(InstallAPIContext &Ctx); diff --git a/clang/include/clang/InstallAPI/FrontendRecords.h b/clang/include/clang/InstallAPI/FrontendRecords.h new file mode 100644 index 000000000000..333015b6a113 --- /dev/null +++ b/clang/include/clang/InstallAPI/FrontendRecords.h @@ -0,0 +1,108 @@ +//===- InstallAPI/FrontendRecords.h ------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INSTALLAPI_FRONTENDRECORDS_H +#define LLVM_CLANG_INSTALLAPI_FRONTENDRECORDS_H + +#include "clang/AST/Availability.h" +#include "clang/AST/DeclObjC.h" +#include "clang/InstallAPI/MachO.h" + +namespace clang { +namespace installapi { + +/// Frontend information captured about records. +struct FrontendAttrs { + const AvailabilityInfo Avail; + const Decl *D; + const HeaderType Access; +}; + +// Represents a collection of frontend records for a library that are tied to a +// darwin target triple. +class FrontendRecordsSlice : public llvm::MachO::RecordsSlice { +public: + FrontendRecordsSlice(const llvm::Triple &T) + : llvm::MachO::RecordsSlice({T}) {} + + /// Add non-ObjC global record with attributes from AST. + /// + /// \param Name The name of symbol. + /// \param Linkage The linkage of symbol. + /// \param GV The kind of global. + /// \param Avail The availability information tied to the active target + /// triple. + /// \param D The pointer to the declaration from traversing AST. + /// \param Access The intended access level of symbol. + /// \param Flags The flags that describe attributes of the symbol. + /// \param Inlined Whether declaration is inlined, only applicable to + /// functions. + /// \return The non-owning pointer to added record in slice. + GlobalRecord *addGlobal(StringRef Name, RecordLinkage Linkage, + GlobalRecord::Kind GV, + const clang::AvailabilityInfo Avail, const Decl *D, + const HeaderType Access, + SymbolFlags Flags = SymbolFlags::None, + bool Inlined = false); + + /// Add ObjC Class record with attributes from AST. + /// + /// \param Name The name of class, not symbol. + /// \param Linkage The linkage of symbol. + /// \param Avail The availability information tied to the active target + /// triple. + /// \param D The pointer to the declaration from traversing AST. + /// \param Access The intended access level of symbol. + /// \param IsEHType Whether declaration has an exception attribute. + /// \return The non-owning pointer to added record in slice. + ObjCInterfaceRecord *addObjCInterface(StringRef Name, RecordLinkage Linkage, + const clang::AvailabilityInfo Avail, + const Decl *D, HeaderType Access, + bool IsEHType); + + /// Add ObjC Category record with attributes from AST. + /// + /// \param ClassToExtend The name of class that is extended by category, not + /// symbol. + /// \param CategoryName The name of category, not symbol. + /// \param Avail The availability information tied + /// to the active target triple. + /// \param D The pointer to the declaration from traversing AST. + /// \param Access The intended access level of symbol. + /// \return The non-owning pointer to added record in slice. + ObjCCategoryRecord *addObjCCategory(StringRef ClassToExtend, + StringRef CategoryName, + const clang::AvailabilityInfo Avail, + const Decl *D, HeaderType Access); + + /// Add ObjC IVar record with attributes from AST. + /// + /// \param Container The owning pointer for instance variable. + /// \param Name The name of ivar, not symbol. + /// \param Linkage The linkage of symbol. + /// \param Avail The availability information tied to the active target + /// triple. + /// \param D The pointer to the declaration from traversing AST. + /// \param Access The intended access level of symbol. + /// \param AC The access control tied to the ivar declaration. + /// \return The non-owning pointer to added record in slice. + ObjCIVarRecord *addObjCIVar(ObjCContainerRecord *Container, + StringRef IvarName, RecordLinkage Linkage, + const clang::AvailabilityInfo Avail, + const Decl *D, HeaderType Access, + const clang::ObjCIvarDecl::AccessControl AC); + +private: + /// Mapping of records stored in slice to their frontend attributes. + llvm::DenseMap FrontendRecords; +}; + +} // namespace installapi +} // namespace clang + +#endif // LLVM_CLANG_INSTALLAPI_FRONTENDRECORDS_H diff --git a/clang/include/clang/InstallAPI/MachO.h b/clang/include/clang/InstallAPI/MachO.h new file mode 100644 index 000000000000..55e5591389ce --- /dev/null +++ b/clang/include/clang/InstallAPI/MachO.h @@ -0,0 +1,40 @@ +//===- InstallAPI/MachO.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Imports and forward declarations for llvm::MachO types. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INSTALLAPI_MACHO_H +#define LLVM_CLANG_INSTALLAPI_MACHO_H + +#include "llvm/TextAPI/Architecture.h" +#include "llvm/TextAPI/InterfaceFile.h" +#include "llvm/TextAPI/PackedVersion.h" +#include "llvm/TextAPI/Platform.h" +#include "llvm/TextAPI/RecordVisitor.h" +#include "llvm/TextAPI/Target.h" +#include "llvm/TextAPI/TextAPIWriter.h" +#include "llvm/TextAPI/Utils.h" + +using SymbolFlags = llvm::MachO::SymbolFlags; +using RecordLinkage = llvm::MachO::RecordLinkage; +using Record = llvm::MachO::Record; +using GlobalRecord = llvm::MachO::GlobalRecord; +using ObjCContainerRecord = llvm::MachO::ObjCContainerRecord; +using ObjCInterfaceRecord = llvm::MachO::ObjCInterfaceRecord; +using ObjCCategoryRecord = llvm::MachO::ObjCCategoryRecord; +using ObjCIVarRecord = llvm::MachO::ObjCIVarRecord; +using Records = llvm::MachO::Records; +using BinaryAttrs = llvm::MachO::RecordsSlice::BinaryAttrs; +using SymbolSet = llvm::MachO::SymbolSet; +using FileType = llvm::MachO::FileType; +using PackedVersion = llvm::MachO::PackedVersion; +using Target = llvm::MachO::Target; + +#endif // LLVM_CLANG_INSTALLAPI_MACHO_H diff --git a/clang/lib/InstallAPI/Frontend.cpp b/clang/lib/InstallAPI/Frontend.cpp index 0d526fe1da66..707aeb17dc89 100644 --- a/clang/lib/InstallAPI/Frontend.cpp +++ b/clang/lib/InstallAPI/Frontend.cpp @@ -8,6 +8,7 @@ #include "clang/InstallAPI/Frontend.h" #include "clang/AST/Availability.h" +#include "clang/InstallAPI/FrontendRecords.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" diff --git a/clang/lib/InstallAPI/Visitor.cpp b/clang/lib/InstallAPI/Visitor.cpp index aded94f7a94a..b4ed5974a057 100644 --- a/clang/lib/InstallAPI/Visitor.cpp +++ b/clang/lib/InstallAPI/Visitor.cpp @@ -11,7 +11,7 @@ #include "clang/AST/ParentMapContext.h" #include "clang/AST/VTableBuilder.h" #include "clang/Basic/Linkage.h" -#include "clang/InstallAPI/Frontend.h" +#include "clang/InstallAPI/FrontendRecords.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/IR/DataLayout.h" diff --git a/clang/tools/clang-installapi/ClangInstallAPI.cpp b/clang/tools/clang-installapi/ClangInstallAPI.cpp index c6da1c80a673..15b0baee88bc 100644 --- a/clang/tools/clang-installapi/ClangInstallAPI.cpp +++ b/clang/tools/clang-installapi/ClangInstallAPI.cpp @@ -19,6 +19,8 @@ #include "clang/Driver/Tool.h" #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/InstallAPI/Frontend.h" +#include "clang/InstallAPI/FrontendRecords.h" +#include "clang/InstallAPI/MachO.h" #include "clang/Tooling/Tooling.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/Option/Option.h" @@ -29,8 +31,6 @@ #include "llvm/Support/Process.h" #include "llvm/Support/Signals.h" #include "llvm/TargetParser/Host.h" -#include "llvm/TextAPI/RecordVisitor.h" -#include "llvm/TextAPI/TextAPIWriter.h" #include using namespace clang; @@ -125,7 +125,7 @@ static bool run(ArrayRef Args, const char *ProgName) { // Execute and gather AST results. // An invocation is ran for each unique target triple and for each header // access level. - llvm::MachO::Records FrontendResults; + Records FrontendResults; for (const auto &[Targ, Trip] : Opts.DriverOpts.Targets) { for (const HeaderType Type : {HeaderType::Public, HeaderType::Private, HeaderType::Project}) { diff --git a/clang/tools/clang-installapi/Options.h b/clang/tools/clang-installapi/Options.h index 9d4d841284fd..06f79b62c531 100644 --- a/clang/tools/clang-installapi/Options.h +++ b/clang/tools/clang-installapi/Options.h @@ -13,23 +13,17 @@ #include "clang/Basic/FileManager.h" #include "clang/Frontend/FrontendOptions.h" #include "clang/InstallAPI/Context.h" +#include "clang/InstallAPI/MachO.h" #include "llvm/Option/ArgList.h" #include "llvm/Option/Option.h" #include "llvm/Support/Program.h" #include "llvm/TargetParser/Triple.h" -#include "llvm/TextAPI/Architecture.h" -#include "llvm/TextAPI/InterfaceFile.h" -#include "llvm/TextAPI/PackedVersion.h" -#include "llvm/TextAPI/Platform.h" -#include "llvm/TextAPI/Target.h" -#include "llvm/TextAPI/Utils.h" #include #include #include namespace clang { namespace installapi { -using Macro = std::pair; struct DriverOptions { /// \brief Path to input file lists (JSON). @@ -42,7 +36,7 @@ struct DriverOptions { std::string OutputPath; /// \brief File encoding to print. - llvm::MachO::FileType OutFT = llvm::MachO::FileType::TBD_V5; + FileType OutFT = FileType::TBD_V5; /// \brief Print verbose output. bool Verbose = false; @@ -53,7 +47,7 @@ struct LinkerOptions { std::string InstallName; /// \brief The current version to use for the dynamic library. - llvm::MachO::PackedVersion CurrentVersion; + PackedVersion CurrentVersion; /// \brief Is application extension safe. bool AppExtensionSafe = false; -- GitLab From 6bbb73b4cbc89b7291a8088aaa635814a216fbf6 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Tue, 12 Mar 2024 13:43:29 -0600 Subject: [PATCH 0041/1407] [X86] Fix determining if globals with size <8 bits are large (#84975) Previously any global under 8 bits would accidentally be considered 0 sized, which is considered a large global. --- llvm/lib/Target/TargetMachine.cpp | 2 +- llvm/test/CodeGen/X86/code-model-elf.ll | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/TargetMachine.cpp b/llvm/lib/Target/TargetMachine.cpp index 8b177a89c919..a7fe329b064e 100644 --- a/llvm/lib/Target/TargetMachine.cpp +++ b/llvm/lib/Target/TargetMachine.cpp @@ -92,7 +92,7 @@ bool TargetMachine::isLargeGlobalValue(const GlobalValue *GVal) const { GV->getName().starts_with("__stop_"))) return true; const DataLayout &DL = GV->getParent()->getDataLayout(); - uint64_t Size = DL.getTypeSizeInBits(GV->getValueType()) / 8; + uint64_t Size = DL.getTypeAllocSize(GV->getValueType()); return Size == 0 || Size > LargeDataThreshold; } diff --git a/llvm/test/CodeGen/X86/code-model-elf.ll b/llvm/test/CodeGen/X86/code-model-elf.ll index 4e96d39d153f..0da62e3e7a65 100644 --- a/llvm/test/CodeGen/X86/code-model-elf.ll +++ b/llvm/test/CodeGen/X86/code-model-elf.ll @@ -772,9 +772,7 @@ define dso_local i1 @load_bool() #0 { ; ; MEDIUM-SMALL-DATA-PIC-LABEL: load_bool: ; MEDIUM-SMALL-DATA-PIC: # %bb.0: -; MEDIUM-SMALL-DATA-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax -; MEDIUM-SMALL-DATA-PIC-NEXT: movabsq $bool@GOTOFF, %rcx -; MEDIUM-SMALL-DATA-PIC-NEXT: movzbl (%rax,%rcx), %eax +; MEDIUM-SMALL-DATA-PIC-NEXT: movzbl bool(%rip), %eax ; MEDIUM-SMALL-DATA-PIC-NEXT: retq ; ; MEDIUM-PIC-LABEL: load_bool: -- GitLab From 6095f8bbc410d2f8b926a32ba969d507f7343949 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Tue, 12 Mar 2024 12:52:31 -0700 Subject: [PATCH 0042/1407] Get rid of noisy debug log in verifyOpAndAdjustFlags. (#84677) This debug log adds noise to a large fraction of *other* debug logs when you run with -debug, because it prints "Verifying operation: blah blah\n" whenever those other debug logs dump an op. You can use -debug-only to get around this, but sometimes -debug really is what's called for! --- mlir/lib/IR/AsmPrinter.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp index 8d75349f8eed..456cf6a2c277 100644 --- a/mlir/lib/IR/AsmPrinter.cpp +++ b/mlir/lib/IR/AsmPrinter.cpp @@ -1895,9 +1895,6 @@ static OpPrintingFlags verifyOpAndAdjustFlags(Operation *op, printerFlags.shouldAssumeVerified()) return printerFlags; - LLVM_DEBUG(llvm::dbgs() << DEBUG_TYPE << ": Verifying operation: " - << op->getName() << "\n"); - // Ignore errors emitted by the verifier. We check the thread id to avoid // consuming other threads' errors. auto parentThreadId = llvm::get_threadid(); -- GitLab From 97fb91ee665660036f8beffd064b44c6fbbf1b73 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 12 Mar 2024 12:55:47 -0700 Subject: [PATCH 0043/1407] [SLP][NFC]Add a test with non-profitable alternate vectorized instructions. --- .../SLPVectorizer/alternate-non-profitable.ll | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 llvm/test/Transforms/SLPVectorizer/alternate-non-profitable.ll diff --git a/llvm/test/Transforms/SLPVectorizer/alternate-non-profitable.ll b/llvm/test/Transforms/SLPVectorizer/alternate-non-profitable.ll new file mode 100644 index 000000000000..c6e2cf5543e1 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/alternate-non-profitable.ll @@ -0,0 +1,190 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-10000 < %s | FileCheck %s + +define <2 x float> @test_fdiv(float %a, i1 %cmp) { +; CHECK-LABEL: define <2 x float> @test_fdiv( +; CHECK-SAME: float [[A:%.*]], i1 [[CMP:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = fdiv float [[A]], 3.000000e+00 +; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i64 1 +; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[CMP]], <2 x float> , <2 x float> [[TMP2]] +; CHECK-NEXT: ret <2 x float> [[TMP3]] +; + %1 = fdiv float %a, 3.000000e+00 + %2 = insertelement <2 x float> poison, float %1, i64 1 + %3 = select i1 %cmp, <2 x float> , <2 x float> %2 + ret <2 x float> %3 +} + +define <2 x float> @test_frem(float %a, i1 %cmp) { +; CHECK-LABEL: define <2 x float> @test_frem( +; CHECK-SAME: float [[A:%.*]], i1 [[CMP:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = frem float [[A]], 3.000000e+00 +; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i64 1 +; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[CMP]], <2 x float> , <2 x float> [[TMP2]] +; CHECK-NEXT: ret <2 x float> [[TMP3]] +; + %1 = frem float %a, 3.000000e+00 + %2 = insertelement <2 x float> poison, float %1, i64 1 + %3 = select i1 %cmp, <2 x float> , <2 x float> %2 + ret <2 x float> %3 +} + +define <2 x float> @replace_through_casts(i16 %inp) { +; CHECK-LABEL: define <2 x float> @replace_through_casts( +; CHECK-SAME: i16 [[INP:%.*]]) { +; CHECK-NEXT: [[ADD:%.*]] = add nsw i16 [[INP]], -10 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i16> poison, i16 [[INP]], i32 0 +; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i16> [[TMP1]], i16 [[ADD]], i32 1 +; CHECK-NEXT: [[TMP3:%.*]] = uitofp <2 x i16> [[TMP2]] to <2 x float> +; CHECK-NEXT: [[TMP4:%.*]] = sitofp <2 x i16> [[TMP2]] to <2 x float> +; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> [[TMP4]], <2 x i32> +; CHECK-NEXT: ret <2 x float> [[R]] +; + %add = add nsw i16 %inp, -10 + %1 = uitofp i16 %inp to float + %2 = sitofp i16 %add to float + %3 = insertelement <2 x float> poison, float %1, i64 0 + %r = insertelement <2 x float> %3, float %2, i64 1 + ret <2 x float> %r +} + +define <2 x float> @replace_through_casts_and_binop(i16 %inp) { +; CHECK-LABEL: define <2 x float> @replace_through_casts_and_binop( +; CHECK-SAME: i16 [[INP:%.*]]) { +; CHECK-NEXT: [[ADD:%.*]] = add nsw i16 [[INP]], -10 +; CHECK-NEXT: [[MUL:%.*]] = mul nsw i16 [[INP]], 5 +; CHECK-NEXT: [[TMP1:%.*]] = uitofp i16 [[MUL]] to float +; CHECK-NEXT: [[TMP2:%.*]] = fadd float [[TMP1]], 2.000000e+00 +; CHECK-NEXT: [[TMP3:%.*]] = sitofp i16 [[ADD]] to float +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i64 0 +; CHECK-NEXT: [[R:%.*]] = insertelement <2 x float> [[TMP4]], float [[TMP3]], i64 1 +; CHECK-NEXT: ret <2 x float> [[R]] +; + %add = add nsw i16 %inp, -10 + %mul = mul nsw i16 %inp, 5 + %1 = uitofp i16 %mul to float + %2 = fadd float %1, 2.000000e+00 + %3 = sitofp i16 %add to float + %4 = insertelement <2 x float> poison, float %2, i64 0 + %r = insertelement <2 x float> %4, float %3, i64 1 + ret <2 x float> %r +} + +define <2 x float> @replace_through_casts_and_binop_and_unop(i16 %inp) { +; CHECK-LABEL: define <2 x float> @replace_through_casts_and_binop_and_unop( +; CHECK-SAME: i16 [[INP:%.*]]) { +; CHECK-NEXT: [[ADD:%.*]] = add nsw i16 [[INP]], -10 +; CHECK-NEXT: [[TMP1:%.*]] = sitofp i16 [[ADD]] to float +; CHECK-NEXT: [[TMP2:%.*]] = fneg float [[TMP1]] +; CHECK-NEXT: [[TMP3:%.*]] = uitofp i16 [[ADD]] to float +; CHECK-NEXT: [[TMP4:%.*]] = fadd float [[TMP3]], 2.000000e+00 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[TMP4]], i64 0 +; CHECK-NEXT: [[R:%.*]] = insertelement <2 x float> [[TMP5]], float [[TMP2]], i64 1 +; CHECK-NEXT: ret <2 x float> [[R]] +; + %add = add nsw i16 %inp, -10 + %1 = sitofp i16 %add to float + %2 = fneg float %1 + %3 = uitofp i16 %add to float + %4 = fadd float %3, 2.000000e+00 + %5 = insertelement <2 x float> poison, float %4, i64 0 + %r = insertelement <2 x float> %5, float %2, i64 1 + ret <2 x float> %r +} + +define <2 x float> @replace_through_casts_through_splat(i16 %inp) { +; CHECK-LABEL: define <2 x float> @replace_through_casts_through_splat( +; CHECK-SAME: i16 [[INP:%.*]]) { +; CHECK-NEXT: [[ADD:%.*]] = add nsw i16 [[INP]], -10 +; CHECK-NEXT: [[TMP1:%.*]] = uitofp i16 [[ADD]] to float +; CHECK-NEXT: [[TMP2:%.*]] = fadd float [[TMP1]], 2.000000e+00 +; CHECK-NEXT: [[TMP3:%.*]] = sitofp i16 [[ADD]] to float +; CHECK-NEXT: [[TMP4:%.*]] = fneg float [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i64 0 +; CHECK-NEXT: [[R:%.*]] = insertelement <2 x float> [[TMP5]], float [[TMP4]], i64 1 +; CHECK-NEXT: ret <2 x float> [[R]] +; + %add = add nsw i16 %inp, -10 + %1 = uitofp i16 %add to float + %2 = fadd float %1, 2.000000e+00 + %3 = sitofp i16 %add to float + %4 = fneg float %3 + %5 = insertelement <2 x float> poison, float %2, i64 0 + %r = insertelement <2 x float> %5, float %4, i64 1 + ret <2 x float> %r +} + +define <2 x i32> @replace_through_int_casts(i16 %inp, <2 x i16> %dead) { +; CHECK-LABEL: define <2 x i32> @replace_through_int_casts( +; CHECK-SAME: i16 [[INP:%.*]], <2 x i16> [[DEAD:%.*]]) { +; CHECK-NEXT: [[ADD:%.*]] = add nsw i16 [[INP]], -10 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i16> poison, i16 [[INP]], i32 0 +; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i16> [[TMP1]], i16 [[ADD]], i32 1 +; CHECK-NEXT: [[TMP3:%.*]] = zext <2 x i16> [[TMP2]] to <2 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> +; CHECK-NEXT: ret <2 x i32> [[R]] +; + %add = add nsw i16 %inp, -10 + %1 = zext i16 %inp to i32 + %2 = sext i16 %add to i32 + %3 = insertelement <2 x i32> poison, i32 %1, i64 0 + %r = insertelement <2 x i32> %3, i32 %2, i64 1 + ret <2 x i32> %r +} + +define <2 x i32> @replace_through_int_casts_ele0_only(i16 %inp, <2 x i16> %dead) { +; CHECK-LABEL: define <2 x i32> @replace_through_int_casts_ele0_only( +; CHECK-SAME: i16 [[INP:%.*]], <2 x i16> [[DEAD:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i16> poison, i16 [[INP]], i32 0 +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP3:%.*]] = zext <2 x i16> [[TMP2]] to <2 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i32> +; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> +; CHECK-NEXT: ret <2 x i32> [[R]] +; + %2 = sext i16 %inp to i32 + %4 = zext i16 %inp to i32 + %5 = insertelement <2 x i32> poison, i32 %4, i64 0 + %r = insertelement <2 x i32> %5, i32 %2, i64 1 + ret <2 x i32> %r +} + +define <2 x i8> @replace_through_binop_fail_cant_speculate(i8 %inp, <2 x i8> %d, <2 x i8> %any) { +; CHECK-LABEL: define <2 x i8> @replace_through_binop_fail_cant_speculate( +; CHECK-SAME: i8 [[INP:%.*]], <2 x i8> [[D:%.*]], <2 x i8> [[ANY:%.*]]) { +; CHECK-NEXT: [[ADD:%.*]] = add i8 [[INP]], 5 +; CHECK-NEXT: [[V0:%.*]] = insertelement <2 x i8> poison, i8 [[INP]], i64 0 +; CHECK-NEXT: [[V:%.*]] = insertelement <2 x i8> [[V0]], i8 [[ADD]], i64 1 +; CHECK-NEXT: [[DIV0:%.*]] = sdiv <2 x i8> , [[V]] +; CHECK-NEXT: [[TMP1:%.*]] = xor i8 [[INP]], 123 +; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i8> [[DIV0]], i8 [[TMP1]], i64 0 +; CHECK-NEXT: ret <2 x i8> [[R]] +; + %add = add i8 %inp, 5 + %v0 = insertelement <2 x i8> poison, i8 %inp, i64 0 + %v = insertelement <2 x i8> %v0, i8 %add, i64 1 + %div0 = sdiv <2 x i8> , %v + %1 = xor i8 %inp, 123 + %r = insertelement <2 x i8> %div0, i8 %1, i64 0 + ret <2 x i8> %r +} + +define <2 x i8> @replace_through_binop_preserve_flags(i8 %inp, <2 x i8> %d, <2 x i8> %any) { +; CHECK-LABEL: define <2 x i8> @replace_through_binop_preserve_flags( +; CHECK-SAME: i8 [[INP:%.*]], <2 x i8> [[D:%.*]], <2 x i8> [[ANY:%.*]]) { +; CHECK-NEXT: [[ADD:%.*]] = xor i8 [[INP]], 5 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> poison, i8 [[INP]], i32 0 +; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i8> [[TMP1]], i8 [[ADD]], i32 1 +; CHECK-NEXT: [[TMP3:%.*]] = xor <2 x i8> [[TMP2]], +; CHECK-NEXT: [[TMP4:%.*]] = add nsw <2 x i8> [[TMP2]], +; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i8> [[TMP3]], <2 x i8> [[TMP4]], <2 x i32> +; CHECK-NEXT: ret <2 x i8> [[R]] +; + %add = xor i8 %inp, 5 + %1 = xor i8 %inp, 123 + %2 = add nsw i8 %add, 1 + %3 = insertelement <2 x i8> poison, i8 %1, i64 0 + %r = insertelement <2 x i8> %3, i8 %2, i64 1 + ret <2 x i8> %r +} -- GitLab From 2377beba8d10ce1092db7f8ddd5b10a2c0d3bfd1 Mon Sep 17 00:00:00 2001 From: Daniel Thornburgh Date: Tue, 12 Mar 2024 13:33:12 -0700 Subject: [PATCH 0044/1407] [Fuchsia] Add LLDB_TEST_USE_VENDOR_PACKAGES to boostrap passthrough --- clang/cmake/caches/Fuchsia.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/clang/cmake/caches/Fuchsia.cmake b/clang/cmake/caches/Fuchsia.cmake index fe925901eb3d..1209fd935986 100644 --- a/clang/cmake/caches/Fuchsia.cmake +++ b/clang/cmake/caches/Fuchsia.cmake @@ -65,6 +65,7 @@ set(_FUCHSIA_BOOTSTRAP_PASSTHROUGH LLDB_EMBED_PYTHON_HOME LLDB_PYTHON_HOME LLDB_PYTHON_RELATIVE_PATH + LLDB_TEST_USE_VENDOR_PACKAGES Python3_EXECUTABLE Python3_LIBRARIES Python3_INCLUDE_DIRS -- GitLab From 54f631d11640732af0dc9c2aa53af4e118d9fe36 Mon Sep 17 00:00:00 2001 From: "S. Bharadwaj Yadavalli" Date: Tue, 12 Mar 2024 16:51:18 -0400 Subject: [PATCH 0045/1407] [DirectX][NFC] Model precise overload type specification of DXIL Ops (#83917) Implement an abstraction to specify precise overload types supported by DXIL ops. These overload types are typically a subset of LLVM intrinsics. Implement the corresponding changes in DXILEmitter backend. Add tests to verify expected errors for unsupported overload types at code generation time. Add tests to check for correct overload error output. --- llvm/lib/Target/DirectX/DXIL.td | 55 ++++++++- llvm/lib/Target/DirectX/DXILOpBuilder.cpp | 4 +- llvm/test/CodeGen/DirectX/exp2_error.ll | 13 ++ llvm/test/CodeGen/DirectX/frac.ll | 3 - llvm/test/CodeGen/DirectX/frac_error.ll | 14 +++ llvm/test/CodeGen/DirectX/round_error.ll | 13 ++ llvm/test/CodeGen/DirectX/sin.ll | 22 +--- llvm/test/CodeGen/DirectX/sin_error.ll | 14 +++ llvm/utils/TableGen/DXILEmitter.cpp | 140 +++++++++++++++------- 9 files changed, 203 insertions(+), 75 deletions(-) create mode 100644 llvm/test/CodeGen/DirectX/exp2_error.ll create mode 100644 llvm/test/CodeGen/DirectX/frac_error.ll create mode 100644 llvm/test/CodeGen/DirectX/round_error.ll create mode 100644 llvm/test/CodeGen/DirectX/sin_error.ll diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index 9536a01e125b..66b0ef24332c 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -205,28 +205,71 @@ defset list OpClasses = { def writeSamplerFeedbackBias : DXILOpClass; def writeSamplerFeedbackGrad : DXILOpClass; def writeSamplerFeedbackLevel: DXILOpClass; + + // This is a sentinel definition. Hence placed at the end of the list + // and not as part of the above alphabetically sorted valid definitions. + // Additionally it is capitalized unlike all the others. + def UnknownOpClass: DXILOpClass; +} + +// Several of the overloaded DXIL Operations support for data types +// that are a subset of the overloaded LLVM intrinsics that they map to. +// For e.g., llvm.sin.* intrinsic operates on any floating-point type and +// maps for lowering to DXIL Op Sin. However, valid overloads of DXIL Sin +// operation overloads are half (f16) and float (f32) only. +// +// The following abstracts overload types specific to DXIL operations. + +class DXILType : LLVMType { + let isAny = 1; + int isI16OrI32 = 0; + int isHalfOrFloat = 0; } +// Concrete records for various overload types supported specifically by +// DXIL Operations. +let isI16OrI32 = 1 in + def llvm_i16ori32_ty : DXILType; + +let isHalfOrFloat = 1 in + def llvm_halforfloat_ty : DXILType; + // Abstraction DXIL Operation to LLVM intrinsic -class DXILOpMapping { +class DXILOpMappingBase { + int OpCode = 0; // Opcode of DXIL Operation + DXILOpClass OpClass = UnknownOpClass;// Class of DXIL Operation. + Intrinsic LLVMIntrinsic = ?; // LLVM Intrinsic DXIL Operation maps to + string Doc = ""; // A short description of the operation + list OpTypes = ?; // Valid types of DXIL Operation in the + // format [returnTy, param1ty, ...] +} + +class DXILOpMapping opTys = []> : DXILOpMappingBase { int OpCode = opCode; // Opcode corresponding to DXIL Operation - DXILOpClass OpClass = opClass; // Class of DXIL Operation. + DXILOpClass OpClass = opClass; // Class of DXIL Operation. Intrinsic LLVMIntrinsic = intrinsic; // LLVM Intrinsic the DXIL Operation maps string Doc = doc; // to a short description of the operation + list OpTypes = !if(!eq(!size(opTys), 0), LLVMIntrinsic.Types, opTys); } // Concrete definition of DXIL Operation mapping to corresponding LLVM intrinsic def Sin : DXILOpMapping<13, unary, int_sin, - "Returns sine(theta) for theta in radians.">; + "Returns sine(theta) for theta in radians.", + [llvm_halforfloat_ty, LLVMMatchType<0>]>; def Exp2 : DXILOpMapping<21, unary, int_exp2, "Returns the base 2 exponential, or 2**x, of the specified value." - "exp2(x) = 2**x.">; + "exp2(x) = 2**x.", + [llvm_halforfloat_ty, LLVMMatchType<0>]>; def Frac : DXILOpMapping<22, unary, int_dx_frac, "Returns a fraction from 0 to 1 that represents the " - "decimal part of the input.">; + "decimal part of the input.", + [llvm_halforfloat_ty, LLVMMatchType<0>]>; def Round : DXILOpMapping<26, unary, int_round, "Returns the input rounded to the nearest integer" - "within a floating-point type.">; + "within a floating-point type.", + [llvm_halforfloat_ty, LLVMMatchType<0>]>; def UMax : DXILOpMapping<39, binary, int_umax, "Unsigned integer maximum. UMax(a,b) = a > b ? a : b">; def FMad : DXILOpMapping<46, tertiary, int_fmuladd, diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp index 21a20d45b922..11b24d044923 100644 --- a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp +++ b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp @@ -254,10 +254,8 @@ static FunctionCallee getOrCreateDXILOpFunction(dxil::OpCode DXILOp, const OpCodeProperty *Prop = getOpCodeProperty(DXILOp); OverloadKind Kind = getOverloadKind(OverloadTy); - // FIXME: find the issue and report error in clang instead of check it in - // backend. if ((Prop->OverloadTys & (uint16_t)Kind) == 0) { - llvm_unreachable("invalid overload"); + report_fatal_error("Invalid Overload Type", /* gen_crash_diag=*/false); } std::string FnName = constructOverloadName(Kind, OverloadTy, *Prop); diff --git a/llvm/test/CodeGen/DirectX/exp2_error.ll b/llvm/test/CodeGen/DirectX/exp2_error.ll new file mode 100644 index 000000000000..6b9126785fd4 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/exp2_error.ll @@ -0,0 +1,13 @@ +; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s + +; DXIL operation exp2 does not support double overload type +; CHECK: LLVM ERROR: Invalid Overload + +define noundef double @exp2_double(double noundef %a) #0 { +entry: + %a.addr = alloca double, align 8 + store double %a, ptr %a.addr, align 8 + %0 = load double, ptr %a.addr, align 8 + %elt.exp2 = call double @llvm.exp2.f64(double %0) + ret double %elt.exp2 +} diff --git a/llvm/test/CodeGen/DirectX/frac.ll b/llvm/test/CodeGen/DirectX/frac.ll index ab605ed6084a..ae86fe06654d 100644 --- a/llvm/test/CodeGen/DirectX/frac.ll +++ b/llvm/test/CodeGen/DirectX/frac.ll @@ -29,6 +29,3 @@ entry: %dx.frac = call half @llvm.dx.frac.f16(half %0) ret half %dx.frac } - -; Function Attrs: nocallback nofree nosync nounwind readnone speculatable willreturn -declare half @llvm.dx.frac.f16(half) #1 diff --git a/llvm/test/CodeGen/DirectX/frac_error.ll b/llvm/test/CodeGen/DirectX/frac_error.ll new file mode 100644 index 000000000000..ebce76105ad4 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/frac_error.ll @@ -0,0 +1,14 @@ +; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s + +; DXIL operation frac does not support double overload type +; CHECK: LLVM ERROR: Invalid Overload Type + +; Function Attrs: noinline nounwind optnone +define noundef double @frac_double(double noundef %a) #0 { +entry: + %a.addr = alloca double, align 8 + store double %a, ptr %a.addr, align 8 + %0 = load double, ptr %a.addr, align 8 + %dx.frac = call double @llvm.dx.frac.f64(double %0) + ret double %dx.frac +} diff --git a/llvm/test/CodeGen/DirectX/round_error.ll b/llvm/test/CodeGen/DirectX/round_error.ll new file mode 100644 index 000000000000..3bd87b2bbf02 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/round_error.ll @@ -0,0 +1,13 @@ +; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s + +; This test is expected to fail with the following error +; CHECK: LLVM ERROR: Invalid Overload Type + +define noundef double @round_double(double noundef %a) #0 { +entry: + %a.addr = alloca double, align 8 + store double %a, ptr %a.addr, align 8 + %0 = load double, ptr %a.addr, align 8 + %elt.round = call double @llvm.round.f64(double %0) + ret double %elt.round +} diff --git a/llvm/test/CodeGen/DirectX/sin.ll b/llvm/test/CodeGen/DirectX/sin.ll index bb31d28bfcfe..1f285c433581 100644 --- a/llvm/test/CodeGen/DirectX/sin.ll +++ b/llvm/test/CodeGen/DirectX/sin.ll @@ -4,11 +4,8 @@ ; CHECK:call float @dx.op.unary.f32(i32 13, float %{{.*}}) ; CHECK:call half @dx.op.unary.f16(i32 13, half %{{.*}}) -target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64" -target triple = "dxil-pc-shadermodel6.7-library" - ; Function Attrs: noinline nounwind optnone -define noundef float @_Z3foof(float noundef %a) #0 { +define noundef float @sin_float(float noundef %a) #0 { entry: %a.addr = alloca float, align 4 store float %a, ptr %a.addr, align 4 @@ -17,11 +14,8 @@ entry: ret float %1 } -; Function Attrs: nocallback nofree nosync nounwind readnone speculatable willreturn -declare float @llvm.sin.f32(float) #1 - ; Function Attrs: noinline nounwind optnone -define noundef half @_Z3barDh(half noundef %a) #0 { +define noundef half @sin_half(half noundef %a) #0 { entry: %a.addr = alloca half, align 2 store half %a, ptr %a.addr, align 2 @@ -29,15 +23,3 @@ entry: %1 = call half @llvm.sin.f16(half %0) ret half %1 } - -; Function Attrs: nocallback nofree nosync nounwind readnone speculatable willreturn -declare half @llvm.sin.f16(half) #1 - -attributes #0 = { noinline nounwind optnone "frame-pointer"="none" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" } -attributes #1 = { nocallback nofree nosync nounwind readnone speculatable willreturn } - -!llvm.module.flags = !{!0} -!llvm.ident = !{!1} - -!0 = !{i32 1, !"wchar_size", i32 4} -!1 = !{!"clang version 15.0.0 (https://github.com/llvm/llvm-project.git 73417c517644db5c419c85c0b3cb6750172fcab5)"} diff --git a/llvm/test/CodeGen/DirectX/sin_error.ll b/llvm/test/CodeGen/DirectX/sin_error.ll new file mode 100644 index 000000000000..ece0e530315b --- /dev/null +++ b/llvm/test/CodeGen/DirectX/sin_error.ll @@ -0,0 +1,14 @@ +; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s + +; DXIL operation sin does not support double overload type +; CHECK: LLVM ERROR: Invalid Overload + +define noundef double @sin_double(double noundef %a) #0 { +entry: + %a.addr = alloca double, align 8 + store double %a, ptr %a.addr, align 8 + %0 = load double, ptr %a.addr, align 8 + %1 = call double @llvm.sin.f64(double %0) + ret double %1 +} + diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp index fc958f532873..59089929837e 100644 --- a/llvm/utils/TableGen/DXILEmitter.cpp +++ b/llvm/utils/TableGen/DXILEmitter.cpp @@ -22,6 +22,7 @@ #include "llvm/Support/DXILABI.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" +#include using namespace llvm; using namespace llvm::dxil; @@ -38,8 +39,8 @@ struct DXILOperationDesc { int OpCode; // ID of DXIL operation StringRef OpClass; // name of the opcode class StringRef Doc; // the documentation description of this instruction - SmallVector OpTypes; // Vector of operand types - - // return type is at index 0 + SmallVector OpTypes; // Vector of operand type records - + // return type is at index 0 SmallVector OpAttributes; // operation attribute represented as strings StringRef Intrinsic; // The llvm intrinsic map to OpName. Default is "" which @@ -57,20 +58,21 @@ struct DXILOperationDesc { DXILShaderModel ShaderModel; // minimum shader model required DXILShaderModel ShaderModelTranslated; // minimum shader model required with // translation by linker - int OverloadParamIndex; // parameter index which control the overload. - // When < 0, should be only 1 overload type. + int OverloadParamIndex; // Index of parameter with overload type. + // -1 : no overload types SmallVector counters; // counters for this inst. DXILOperationDesc(const Record *); }; } // end anonymous namespace -/// Convert DXIL type name string to dxil::ParameterKind +/// Return dxil::ParameterKind corresponding to input LLVMType record /// -/// \param VT Simple Value Type +/// \param R TableGen def record of class LLVMType /// \return ParameterKind As defined in llvm/Support/DXILABI.h -static ParameterKind getParameterKind(MVT::SimpleValueType VT) { - switch (VT) { +static ParameterKind getParameterKind(const Record *R) { + auto VTRec = R->getValueAsDef("VT"); + switch (getValueType(VTRec)) { case MVT::isVoid: return ParameterKind::VOID; case MVT::f16: @@ -90,6 +92,12 @@ static ParameterKind getParameterKind(MVT::SimpleValueType VT) { case MVT::fAny: case MVT::iAny: return ParameterKind::OVERLOAD; + case MVT::Other: + // Handle DXIL-specific overload types + if (R->getValueAsInt("isHalfOrFloat") || R->getValueAsInt("isI16OrI32")) { + return ParameterKind::OVERLOAD; + } + LLVM_FALLTHROUGH; default: llvm_unreachable("Support for specified DXIL Type not yet implemented"); } @@ -106,45 +114,80 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) { Doc = R->getValueAsString("Doc"); + auto TypeRecs = R->getValueAsListOfDefs("OpTypes"); + unsigned TypeRecsSize = TypeRecs.size(); + // Populate OpTypes with return type and parameter types + + // Parameter indices of overloaded parameters. + // This vector contains overload parameters in the order order used to + // resolve an LLVMMatchType in accordance with convention outlined in + // the comment before the definition of class LLVMMatchType in + // llvm/IR/Intrinsics.td + SmallVector OverloadParamIndices; + for (unsigned i = 0; i < TypeRecsSize; i++) { + auto TR = TypeRecs[i]; + // Track operation parameter indices of any overload types + auto isAny = TR->getValueAsInt("isAny"); + if (isAny == 1) { + // TODO: At present it is expected that all overload types in a DXIL Op + // are of the same type. Hence, OverloadParamIndices will have only one + // element. This implies we do not need a vector. However, until more + // (all?) DXIL Ops are added in DXIL.td, a vector is being used to flag + // cases this assumption would not hold. + if (!OverloadParamIndices.empty()) { + bool knownType = true; + // Ensure that the same overload type registered earlier is being used + for (auto Idx : OverloadParamIndices) { + if (TR != TypeRecs[Idx]) { + knownType = false; + break; + } + } + if (!knownType) { + report_fatal_error("Specification of multiple differing overload " + "parameter types not yet supported", + false); + } + } else { + OverloadParamIndices.push_back(i); + } + } + // Populate OpTypes array according to the type specification + if (TR->isAnonymous()) { + // Check prior overload types exist + assert(!OverloadParamIndices.empty() && + "No prior overloaded parameter found to match."); + // Get the parameter index of anonymous type, TR, references + auto OLParamIndex = TR->getValueAsInt("Number"); + // Resolve and insert the type to that at OLParamIndex + OpTypes.emplace_back(TypeRecs[OLParamIndex]); + } else { + // A non-anonymous type. Just record it in OpTypes + OpTypes.emplace_back(TR); + } + } + + // Set the index of the overload parameter, if any. + OverloadParamIndex = -1; // default; indicating none + if (!OverloadParamIndices.empty()) { + if (OverloadParamIndices.size() > 1) + report_fatal_error("Multiple overload type specification not supported", + false); + OverloadParamIndex = OverloadParamIndices[0]; + } + // Get the operation class + OpClass = R->getValueAsDef("OpClass")->getName(); + if (R->getValue("LLVMIntrinsic")) { auto *IntrinsicDef = R->getValueAsDef("LLVMIntrinsic"); auto DefName = IntrinsicDef->getName(); assert(DefName.starts_with("int_") && "invalid intrinsic name"); // Remove the int_ from intrinsic name. Intrinsic = DefName.substr(4); - // TODO: It is expected that return type and parameter types of - // DXIL Operation are the same as that of the intrinsic. Deviations - // are expected to be encoded in TableGen record specification and - // handled accordingly here. Support to be added later, as needed. - // Get parameter type list of the intrinsic. Types attribute contains - // the list of as [returnType, param1Type,, param2Type, ...] - - OverloadParamIndex = -1; - auto TypeRecs = IntrinsicDef->getValueAsListOfDefs("Types"); - unsigned TypeRecsSize = TypeRecs.size(); - // Populate return type and parameter type names - for (unsigned i = 0; i < TypeRecsSize; i++) { - auto TR = TypeRecs[i]; - OpTypes.emplace_back(getValueType(TR->getValueAsDef("VT"))); - // Get the overload parameter index. - // TODO : Seems hacky. Is it possible that more than one parameter can - // be of overload kind?? - // TODO: Check for any additional constraints specified for DXIL operation - // restricting return type. - if (i > 0) { - auto &CurParam = OpTypes.back(); - if (getParameterKind(CurParam) >= ParameterKind::OVERLOAD) { - OverloadParamIndex = i; - } - } - } - // Get the operation class - OpClass = R->getValueAsDef("OpClass")->getName(); - - // NOTE: For now, assume that attributes of DXIL Operation are the same as + // TODO: For now, assume that attributes of DXIL Operation are the same as // that of the intrinsic. Deviations are expected to be encoded in TableGen // record specification and handled accordingly here. Support to be added - // later. + // as needed. auto IntrPropList = IntrinsicDef->getValueAsListInit("IntrProperties"); auto IntrPropListSize = IntrPropList->size(); for (unsigned i = 0; i < IntrPropListSize; i++) { @@ -191,12 +234,13 @@ static std::string getParameterKindStr(ParameterKind Kind) { } /// Return a string representation of OverloadKind enum that maps to -/// input Simple Value Type enum -/// \param VT Simple Value Type enum +/// input LLVMType record +/// \param R TableGen def record of class LLVMType /// \return std::string string representation of OverloadKind -static std::string getOverloadKindStr(MVT::SimpleValueType VT) { - switch (VT) { +static std::string getOverloadKindStr(const Record *R) { + auto VTRec = R->getValueAsDef("VT"); + switch (getValueType(VTRec)) { case MVT::isVoid: return "OverloadKind::VOID"; case MVT::f16: @@ -219,6 +263,16 @@ static std::string getOverloadKindStr(MVT::SimpleValueType VT) { return "OverloadKind::I16 | OverloadKind::I32 | OverloadKind::I64"; case MVT::fAny: return "OverloadKind::HALF | OverloadKind::FLOAT | OverloadKind::DOUBLE"; + case MVT::Other: + // Handle DXIL-specific overload types + { + if (R->getValueAsInt("isHalfOrFloat")) { + return "OverloadKind::HALF | OverloadKind::FLOAT"; + } else if (R->getValueAsInt("isI16OrI32")) { + return "OverloadKind::I16 | OverloadKind::I32"; + } + } + LLVM_FALLTHROUGH; default: llvm_unreachable( "Support for specified parameter OverloadKind not yet implemented"); -- GitLab From e9492ccae085b5feb850ff17a96fe8211f7f6d7d Mon Sep 17 00:00:00 2001 From: Jason Eckhardt Date: Tue, 12 Mar 2024 16:01:58 -0500 Subject: [PATCH 0046/1407] [TableGen] DecoderEmitter clean-ups and modernization. (#84832) The decoder emitter is showing some signs of age. This patch makes a few kinds of clean-ups: - Use ranged-for more widely, including using enumerate() for those loops maintaining a loop index along with the items. - Reduce the number of arguments to fieldFromInsn (removes an out reference parameter: CodingStandards). The insn_t argument to insnWithID can/should probably be removed soon too since modern C++ allows us to return a local container without a copy. - Use raw strings for the large emitted code segments. This enhances both readability and modifiability. --- llvm/utils/TableGen/DecoderEmitter.cpp | 653 ++++++++++++------------- 1 file changed, 317 insertions(+), 336 deletions(-) diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp index 88f245238138..dd78dc02159b 100644 --- a/llvm/utils/TableGen/DecoderEmitter.cpp +++ b/llvm/utils/TableGen/DecoderEmitter.cpp @@ -226,7 +226,7 @@ static BitsInit &getBitsField(const Record &def, StringRef str) { VarLenInst VLI = VarLenInst(cast(RV->getValue()), RV); SmallVector Bits; - for (auto &SI : VLI) { + for (const auto &SI : VLI) { if (const BitsInit *BI = dyn_cast(SI.Value)) { for (unsigned Idx = 0U; Idx < BI->getNumBits(); ++Idx) { Bits.push_back(BI->getBit(Idx)); @@ -441,16 +441,15 @@ public: protected: // Populates the insn given the uid. void insnWithID(insn_t &Insn, unsigned Opcode) const { - BitsInit &Bits = getBitsField(*AllInstructions[Opcode].EncodingDef, "Inst"); - Insn.resize(BitWidth > Bits.getNumBits() ? BitWidth : Bits.getNumBits(), - BIT_UNSET); + const Record *EncodingDef = AllInstructions[Opcode].EncodingDef; + BitsInit &Bits = getBitsField(*EncodingDef, "Inst"); + Insn.resize(std::max(BitWidth, Bits.getNumBits()), BIT_UNSET); // We may have a SoftFail bitmask, which specifies a mask where an encoding // may differ from the value in "Inst" and yet still be valid, but the // disassembler should return SoftFail instead of Success. // // This is used for marking UNPREDICTABLE instructions in the ARM world. - const RecordVal *RV = - AllInstructions[Opcode].EncodingDef->getValue("SoftFail"); + const RecordVal *RV = EncodingDef->getValue("SoftFail"); const BitsInit *SFBits = RV ? dyn_cast(RV->getValue()) : nullptr; for (unsigned i = 0; i < Bits.getNumBits(); ++i) { if (SFBits && bitFromBits(*SFBits, i) == BIT_TRUE) @@ -472,10 +471,11 @@ protected: // Populates the field of the insn given the start position and the number of // consecutive bits to scan for. // - // Returns false if there exists any uninitialized bit value in the range. - // Returns true, otherwise. - bool fieldFromInsn(uint64_t &Field, insn_t &Insn, unsigned StartBit, - unsigned NumBits) const; + // Returns a pair of values (indicator, field), where the indicator is false + // if there exists any uninitialized bit value in the range and true if all + // bits are well-known. The second value is the potentially populated field. + std::pair fieldFromInsn(const insn_t &Insn, unsigned StartBit, + unsigned NumBits) const; /// dumpFilterArray - dumpFilterArray prints out debugging info for the given /// filter array as a series of chars. @@ -581,26 +581,25 @@ Filter::Filter(FilterChooser &owner, unsigned startBit, unsigned numBits, NumFiltered = 0; LastOpcFiltered = {0, 0}; - for (unsigned i = 0, e = Owner->Opcodes.size(); i != e; ++i) { + for (const auto &OpcPair : Owner->Opcodes) { insn_t Insn; // Populates the insn given the uid. - Owner->insnWithID(Insn, Owner->Opcodes[i].EncodingID); + Owner->insnWithID(Insn, OpcPair.EncodingID); - uint64_t Field; // Scans the segment for possibly well-specified encoding bits. - bool ok = Owner->fieldFromInsn(Field, Insn, StartBit, NumBits); + auto [Ok, Field] = Owner->fieldFromInsn(Insn, StartBit, NumBits); - if (ok) { + if (Ok) { // The encoding bits are well-known. Lets add the uid of the // instruction into the bucket keyed off the constant field value. - LastOpcFiltered = Owner->Opcodes[i]; + LastOpcFiltered = OpcPair; FilteredInstructions[Field].push_back(LastOpcFiltered); ++NumFiltered; } else { // Some of the encoding bit(s) are unspecified. This contributes to // one additional member of "Variable" instructions. - VariableInstructions.push_back(Owner->Opcodes[i]); + VariableInstructions.push_back(OpcPair); } } @@ -699,7 +698,7 @@ void Filter::emitTableEntry(DecoderTableInfo &TableInfo) const { size_t PrevFilter = 0; bool HasFallthrough = false; - for (auto &Filter : FilterChooserMap) { + for (const auto &Filter : FilterChooserMap) { // Field value -1 implies a non-empty set of variable instructions. // See also recurse(). if (Filter.first == NO_FIXED_SEGMENTS_SENTINEL) { @@ -784,7 +783,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table, // is used below to index into NumberedEncodings. DenseMap OpcodeToEncodingID; OpcodeToEncodingID.reserve(EncodingIDs.size()); - for (auto &EI : EncodingIDs) + for (const auto &EI : EncodingIDs) OpcodeToEncodingID[EI.Opcode] = EI.EncodingID; OS.indent(Indentation) << "static const uint8_t DecoderTable" << Namespace @@ -1038,27 +1037,29 @@ void DecoderEmitter::emitDecoderFunction(formatted_raw_ostream &OS, } OS.indent(Indentation) << "}\n"; Indentation -= 2; - OS.indent(Indentation) << "}\n\n"; + OS.indent(Indentation) << "}\n"; } // Populates the field of the insn given the start position and the number of // consecutive bits to scan for. // -// Returns false if and on the first uninitialized bit value encountered. -// Returns true, otherwise. -bool FilterChooser::fieldFromInsn(uint64_t &Field, insn_t &Insn, - unsigned StartBit, unsigned NumBits) const { - Field = 0; +// Returns a pair of values (indicator, field), where the indicator is false +// if there exists any uninitialized bit value in the range and true if all +// bits are well-known. The second value is the potentially populated field. +std::pair FilterChooser::fieldFromInsn(const insn_t &Insn, + unsigned StartBit, + unsigned NumBits) const { + uint64_t Field = 0; for (unsigned i = 0; i < NumBits; ++i) { if (Insn[StartBit + i] == BIT_UNSET) - return false; + return {false, Field}; if (Insn[StartBit + i] == BIT_TRUE) Field = Field | (1ULL << i); } - return true; + return {true, Field}; } /// dumpFilterArray - dumpFilterArray prints out debugging info for the given @@ -1246,14 +1247,14 @@ unsigned FilterChooser::getDecoderIndex(DecoderSet &Decoders, unsigned Opc, // If ParenIfBinOp is true, print a surrounding () if Val uses && or ||. bool FilterChooser::emitPredicateMatchAux(const Init &Val, bool ParenIfBinOp, raw_ostream &OS) const { - if (auto *D = dyn_cast(&Val)) { + if (const auto *D = dyn_cast(&Val)) { if (!D->getDef()->isSubClassOf("SubtargetFeature")) return true; OS << "Bits[" << Emitter->PredicateNamespace << "::" << D->getAsString() << "]"; return false; } - if (auto *D = dyn_cast(&Val)) { + if (const auto *D = dyn_cast(&Val)) { std::string Op = D->getOperator()->getAsString(); if (Op == "not" && D->getNumArgs() == 1) { OS << '!'; @@ -1350,9 +1351,9 @@ void FilterChooser::emitPredicateTableEntry(DecoderTableInfo &TableInfo, encodeULEB128(PIdx, S); TableInfo.Table.push_back(MCD::OPC_CheckPredicate); - // Predicate index - for (unsigned i = 0, e = PBytes.size(); i != e; ++i) - TableInfo.Table.push_back(PBytes[i]); + // Predicate index. + for (const auto PB : PBytes) + TableInfo.Table.push_back(PB); // Push location for NumToSkip backpatching. TableInfo.FixupStack.back().push_back(TableInfo.Table.size()); TableInfo.Table.push_back(0); @@ -1362,13 +1363,13 @@ void FilterChooser::emitPredicateTableEntry(DecoderTableInfo &TableInfo, void FilterChooser::emitSoftFailTableEntry(DecoderTableInfo &TableInfo, unsigned Opc) const { - const RecordVal *RV = AllInstructions[Opc].EncodingDef->getValue("SoftFail"); + const Record *EncodingDef = AllInstructions[Opc].EncodingDef; + const RecordVal *RV = EncodingDef->getValue("SoftFail"); BitsInit *SFBits = RV ? dyn_cast(RV->getValue()) : nullptr; if (!SFBits) return; - BitsInit *InstBits = - AllInstructions[Opc].EncodingDef->getValueAsBitsInit("Inst"); + BitsInit *InstBits = EncodingDef->getValueAsBitsInit("Inst"); APInt PositiveMask(BitWidth, 0ULL); APInt NegativeMask(BitWidth, 0ULL); @@ -1495,9 +1496,9 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo, raw_svector_ostream S(Bytes); encodeULEB128(DIdx, S); - // Decoder index - for (unsigned i = 0, e = Bytes.size(); i != e; ++i) - TableInfo.Table.push_back(Bytes[i]); + // Decoder index. + for (const auto B : Bytes) + TableInfo.Table.push_back(B); if (!HasCompleteDecoder) { // Push location for NumToSkip backpatching. @@ -1566,7 +1567,7 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) { if (AllowMixed && !Greedy) { assert(numInstructions == 3); - for (auto Opcode : Opcodes) { + for (const auto &Opcode : Opcodes) { std::vector StartBits; std::vector EndBits; std::vector FieldVals; @@ -1613,10 +1614,10 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) { else bitAttrs.push_back(ATTR_NONE); - for (unsigned InsnIndex = 0; InsnIndex < numInstructions; ++InsnIndex) { + for (const auto &OpcPair : Opcodes) { insn_t insn; - insnWithID(insn, Opcodes[InsnIndex].EncodingID); + insnWithID(insn, OpcPair.EncodingID); for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex) { switch (bitAttrs[BitIndex]) { @@ -1760,14 +1761,14 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) { bool AllUseless = true; unsigned BestScore = 0; - for (unsigned i = 0, e = Filters.size(); i != e; ++i) { - unsigned Usefulness = Filters[i].usefulness(); + for (const auto &[Idx, Filter] : enumerate(Filters)) { + unsigned Usefulness = Filter.usefulness(); if (Usefulness) AllUseless = false; if (Usefulness > BestScore) { - BestIndex = i; + BestIndex = Idx; BestScore = Usefulness; } } @@ -1892,8 +1893,7 @@ void parseVarLenInstOperand(const Record &Def, VarLenInst VLI(cast(RV->getValue()), RV); SmallVector TiedTo; - for (unsigned Idx = 0; Idx < CGI.Operands.size(); ++Idx) { - auto &Op = CGI.Operands[Idx]; + for (const auto &[Idx, Op] : enumerate(CGI.Operands)) { if (Op.MIOperandInfo && Op.MIOperandInfo->getNumArgs() > 0) for (auto *Arg : Op.MIOperandInfo->getArgs()) Operands.push_back(getOpInfo(cast(Arg)->getDef())); @@ -1909,7 +1909,7 @@ void parseVarLenInstOperand(const Record &Def, } unsigned CurrBitPos = 0; - for (auto &EncodingSegment : VLI) { + for (const auto &EncodingSegment : VLI) { unsigned Offset = 0; StringRef OpName; @@ -2028,26 +2028,23 @@ populateInstruction(CodeGenTarget &Target, const Record &EncodingDef, std::vector> InOutOperands; DagInit *Out = Def.getValueAsDag("OutOperandList"); DagInit *In = Def.getValueAsDag("InOperandList"); - for (unsigned i = 0; i < Out->getNumArgs(); ++i) - InOutOperands.push_back(std::pair(Out->getArg(i), Out->getArgNameStr(i))); - for (unsigned i = 0; i < In->getNumArgs(); ++i) - InOutOperands.push_back(std::pair(In->getArg(i), In->getArgNameStr(i))); + for (const auto &[Idx, Arg] : enumerate(Out->getArgs())) + InOutOperands.push_back(std::pair(Arg, Out->getArgNameStr(Idx))); + for (const auto &[Idx, Arg] : enumerate(In->getArgs())) + InOutOperands.push_back(std::pair(Arg, In->getArgNameStr(Idx))); // Search for tied operands, so that we can correctly instantiate // operands that are not explicitly represented in the encoding. std::map TiedNames; - for (unsigned i = 0; i < CGI.Operands.size(); ++i) { - auto &Op = CGI.Operands[i]; - for (unsigned j = 0; j < Op.Constraints.size(); ++j) { - const CGIOperandList::ConstraintInfo &CI = Op.Constraints[j]; + for (const auto &[I, Op] : enumerate(CGI.Operands)) { + for (const auto &[J, CI] : enumerate(Op.Constraints)) { if (CI.isTied()) { - int tiedTo = CI.getTiedOperand(); std::pair SO = - CGI.Operands.getSubOperandNumber(tiedTo); + CGI.Operands.getSubOperandNumber(CI.getTiedOperand()); std::string TiedName = CGI.Operands[SO.first].SubOpNames[SO.second]; if (TiedName.empty()) TiedName = CGI.Operands[SO.first].Name; - std::string MyName = Op.SubOpNames[j]; + std::string MyName = Op.SubOpNames[J]; if (MyName.empty()) MyName = Op.Name; @@ -2099,10 +2096,9 @@ populateInstruction(CodeGenTarget &Target, const Record &EncodingDef, // Decode each of the sub-ops separately. assert(SubOps && SubArgDag->getNumArgs() == SubOps->getNumArgs()); - for (unsigned i = 0; i < SubOps->getNumArgs(); ++i) { - StringRef SubOpName = SubArgDag->getArgNameStr(i); - OperandInfo SubOpInfo = - getOpInfo(cast(SubOps->getArg(i))->getDef()); + for (const auto &[I, Arg] : enumerate(SubOps->getArgs())) { + StringRef SubOpName = SubArgDag->getArgNameStr(I); + OperandInfo SubOpInfo = getOpInfo(cast(Arg)->getDef()); addOneOperandFields(EncodingDef, Bits, TiedNames, SubOpName, SubOpInfo); @@ -2169,273 +2165,253 @@ populateInstruction(CodeGenTarget &Target, const Record &EncodingDef, // using the VS compiler. It has a bug which causes the function // to be optimized out in some circumstances. See llvm.org/pr38292 static void emitFieldFromInstruction(formatted_raw_ostream &OS) { - OS << "// Helper functions for extracting fields from encoded instructions.\n" - << "// InsnType must either be integral or an APInt-like object that " - "must:\n" - << "// * be default-constructible and copy-constructible\n" - << "// * be constructible from an APInt (this can be private)\n" - << "// * Support insertBits(bits, startBit, numBits)\n" - << "// * Support extractBitsAsZExtValue(numBits, startBit)\n" - << "// * Support the ~, &, ==, and != operators with other objects of " - "the same type\n" - << "// * Support the != and bitwise & with uint64_t\n" - << "// * Support put (<<) to raw_ostream&\n" - << "template \n" - << "#if defined(_MSC_VER) && !defined(__clang__)\n" - << "__declspec(noinline)\n" - << "#endif\n" - << "static std::enable_if_t::value, InsnType>\n" - << "fieldFromInstruction(const InsnType &insn, unsigned startBit,\n" - << " unsigned numBits) {\n" - << " assert(startBit + numBits <= 64 && \"Cannot support >64-bit " - "extractions!\");\n" - << " assert(startBit + numBits <= (sizeof(InsnType) * 8) &&\n" - << " \"Instruction field out of bounds!\");\n" - << " InsnType fieldMask;\n" - << " if (numBits == sizeof(InsnType) * 8)\n" - << " fieldMask = (InsnType)(-1LL);\n" - << " else\n" - << " fieldMask = (((InsnType)1 << numBits) - 1) << startBit;\n" - << " return (insn & fieldMask) >> startBit;\n" - << "}\n" - << "\n" - << "template \n" - << "static std::enable_if_t::value, " - "uint64_t>\n" - << "fieldFromInstruction(const InsnType &insn, unsigned startBit,\n" - << " unsigned numBits) {\n" - << " return insn.extractBitsAsZExtValue(numBits, startBit);\n" - << "}\n\n"; + OS << R"( +// Helper functions for extracting fields from encoded instructions. +// InsnType must either be integral or an APInt-like object that must: +// * be default-constructible and copy-constructible +// * be constructible from an APInt (this can be private) +// * Support insertBits(bits, startBit, numBits) +// * Support extractBitsAsZExtValue(numBits, startBit) +// * Support the ~, &, ==, and != operators with other objects of the same type +// * Support the != and bitwise & with uint64_t +// * Support put (<<) to raw_ostream& +template +#if defined(_MSC_VER) && !defined(__clang__) +__declspec(noinline) +#endif +static std::enable_if_t::value, InsnType> +fieldFromInstruction(const InsnType &insn, unsigned startBit, + unsigned numBits) { + assert(startBit + numBits <= 64 && "Cannot support >64-bit extractions!"); + assert(startBit + numBits <= (sizeof(InsnType) * 8) && + "Instruction field out of bounds!"); + InsnType fieldMask; + if (numBits == sizeof(InsnType) * 8) + fieldMask = (InsnType)(-1LL); + else + fieldMask = (((InsnType)1 << numBits) - 1) << startBit; + return (insn & fieldMask) >> startBit; +} + +template +static std::enable_if_t::value, uint64_t> +fieldFromInstruction(const InsnType &insn, unsigned startBit, + unsigned numBits) { + return insn.extractBitsAsZExtValue(numBits, startBit); +} +)"; } // emitInsertBits - Emit the templated helper function insertBits(). static void emitInsertBits(formatted_raw_ostream &OS) { - OS << "// Helper function for inserting bits extracted from an encoded " - "instruction into\n" - << "// a field.\n" - << "template \n" - << "static std::enable_if_t::value>\n" - << "insertBits(InsnType &field, InsnType bits, unsigned startBit, " - "unsigned numBits) {\n" - << " assert(startBit + numBits <= sizeof field * 8);\n" - << " field |= (InsnType)bits << startBit;\n" - << "}\n" - << "\n" - << "template \n" - << "static std::enable_if_t::value>\n" - << "insertBits(InsnType &field, uint64_t bits, unsigned startBit, " - "unsigned numBits) {\n" - << " field.insertBits(bits, startBit, numBits);\n" - << "}\n\n"; + OS << R"( +// Helper function for inserting bits extracted from an encoded instruction into +// a field. +template +static std::enable_if_t::value> +insertBits(InsnType &field, InsnType bits, unsigned startBit, unsigned numBits) { + assert(startBit + numBits <= sizeof field * 8); + field |= (InsnType)bits << startBit; +} + +template +static std::enable_if_t::value> +insertBits(InsnType &field, uint64_t bits, unsigned startBit, unsigned numBits) { + field.insertBits(bits, startBit, numBits); +} +)"; } // emitDecodeInstruction - Emit the templated helper function // decodeInstruction(). static void emitDecodeInstruction(formatted_raw_ostream &OS, bool IsVarLenInst) { - OS << "template \n" - << "static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], " - "MCInst &MI,\n" - << " InsnType insn, uint64_t " - "Address,\n" - << " const MCDisassembler *DisAsm,\n" - << " const MCSubtargetInfo &STI"; + OS << R"( +template +static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI, + InsnType insn, uint64_t Address, + const MCDisassembler *DisAsm, + const MCSubtargetInfo &STI)"; if (IsVarLenInst) { - OS << ",\n" - << " llvm::function_ref makeUp"; + OS << ",\n " + "llvm::function_ref makeUp"; } - OS << ") {\n" - << " const FeatureBitset &Bits = STI.getFeatureBits();\n" - << "\n" - << " const uint8_t *Ptr = DecodeTable;\n" - << " uint64_t CurFieldValue = 0;\n" - << " DecodeStatus S = MCDisassembler::Success;\n" - << " while (true) {\n" - << " ptrdiff_t Loc = Ptr - DecodeTable;\n" - << " switch (*Ptr) {\n" - << " default:\n" - << " errs() << Loc << \": Unexpected decode table opcode!\\n\";\n" - << " return MCDisassembler::Fail;\n" - << " case MCD::OPC_ExtractField: {\n" - << " // Decode the start value.\n" - << " unsigned DecodedLen;\n" - << " unsigned Start = decodeULEB128(++Ptr, &DecodedLen);\n" - << " Ptr += DecodedLen;\n" - << " unsigned Len = *Ptr++;\n"; + OS << R"() { + const FeatureBitset &Bits = STI.getFeatureBits(); + + const uint8_t *Ptr = DecodeTable; + uint64_t CurFieldValue = 0; + DecodeStatus S = MCDisassembler::Success; + while (true) { + ptrdiff_t Loc = Ptr - DecodeTable; + switch (*Ptr) { + default: + errs() << Loc << ": Unexpected decode table opcode!\n"; + return MCDisassembler::Fail; + case MCD::OPC_ExtractField: { + // Decode the start value. + unsigned DecodedLen; + unsigned Start = decodeULEB128(++Ptr, &DecodedLen); + Ptr += DecodedLen; + unsigned Len = *Ptr++;)"; if (IsVarLenInst) - OS << " makeUp(insn, Start + Len);\n"; - OS << " CurFieldValue = fieldFromInstruction(insn, Start, Len);\n" - << " LLVM_DEBUG(dbgs() << Loc << \": OPC_ExtractField(\" << Start << " - "\", \"\n" - << " << Len << \"): \" << CurFieldValue << \"\\n\");\n" - << " break;\n" - << " }\n" - << " case MCD::OPC_FilterValue: {\n" - << " // Decode the field value.\n" - << " unsigned Len;\n" - << " uint64_t Val = decodeULEB128(++Ptr, &Len);\n" - << " Ptr += Len;\n" - << " // NumToSkip is a plain 24-bit integer.\n" - << " unsigned NumToSkip = *Ptr++;\n" - << " NumToSkip |= (*Ptr++) << 8;\n" - << " NumToSkip |= (*Ptr++) << 16;\n" - << "\n" - << " // Perform the filter operation.\n" - << " if (Val != CurFieldValue)\n" - << " Ptr += NumToSkip;\n" - << " LLVM_DEBUG(dbgs() << Loc << \": OPC_FilterValue(\" << Val << " - "\", \" << NumToSkip\n" - << " << \"): \" << ((Val != CurFieldValue) ? \"FAIL:\" " - ": \"PASS:\")\n" - << " << \" continuing at \" << (Ptr - DecodeTable) << " - "\"\\n\");\n" - << "\n" - << " break;\n" - << " }\n" - << " case MCD::OPC_CheckField: {\n" - << " // Decode the start value.\n" - << " unsigned Len;\n" - << " unsigned Start = decodeULEB128(++Ptr, &Len);\n" - << " Ptr += Len;\n" - << " Len = *Ptr;\n"; + OS << "\n makeUp(insn, Start + Len);"; + OS << R"( + CurFieldValue = fieldFromInstruction(insn, Start, Len); + LLVM_DEBUG(dbgs() << Loc << ": OPC_ExtractField(" << Start << ", " + << Len << "): " << CurFieldValue << "\n"); + break; + } + case MCD::OPC_FilterValue: { + // Decode the field value. + unsigned Len; + uint64_t Val = decodeULEB128(++Ptr, &Len); + Ptr += Len; + // NumToSkip is a plain 24-bit integer. + unsigned NumToSkip = *Ptr++; + NumToSkip |= (*Ptr++) << 8; + NumToSkip |= (*Ptr++) << 16; + + // Perform the filter operation. + if (Val != CurFieldValue) + Ptr += NumToSkip; + LLVM_DEBUG(dbgs() << Loc << ": OPC_FilterValue(" << Val << ", " << NumToSkip + << "): " << ((Val != CurFieldValue) ? "FAIL:" : "PASS:") + << " continuing at " << (Ptr - DecodeTable) << "\n"); + + break; + } + case MCD::OPC_CheckField: { + // Decode the start value. + unsigned Len; + unsigned Start = decodeULEB128(++Ptr, &Len); + Ptr += Len; + Len = *Ptr;)"; if (IsVarLenInst) - OS << " makeUp(insn, Start + Len);\n"; - OS << " uint64_t FieldValue = fieldFromInstruction(insn, Start, Len);\n" - << " // Decode the field value.\n" - << " unsigned PtrLen = 0;\n" - << " uint64_t ExpectedValue = decodeULEB128(++Ptr, &PtrLen);\n" - << " Ptr += PtrLen;\n" - << " // NumToSkip is a plain 24-bit integer.\n" - << " unsigned NumToSkip = *Ptr++;\n" - << " NumToSkip |= (*Ptr++) << 8;\n" - << " NumToSkip |= (*Ptr++) << 16;\n" - << "\n" - << " // If the actual and expected values don't match, skip.\n" - << " if (ExpectedValue != FieldValue)\n" - << " Ptr += NumToSkip;\n" - << " LLVM_DEBUG(dbgs() << Loc << \": OPC_CheckField(\" << Start << " - "\", \"\n" - << " << Len << \", \" << ExpectedValue << \", \" << " - "NumToSkip\n" - << " << \"): FieldValue = \" << FieldValue << \", " - "ExpectedValue = \"\n" - << " << ExpectedValue << \": \"\n" - << " << ((ExpectedValue == FieldValue) ? \"PASS\\n\" : " - "\"FAIL\\n\"));\n" - << " break;\n" - << " }\n" - << " case MCD::OPC_CheckPredicate: {\n" - << " unsigned Len;\n" - << " // Decode the Predicate Index value.\n" - << " unsigned PIdx = decodeULEB128(++Ptr, &Len);\n" - << " Ptr += Len;\n" - << " // NumToSkip is a plain 24-bit integer.\n" - << " unsigned NumToSkip = *Ptr++;\n" - << " NumToSkip |= (*Ptr++) << 8;\n" - << " NumToSkip |= (*Ptr++) << 16;\n" - << " // Check the predicate.\n" - << " bool Pred;\n" - << " if (!(Pred = checkDecoderPredicate(PIdx, Bits)))\n" - << " Ptr += NumToSkip;\n" - << " (void)Pred;\n" - << " LLVM_DEBUG(dbgs() << Loc << \": OPC_CheckPredicate(\" << PIdx " - "<< \"): \"\n" - << " << (Pred ? \"PASS\\n\" : \"FAIL\\n\"));\n" - << "\n" - << " break;\n" - << " }\n" - << " case MCD::OPC_Decode: {\n" - << " unsigned Len;\n" - << " // Decode the Opcode value.\n" - << " unsigned Opc = decodeULEB128(++Ptr, &Len);\n" - << " Ptr += Len;\n" - << " unsigned DecodeIdx = decodeULEB128(Ptr, &Len);\n" - << " Ptr += Len;\n" - << "\n" - << " MI.clear();\n" - << " MI.setOpcode(Opc);\n" - << " bool DecodeComplete;\n"; + OS << "\n makeUp(insn, Start + Len);"; + OS << R"( + uint64_t FieldValue = fieldFromInstruction(insn, Start, Len); + // Decode the field value. + unsigned PtrLen = 0; + uint64_t ExpectedValue = decodeULEB128(++Ptr, &PtrLen); + Ptr += PtrLen; + // NumToSkip is a plain 24-bit integer. + unsigned NumToSkip = *Ptr++; + NumToSkip |= (*Ptr++) << 8; + NumToSkip |= (*Ptr++) << 16; + + // If the actual and expected values don't match, skip. + if (ExpectedValue != FieldValue) + Ptr += NumToSkip; + LLVM_DEBUG(dbgs() << Loc << ": OPC_CheckField(" << Start << ", " + << Len << ", " << ExpectedValue << ", " << NumToSkip + << "): FieldValue = " << FieldValue << ", ExpectedValue = " + << ExpectedValue << ": " + << ((ExpectedValue == FieldValue) ? "PASS\n" : "FAIL\n")); + break; + } + case MCD::OPC_CheckPredicate: { + unsigned Len; + // Decode the Predicate Index value. + unsigned PIdx = decodeULEB128(++Ptr, &Len); + Ptr += Len; + // NumToSkip is a plain 24-bit integer. + unsigned NumToSkip = *Ptr++; + NumToSkip |= (*Ptr++) << 8; + NumToSkip |= (*Ptr++) << 16; + // Check the predicate. + bool Pred; + if (!(Pred = checkDecoderPredicate(PIdx, Bits))) + Ptr += NumToSkip; + (void)Pred; + LLVM_DEBUG(dbgs() << Loc << ": OPC_CheckPredicate(" << PIdx << "): " + << (Pred ? "PASS\n" : "FAIL\n")); + + break; + } + case MCD::OPC_Decode: { + unsigned Len; + // Decode the Opcode value. + unsigned Opc = decodeULEB128(++Ptr, &Len); + Ptr += Len; + unsigned DecodeIdx = decodeULEB128(Ptr, &Len); + Ptr += Len; + + MI.clear(); + MI.setOpcode(Opc); + bool DecodeComplete;)"; if (IsVarLenInst) { - OS << " Len = InstrLenTable[Opc];\n" - << " makeUp(insn, Len);\n"; + OS << "\n Len = InstrLenTable[Opc];\n" + << " makeUp(insn, Len);"; + } + OS << R"( + S = decodeToMCInst(S, DecodeIdx, insn, MI, Address, DisAsm, DecodeComplete); + assert(DecodeComplete); + + LLVM_DEBUG(dbgs() << Loc << ": OPC_Decode: opcode " << Opc + << ", using decoder " << DecodeIdx << ": " + << (S != MCDisassembler::Fail ? "PASS" : "FAIL") << "\n"); + return S; + } + case MCD::OPC_TryDecode: { + unsigned Len; + // Decode the Opcode value. + unsigned Opc = decodeULEB128(++Ptr, &Len); + Ptr += Len; + unsigned DecodeIdx = decodeULEB128(Ptr, &Len); + Ptr += Len; + // NumToSkip is a plain 24-bit integer. + unsigned NumToSkip = *Ptr++; + NumToSkip |= (*Ptr++) << 8; + NumToSkip |= (*Ptr++) << 16; + + // Perform the decode operation. + MCInst TmpMI; + TmpMI.setOpcode(Opc); + bool DecodeComplete; + S = decodeToMCInst(S, DecodeIdx, insn, TmpMI, Address, DisAsm, DecodeComplete); + LLVM_DEBUG(dbgs() << Loc << ": OPC_TryDecode: opcode " << Opc + << ", using decoder " << DecodeIdx << ": "); + + if (DecodeComplete) { + // Decoding complete. + LLVM_DEBUG(dbgs() << (S != MCDisassembler::Fail ? "PASS" : "FAIL") << "\n"); + MI = TmpMI; + return S; + } else { + assert(S == MCDisassembler::Fail); + // If the decoding was incomplete, skip. + Ptr += NumToSkip; + LLVM_DEBUG(dbgs() << "FAIL: continuing at " << (Ptr - DecodeTable) << "\n"); + // Reset decode status. This also drops a SoftFail status that could be + // set before the decode attempt. + S = MCDisassembler::Success; + } + break; + } + case MCD::OPC_SoftFail: { + // Decode the mask values. + unsigned Len; + uint64_t PositiveMask = decodeULEB128(++Ptr, &Len); + Ptr += Len; + uint64_t NegativeMask = decodeULEB128(Ptr, &Len); + Ptr += Len; + bool Fail = (insn & PositiveMask) != 0 || (~insn & NegativeMask) != 0; + if (Fail) + S = MCDisassembler::SoftFail; + LLVM_DEBUG(dbgs() << Loc << ": OPC_SoftFail: " << (Fail ? "FAIL\n" : "PASS\n")); + break; + } + case MCD::OPC_Fail: { + LLVM_DEBUG(dbgs() << Loc << ": OPC_Fail\n"); + return MCDisassembler::Fail; + } + } } - OS << " S = decodeToMCInst(S, DecodeIdx, insn, MI, Address, DisAsm, " - "DecodeComplete);\n" - << " assert(DecodeComplete);\n" - << "\n" - << " LLVM_DEBUG(dbgs() << Loc << \": OPC_Decode: opcode \" << Opc\n" - << " << \", using decoder \" << DecodeIdx << \": \"\n" - << " << (S != MCDisassembler::Fail ? \"PASS\" : " - "\"FAIL\") << \"\\n\");\n" - << " return S;\n" - << " }\n" - << " case MCD::OPC_TryDecode: {\n" - << " unsigned Len;\n" - << " // Decode the Opcode value.\n" - << " unsigned Opc = decodeULEB128(++Ptr, &Len);\n" - << " Ptr += Len;\n" - << " unsigned DecodeIdx = decodeULEB128(Ptr, &Len);\n" - << " Ptr += Len;\n" - << " // NumToSkip is a plain 24-bit integer.\n" - << " unsigned NumToSkip = *Ptr++;\n" - << " NumToSkip |= (*Ptr++) << 8;\n" - << " NumToSkip |= (*Ptr++) << 16;\n" - << "\n" - << " // Perform the decode operation.\n" - << " MCInst TmpMI;\n" - << " TmpMI.setOpcode(Opc);\n" - << " bool DecodeComplete;\n" - << " S = decodeToMCInst(S, DecodeIdx, insn, TmpMI, Address, DisAsm, " - "DecodeComplete);\n" - << " LLVM_DEBUG(dbgs() << Loc << \": OPC_TryDecode: opcode \" << " - "Opc\n" - << " << \", using decoder \" << DecodeIdx << \": \");\n" - << "\n" - << " if (DecodeComplete) {\n" - << " // Decoding complete.\n" - << " LLVM_DEBUG(dbgs() << (S != MCDisassembler::Fail ? \"PASS\" : " - "\"FAIL\") << \"\\n\");\n" - << " MI = TmpMI;\n" - << " return S;\n" - << " } else {\n" - << " assert(S == MCDisassembler::Fail);\n" - << " // If the decoding was incomplete, skip.\n" - << " Ptr += NumToSkip;\n" - << " LLVM_DEBUG(dbgs() << \"FAIL: continuing at \" << (Ptr - " - "DecodeTable) << \"\\n\");\n" - << " // Reset decode status. This also drops a SoftFail status " - "that could be\n" - << " // set before the decode attempt.\n" - << " S = MCDisassembler::Success;\n" - << " }\n" - << " break;\n" - << " }\n" - << " case MCD::OPC_SoftFail: {\n" - << " // Decode the mask values.\n" - << " unsigned Len;\n" - << " uint64_t PositiveMask = decodeULEB128(++Ptr, &Len);\n" - << " Ptr += Len;\n" - << " uint64_t NegativeMask = decodeULEB128(Ptr, &Len);\n" - << " Ptr += Len;\n" - << " bool Fail = (insn & PositiveMask) != 0 || (~insn & " - "NegativeMask) != 0;\n" - << " if (Fail)\n" - << " S = MCDisassembler::SoftFail;\n" - << " LLVM_DEBUG(dbgs() << Loc << \": OPC_SoftFail: \" << (Fail ? " - "\"FAIL\\n\" : \"PASS\\n\"));\n" - << " break;\n" - << " }\n" - << " case MCD::OPC_Fail: {\n" - << " LLVM_DEBUG(dbgs() << Loc << \": OPC_Fail\\n\");\n" - << " return MCDisassembler::Fail;\n" - << " }\n" - << " }\n" - << " }\n" - << " llvm_unreachable(\"bogosity detected in disassembler state " - "machine!\");\n" - << "}\n\n"; + llvm_unreachable("bogosity detected in disassembler state machine!"); +} + +)"; } // Helper to propagate SoftFail status. Returns false if the status is Fail; @@ -2443,10 +2419,13 @@ static void emitDecodeInstruction(formatted_raw_ostream &OS, // is correct to propagate the values of this enum; see comment on 'enum // DecodeStatus'.) static void emitCheck(formatted_raw_ostream &OS) { - OS << "static bool Check(DecodeStatus &Out, DecodeStatus In) {\n" - << " Out = static_cast(Out & In);\n" - << " return Out != MCDisassembler::Fail;\n" - << "}\n\n"; + OS << R"( +static bool Check(DecodeStatus &Out, DecodeStatus In) { + Out = static_cast(Out & In); + return Out != MCDisassembler::Fail; +} + +)"; } // Collect all HwModes referenced by the target for encoding purposes, @@ -2469,16 +2448,18 @@ collectHwModesReferencedForEncodings(const CodeGenHwModes &HWM, // Emits disassembler code for instruction decoding. void DecoderEmitter::run(raw_ostream &o) { formatted_raw_ostream OS(o); - OS << "#include \"llvm/MC/MCInst.h\"\n"; - OS << "#include \"llvm/MC/MCSubtargetInfo.h\"\n"; - OS << "#include \"llvm/Support/DataTypes.h\"\n"; - OS << "#include \"llvm/Support/Debug.h\"\n"; - OS << "#include \"llvm/Support/LEB128.h\"\n"; - OS << "#include \"llvm/Support/raw_ostream.h\"\n"; - OS << "#include \"llvm/TargetParser/SubtargetFeature.h\"\n"; - OS << "#include \n"; - OS << '\n'; - OS << "namespace llvm {\n\n"; + OS << R"( +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/Support/DataTypes.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/LEB128.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TargetParser/SubtargetFeature.h" +#include + +namespace llvm { +)"; emitFieldFromInstruction(OS); emitInsertBits(OS); @@ -2533,9 +2514,9 @@ void DecoderEmitter::run(raw_ostream &o) { bool IsVarLenInst = Target.hasVariableLengthEncodings(); unsigned MaxInstLen = 0; - for (unsigned i = 0; i < NumberedEncodings.size(); ++i) { - const Record *EncodingDef = NumberedEncodings[i].EncodingDef; - const CodeGenInstruction *Inst = NumberedEncodings[i].Inst; + for (const auto &[NEI, NumberedEncoding] : enumerate(NumberedEncodings)) { + const Record *EncodingDef = NumberedEncoding.EncodingDef; + const CodeGenInstruction *Inst = NumberedEncoding.Inst; const Record *Def = Inst->TheDef; unsigned Size = EncodingDef->getValueAsInt("Size"); if (Def->getValueAsString("Namespace") == "TargetOpcode" || @@ -2546,7 +2527,7 @@ void DecoderEmitter::run(raw_ostream &o) { continue; } - if (i < NumberedInstructions.size()) + if (NEI < NumberedInstructions.size()) NumInstructions++; NumEncodings++; @@ -2556,19 +2537,19 @@ void DecoderEmitter::run(raw_ostream &o) { if (IsVarLenInst) InstrLen.resize(NumberedInstructions.size(), 0); - if (unsigned Len = populateInstruction(Target, *EncodingDef, *Inst, i, + if (unsigned Len = populateInstruction(Target, *EncodingDef, *Inst, NEI, Operands, IsVarLenInst)) { if (IsVarLenInst) { MaxInstLen = std::max(MaxInstLen, Len); - InstrLen[i] = Len; + InstrLen[NEI] = Len; } std::string DecoderNamespace = std::string(EncodingDef->getValueAsString("DecoderNamespace")); - if (!NumberedEncodings[i].HwModeName.empty()) + if (!NumberedEncoding.HwModeName.empty()) DecoderNamespace += - std::string("_") + NumberedEncodings[i].HwModeName.str(); + std::string("_") + NumberedEncoding.HwModeName.str(); OpcMap[std::pair(DecoderNamespace, Size)].emplace_back( - i, Target.getInstrIntValue(Def)); + NEI, Target.getInstrIntValue(Def)); } else { NumEncodingsOmitted++; } -- GitLab From 498b7d2f86b4bceb381e66b093670c7ec4bc6cc4 Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Tue, 12 Mar 2024 14:02:20 -0700 Subject: [PATCH 0047/1407] AMDGPU: Copy TSFlags from Pseudo to DS_Real (#84977) We need TSFalgs from pseudo to real. --- llvm/lib/Target/AMDGPU/DSInstructions.td | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index cc763df5a476..87ace01a6d0e 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -65,6 +65,7 @@ class DS_Real : let SubtargetPredicate = ps.SubtargetPredicate; let WaveSizePredicate = ps.WaveSizePredicate; let OtherPredicates = ps.OtherPredicates; + let TSFlags = ps.TSFlags; let SchedRW = ps.SchedRW; let mayLoad = ps.mayLoad; let mayStore = ps.mayStore; -- GitLab From 1a6ec906fb3781c2fc98979ec37a2a76479b0b08 Mon Sep 17 00:00:00 2001 From: Daniel Paoliello Date: Tue, 12 Mar 2024 14:10:49 -0700 Subject: [PATCH 0048/1407] [Arm64EC] Copy import descriptors to the EC Map (#84834) As noted in , MSVC places import descriptors in both the EC and regular map - that PR moved the descriptors to ONLY the regular map, however this causes linking errors when linking as Arm64EC: ``` bcryptprimitives.lib(bcryptprimitives.dll) : error LNK2001: unresolved external symbol __IMPORT_DESCRIPTOR_bcryptprimitives (EC Symbol) ``` This change copies import descriptors from the regular map to the EC map, which fixes this linking error. --- llvm/include/llvm/Object/COFFImportFile.h | 6 ++++++ llvm/lib/Object/ArchiveWriter.cpp | 11 +++++++++++ llvm/lib/Object/COFFImportFile.cpp | 10 ++++------ llvm/test/tools/llvm-dlltool/arm64ec.test | 6 ++++++ llvm/test/tools/llvm-lib/arm64ec-implib.test | 12 ++++++++++++ 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/llvm/include/llvm/Object/COFFImportFile.h b/llvm/include/llvm/Object/COFFImportFile.h index 402ded0d64fe..7268faa87eb7 100644 --- a/llvm/include/llvm/Object/COFFImportFile.h +++ b/llvm/include/llvm/Object/COFFImportFile.h @@ -26,6 +26,12 @@ namespace llvm { namespace object { +constexpr std::string_view ImportDescriptorPrefix = "__IMPORT_DESCRIPTOR_"; +constexpr std::string_view NullImportDescriptorSymbolName = + "__NULL_IMPORT_DESCRIPTOR"; +constexpr std::string_view NullThunkDataPrefix = "\x7f"; +constexpr std::string_view NullThunkDataSuffix = "_NULL_THUNK_DATA"; + class COFFImportFile : public SymbolicFile { private: enum SymbolIndex { ImpSymbol, ThunkSymbol, ECAuxSymbol, ECThunkSymbol }; diff --git a/llvm/lib/Object/ArchiveWriter.cpp b/llvm/lib/Object/ArchiveWriter.cpp index be51093933a8..e0629747b40c 100644 --- a/llvm/lib/Object/ArchiveWriter.cpp +++ b/llvm/lib/Object/ArchiveWriter.cpp @@ -677,6 +677,13 @@ static bool isECObject(object::SymbolicFile &Obj) { return false; } +bool isImportDescriptor(StringRef Name) { + return Name.starts_with(ImportDescriptorPrefix) || + Name == StringRef{NullImportDescriptorSymbolName} || + (Name.starts_with(NullThunkDataPrefix) && + Name.ends_with(NullThunkDataSuffix)); +} + static Expected> getSymbols(SymbolicFile *Obj, uint16_t Index, raw_ostream &SymNames, @@ -704,6 +711,10 @@ static Expected> getSymbols(SymbolicFile *Obj, if (Map == &SymMap->Map) { Ret.push_back(SymNames.tell()); SymNames << Name << '\0'; + // If EC is enabled, then the import descriptors are NOT put into EC + // objects so we need to copy them to the EC map manually. + if (SymMap->UseECMap && isImportDescriptor(Name)) + SymMap->ECMap[Name] = Index; } } else { Ret.push_back(SymNames.tell()); diff --git a/llvm/lib/Object/COFFImportFile.cpp b/llvm/lib/Object/COFFImportFile.cpp index 376dd126baf6..46c8e702581e 100644 --- a/llvm/lib/Object/COFFImportFile.cpp +++ b/llvm/lib/Object/COFFImportFile.cpp @@ -108,7 +108,7 @@ template static void append(std::vector &B, const T &Data) { } static void writeStringTable(std::vector &B, - ArrayRef Strings) { + ArrayRef Strings) { // The COFF string table consists of a 4-byte value which is the size of the // table, including the length field itself. This value is followed by the // string content itself, which is an array of null-terminated C-style @@ -171,9 +171,6 @@ static Expected replace(StringRef S, StringRef From, return (Twine(S.substr(0, Pos)) + To + S.substr(Pos + From.size())).str(); } -static const std::string NullImportDescriptorSymbolName = - "__NULL_IMPORT_DESCRIPTOR"; - namespace { // This class constructs various small object files necessary to support linking // symbols imported from a DLL. The contents are pretty strictly defined and @@ -192,8 +189,9 @@ class ObjectFactory { public: ObjectFactory(StringRef S, MachineTypes M) : NativeMachine(M), ImportName(S), Library(llvm::sys::path::stem(S)), - ImportDescriptorSymbolName(("__IMPORT_DESCRIPTOR_" + Library).str()), - NullThunkSymbolName(("\x7f" + Library + "_NULL_THUNK_DATA").str()) {} + ImportDescriptorSymbolName((ImportDescriptorPrefix + Library).str()), + NullThunkSymbolName( + (NullThunkDataPrefix + Library + NullThunkDataSuffix).str()) {} // Creates an Import Descriptor. This is a small object file which contains a // reference to the terminators and contains the library name (entry) for the diff --git a/llvm/test/tools/llvm-dlltool/arm64ec.test b/llvm/test/tools/llvm-dlltool/arm64ec.test index e742a77ff78a..b03b4eaf7b2d 100644 --- a/llvm/test/tools/llvm-dlltool/arm64ec.test +++ b/llvm/test/tools/llvm-dlltool/arm64ec.test @@ -12,9 +12,12 @@ ARMAP-NEXT: test_NULL_THUNK_DATA in test.dll ARMAP-EMPTY: ARMAP-NEXT: Archive EC map ARMAP-NEXT: #func in test.dll +ARMAP-NEXT: __IMPORT_DESCRIPTOR_test in test.dll +ARMAP-NEXT: __NULL_IMPORT_DESCRIPTOR in test.dll ARMAP-NEXT: __imp_aux_func in test.dll ARMAP-NEXT: __imp_func in test.dll ARMAP-NEXT: func in test.dll +ARMAP-NEXT: test_NULL_THUNK_DATA in test.dll RUN: llvm-dlltool -m arm64ec -d test.def -N test2.def -l test2.lib RUN: llvm-nm --print-armap test2.lib | FileCheck --check-prefix=ARMAP2 %s @@ -28,9 +31,12 @@ ARMAP2-NEXT: test_NULL_THUNK_DATA in test.dll ARMAP2-EMPTY: ARMAP2-NEXT: Archive EC map ARMAP2-NEXT: #func in test.dll +ARMAP2-NEXT: __IMPORT_DESCRIPTOR_test in test.dll +ARMAP2-NEXT: __NULL_IMPORT_DESCRIPTOR in test.dll ARMAP2-NEXT: __imp_aux_func in test.dll ARMAP2-NEXT: __imp_func in test.dll ARMAP2-NEXT: func in test.dll +ARMAP2-NEXT: test_NULL_THUNK_DATA in test.dll RUN: llvm-dlltool -m arm64ec -d test.def --input-native-def test2.def -l test3.lib RUN: llvm-nm --print-armap test3.lib | FileCheck --check-prefix=ARMAP2 %s diff --git a/llvm/test/tools/llvm-lib/arm64ec-implib.test b/llvm/test/tools/llvm-lib/arm64ec-implib.test index 77bdc23589fd..00eddd2a4752 100644 --- a/llvm/test/tools/llvm-lib/arm64ec-implib.test +++ b/llvm/test/tools/llvm-lib/arm64ec-implib.test @@ -16,6 +16,8 @@ ARMAP-NEXT: #funcexp in test.dll ARMAP-NEXT: #mangledfunc in test.dll ARMAP-NEXT: ?test_cpp_func@@$$hYAHPEAX@Z in test.dll ARMAP-NEXT: ?test_cpp_func@@YAHPEAX@Z in test.dll +ARMAP-NEXT: __IMPORT_DESCRIPTOR_test in test.dll +ARMAP-NEXT: __NULL_IMPORT_DESCRIPTOR in test.dll ARMAP-NEXT: __imp_?test_cpp_func@@YAHPEAX@Z in test.dll ARMAP-NEXT: __imp_aux_?test_cpp_func@@YAHPEAX@Z in test.dll ARMAP-NEXT: __imp_aux_expname in test.dll @@ -28,6 +30,7 @@ ARMAP-NEXT: __imp_mangledfunc in test.dll ARMAP-NEXT: expname in test.dll ARMAP-NEXT: funcexp in test.dll ARMAP-NEXT: mangledfunc in test.dll +ARMAP-NEXT: test_NULL_THUNK_DATA in test.dll RUN: llvm-readobj test.lib | FileCheck -check-prefix=READOBJ %s @@ -122,6 +125,8 @@ ARMAPX-NEXT: #funcexp in test.dll ARMAPX-NEXT: #mangledfunc in test.dll ARMAPX-NEXT: ?test_cpp_func@@$$hYAHPEAX@Z in test.dll ARMAPX-NEXT: ?test_cpp_func@@YAHPEAX@Z in test.dll +ARMAPX-NEXT: __IMPORT_DESCRIPTOR_test in test.dll +ARMAPX-NEXT: __NULL_IMPORT_DESCRIPTOR in test.dll ARMAPX-NEXT: __imp_?test_cpp_func@@YAHPEAX@Z in test.dll ARMAPX-NEXT: __imp_aux_?test_cpp_func@@YAHPEAX@Z in test.dll ARMAPX-NEXT: __imp_aux_expname in test.dll @@ -134,6 +139,7 @@ ARMAPX-NEXT: __imp_mangledfunc in test.dll ARMAPX-NEXT: expname in test.dll ARMAPX-NEXT: funcexp in test.dll ARMAPX-NEXT: mangledfunc in test.dll +ARMAPX-NEXT: test_NULL_THUNK_DATA in test.dll RUN: llvm-readobj testx.lib | FileCheck -check-prefix=READOBJX %s @@ -255,6 +261,8 @@ ARMAPX2-NEXT: #funcexp in test2.dll ARMAPX2-NEXT: #mangledfunc in test2.dll ARMAPX2-NEXT: ?test_cpp_func@@$$hYAHPEAX@Z in test2.dll ARMAPX2-NEXT: ?test_cpp_func@@YAHPEAX@Z in test2.dll +ARMAPX2-NEXT: __IMPORT_DESCRIPTOR_test2 in test2.dll +ARMAPX2-NEXT: __NULL_IMPORT_DESCRIPTOR in test2.dll ARMAPX2-NEXT: __imp_?test_cpp_func@@YAHPEAX@Z in test2.dll ARMAPX2-NEXT: __imp_aux_?test_cpp_func@@YAHPEAX@Z in test2.dll ARMAPX2-NEXT: __imp_aux_expname in test2.dll @@ -267,6 +275,7 @@ ARMAPX2-NEXT: __imp_mangledfunc in test2.dll ARMAPX2-NEXT: expname in test2.dll ARMAPX2-NEXT: funcexp in test2.dll ARMAPX2-NEXT: mangledfunc in test2.dll +ARMAPX2-NEXT: test2_NULL_THUNK_DATA in test2.dll ARMAPX2: test2.dll: ARMAPX2: 00000000 T #funcexp @@ -309,6 +318,8 @@ EXPAS-ARMAP-NEXT: #func1 in test.dll EXPAS-ARMAP-NEXT: #func2 in test.dll EXPAS-ARMAP-NEXT: #func3 in test.dll EXPAS-ARMAP-NEXT: #func4 in test.dll +EXPAS-ARMAP-NEXT: __IMPORT_DESCRIPTOR_test in test.dll +EXPAS-ARMAP-NEXT: __NULL_IMPORT_DESCRIPTOR in test.dll EXPAS-ARMAP-NEXT: __imp_aux_func1 in test.dll EXPAS-ARMAP-NEXT: __imp_aux_func2 in test.dll EXPAS-ARMAP-NEXT: __imp_aux_func3 in test.dll @@ -323,6 +334,7 @@ EXPAS-ARMAP-NEXT: func1 in test.dll EXPAS-ARMAP-NEXT: func2 in test.dll EXPAS-ARMAP-NEXT: func3 in test.dll EXPAS-ARMAP-NEXT: func4 in test.dll +EXPAS-ARMAP-NEXT: test_NULL_THUNK_DATA in test.dll EXPAS-READOBJ: File: test.dll EXPAS-READOBJ-NEXT: Format: COFF-import-file-ARM64EC -- GitLab From d73c2d5df21735805a1f46a85790db64c0615e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Tue, 12 Mar 2024 23:12:46 +0100 Subject: [PATCH 0049/1407] Fix unittest after #84460: only applicable if the platform supports JIT --- .../Interpreter/InterpreterExtensionsTest.cpp | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp index f1c3d65ab0a9..77fd1b4e1981 100644 --- a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp +++ b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp @@ -17,7 +17,9 @@ #include "clang/Sema/Lookup.h" #include "clang/Sema/Sema.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/Support/Error.h" +#include "llvm/Support/TargetSelect.h" #include "llvm/Testing/Support/Error.h" #include "gmock/gmock.h" @@ -27,6 +29,22 @@ using namespace clang; namespace { +static bool HostSupportsJit() { + auto J = llvm::orc::LLJITBuilder().create(); + if (J) + return true; + LLVMConsumeError(llvm::wrap(J.takeError())); + return false; +} + +struct LLVMInitRAII { + LLVMInitRAII() { + llvm::InitializeNativeTarget(); + llvm::InitializeNativeTargetAsmPrinter(); + } + ~LLVMInitRAII() { llvm::llvm_shutdown(); } +} LLVMInit; + class TestCreateResetExecutor : public Interpreter { public: TestCreateResetExecutor(std::unique_ptr CI, @@ -39,6 +57,10 @@ public: }; TEST(InterpreterExtensionsTest, ExecutorCreateReset) { + // Make sure we can create the executer on the platform. + if (!HostSupportsJit()) + GTEST_SKIP(); + clang::IncrementalCompilerBuilder CB; llvm::Error ErrOut = llvm::Error::success(); TestCreateResetExecutor Interp(cantFail(CB.CreateCpp()), ErrOut); -- GitLab From 76f3a084e77991cffbb8108959457ffd75f8e9c8 Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Tue, 12 Mar 2024 15:18:47 -0700 Subject: [PATCH 0050/1407] Update GettingStarted.rst doc with negative refspec to filter user branches (#75015) This allows to keep fetching release branches as well. --- llvm/docs/GettingStarted.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/llvm/docs/GettingStarted.rst b/llvm/docs/GettingStarted.rst index 705f6427d9ed..7ecef78c405b 100644 --- a/llvm/docs/GettingStarted.rst +++ b/llvm/docs/GettingStarted.rst @@ -42,10 +42,14 @@ Getting the Source Code and Building LLVM ``git clone --depth 1 https://github.com/llvm/llvm-project.git`` - * You are likely only interested in the main branch moving forward, if - you don't want `git fetch` (or `git pull`) to download user branches, use: + * You are likely not interested in the user branches in the repo (used for + stacked pull-requests and reverts), you can filter them from your + `git fetch` (or `git pull`) with this configuration: - ``sed 's#fetch = +refs/heads/\*:refs/remotes/origin/\*#fetch = +refs/heads/main:refs/remotes/origin/main#' -i llvm-project/.git/config`` +.. code-block:: console + + git config --add remote.origin.fetch '^refs/heads/users/*' + git config --add remote.origin.fetch '^refs/heads/revert-*' #. Configure and build LLVM and Clang: -- GitLab From beb47e78be6a819b6501f99302c1c4c1ae84b90e Mon Sep 17 00:00:00 2001 From: Michael Buch Date: Tue, 12 Mar 2024 22:19:09 +0000 Subject: [PATCH 0051/1407] [clang][CodeCompletion] Allow debuggers to code-complete reserved identifiers (#84891) --- clang/lib/Sema/SemaCodeComplete.cpp | 4 ++++ clang/test/CodeCompletion/ordinary-name.c | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp index 8d7523900940..73e6baa52782 100644 --- a/clang/lib/Sema/SemaCodeComplete.cpp +++ b/clang/lib/Sema/SemaCodeComplete.cpp @@ -764,6 +764,10 @@ getRequiredQualification(ASTContext &Context, const DeclContext *CurContext, // Filter out names reserved for the implementation if they come from a // system header. static bool shouldIgnoreDueToReservedName(const NamedDecl *ND, Sema &SemaRef) { + // Debuggers want access to all identifiers, including reserved ones. + if (SemaRef.getLangOpts().DebuggerSupport) + return false; + ReservedIdentifierStatus Status = ND->isReserved(SemaRef.getLangOpts()); // Ignore reserved names for compiler provided decls. if (isReservedInAllContexts(Status) && ND->getLocation().isInvalid()) diff --git a/clang/test/CodeCompletion/ordinary-name.c b/clang/test/CodeCompletion/ordinary-name.c index c8181a248daa..939856192061 100644 --- a/clang/test/CodeCompletion/ordinary-name.c +++ b/clang/test/CodeCompletion/ordinary-name.c @@ -5,6 +5,7 @@ typedef struct t _TYPEDEF; void foo() { int y; // RUN: %clang_cc1 -isystem %S/Inputs -fsyntax-only -code-completion-at=%s:%(line-1):9 %s -o - | FileCheck -check-prefix=CHECK-CC1 %s + // CHECK-CC1-NOT: __builtin_va_list // CHECK-CC1-NOT: __INTEGER_TYPE // CHECK-CC1: _Imaginary // CHECK-CC1: _MyPrivateType @@ -15,4 +16,8 @@ void foo() { // CHECK-CC1: y // PR8744 - // RUN: %clang_cc1 -isystem %S/Inputs -fsyntax-only -code-completion-at=%s:%(line-17):11 %s + // RUN: %clang_cc1 -isystem %S/Inputs -fsyntax-only -code-completion-at=%s:%(line-18):11 %s + + // RUN: %clang_cc1 -isystem %S/Inputs -fsyntax-only -fdebugger-support -code-completion-at=%s:%(line-15):9 %s -o - | FileCheck -check-prefix=CHECK-DBG %s + // CHECK-DBG: __builtin_va_list + // CHECK-DBG: __INTEGER_TYPE -- GitLab From 88bf64097e453deca73c91ec7de1af7eebe296a9 Mon Sep 17 00:00:00 2001 From: Michael Buch Date: Tue, 12 Mar 2024 22:19:27 +0000 Subject: [PATCH 0052/1407] [lldb][test] TestExprCompletion.py: add tests for completion of reserved identifiers (#84890) --- lldb/test/API/commands/expression/completion/Makefile | 1 + .../commands/expression/completion/TestExprCompletion.py | 5 +++++ lldb/test/API/commands/expression/completion/main.cpp | 5 +++++ .../API/commands/expression/completion/sys/reserved.h | 8 ++++++++ 4 files changed, 19 insertions(+) create mode 100644 lldb/test/API/commands/expression/completion/sys/reserved.h diff --git a/lldb/test/API/commands/expression/completion/Makefile b/lldb/test/API/commands/expression/completion/Makefile index 020dce7c31d1..9882622b2189 100644 --- a/lldb/test/API/commands/expression/completion/Makefile +++ b/lldb/test/API/commands/expression/completion/Makefile @@ -1,3 +1,4 @@ CXX_SOURCES := main.cpp other.cpp +CXXFLAGS += -isystem $(SRCDIR)/sys include Makefile.rules diff --git a/lldb/test/API/commands/expression/completion/TestExprCompletion.py b/lldb/test/API/commands/expression/completion/TestExprCompletion.py index c6a1e3c0f422..022b9436ee8e 100644 --- a/lldb/test/API/commands/expression/completion/TestExprCompletion.py +++ b/lldb/test/API/commands/expression/completion/TestExprCompletion.py @@ -246,6 +246,11 @@ class CommandLineExprCompletionTestCase(TestBase): "expr some_expr.Self(). FooNoArgs", "expr some_expr.Self(). FooNoArgsBar()" ) + self.complete_from_to("expr myVec.__f", "expr myVec.__func()") + self.complete_from_to("expr myVec._F", "expr myVec._Func()") + self.complete_from_to("expr myVec.__m", "expr myVec.__mem") + self.complete_from_to("expr myVec._M", "expr myVec._Mem") + def test_expr_completion_with_descriptions(self): self.build() self.main_source = "main.cpp" diff --git a/lldb/test/API/commands/expression/completion/main.cpp b/lldb/test/API/commands/expression/completion/main.cpp index 908bebbebff5..5e03805a7a4d 100644 --- a/lldb/test/API/commands/expression/completion/main.cpp +++ b/lldb/test/API/commands/expression/completion/main.cpp @@ -1,3 +1,5 @@ +#include + namespace LongNamespaceName { class NestedClass { long m; }; } // Defined in other.cpp, we only have a forward declaration here. @@ -31,5 +33,8 @@ int main() some_expr.FooNumbersBar1(); Expr::StaticMemberMethodBar(); ForwardDecl *fwd_decl_ptr = &fwd_decl; + MyVec myVec; + myVec.__func(); + myVec._Func(); return 0; // Break here } diff --git a/lldb/test/API/commands/expression/completion/sys/reserved.h b/lldb/test/API/commands/expression/completion/sys/reserved.h new file mode 100644 index 000000000000..0ce10ebec62b --- /dev/null +++ b/lldb/test/API/commands/expression/completion/sys/reserved.h @@ -0,0 +1,8 @@ +class MyVec { + int __mem; + int _Mem; + +public: + void __func() {} + void _Func() {} +}; -- GitLab From 47625e47db1d8fef6936ef48103e9aeb1fa3d328 Mon Sep 17 00:00:00 2001 From: Dave Clausen Date: Tue, 12 Mar 2024 18:36:48 -0400 Subject: [PATCH 0053/1407] Fix race in the implementation of __tsan_acquire() (#84923) `__tsan::Acquire()`, which is called by `__tsan_acquire()`, has a performance optimization which attempts to avoid acquiring the atomic variable's mutex if the variable has no associated memory model state. However, if the atomic variable was recently written to by a `compare_exchange_weak/strong` on another thread, the memory model state may be created *after* the atomic variable is updated. This is a data race, and can cause the thread calling `Acquire()` to not realize that the atomic variable was previously written to by another thread. Specifically, if you have code that writes to an atomic variable using `compare_exchange_weak/strong`, and then in another thread you read the value using a relaxed load, followed by an `atomic_thread_fence(memory_order_acquire)`, followed by a call to `__tsan_acquire()`, TSAN may not realize that the store happened before the fence, and so it will complain about any other variables you access from both threads if the thread-safety of those accesses depended on the happens-before relationship between the store and the fence. This change eliminates the unsafe optimization in `Acquire()`. Now, `Acquire()` acquires the mutex before checking for the existence of the memory model state. --- compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp | 2 +- .../tsan/compare_exchange_acquire_fence.cpp | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 compiler-rt/test/tsan/compare_exchange_acquire_fence.cpp diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp index 2e978852ea7d..2a8aa1915c9a 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp @@ -446,9 +446,9 @@ void Acquire(ThreadState *thr, uptr pc, uptr addr) { if (!s) return; SlotLocker locker(thr); + ReadLock lock(&s->mtx); if (!s->clock) return; - ReadLock lock(&s->mtx); thr->clock.Acquire(s->clock); } diff --git a/compiler-rt/test/tsan/compare_exchange_acquire_fence.cpp b/compiler-rt/test/tsan/compare_exchange_acquire_fence.cpp new file mode 100644 index 000000000000..b9fd0c5ad21f --- /dev/null +++ b/compiler-rt/test/tsan/compare_exchange_acquire_fence.cpp @@ -0,0 +1,43 @@ +// RUN: %clangxx_tsan -O1 %s -o %t && %run %t 2>&1 +// This is a correct program and tsan should not report a race. +// +// Verify that there is a happens-before relationship between a +// memory_order_release store that happens as part of a successful +// compare_exchange_strong(), and an atomic_thread_fence(memory_order_acquire) +// that happens after a relaxed load. + +#include +#include +#include +#include +#include + +std::atomic a; +unsigned int b; +constexpr int loops = 100000; + +void Thread1() { + for (int i = 0; i < loops; ++i) { + while (a.load(std::memory_order_acquire)) { + } + b = i; + bool expected = false; + a.compare_exchange_strong(expected, true, std::memory_order_acq_rel); + } +} + +int main() { + std::thread t(Thread1); + unsigned int sum = 0; + for (int i = 0; i < loops; ++i) { + while (!a.load(std::memory_order_relaxed)) { + } + std::atomic_thread_fence(std::memory_order_acquire); + __tsan_acquire(&a); + sum += b; + a.store(false, std::memory_order_release); + } + t.join(); + fprintf(stderr, "DONE: %u\n", sum); + return 0; +} -- GitLab From fd32e744a58fe61b4bd6acfa1d501bc1d6c1d96f Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Tue, 12 Mar 2024 15:52:27 -0700 Subject: [PATCH 0054/1407] [BOLT] Add support for Linux kernel PCI fixup section (#84982) .pci_fixup section contains a table with entries allowing to invoke a fixup hook whenever a problem is encountered with a PCI device. The hookup code typically points to the start of a function. As we are not relocating functions in the kernel (at least not yet), verify this assumption while reading the table and ignore any functions with a fixup code in the middle. --- bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 126 +++++++++++++++++------ bolt/test/X86/linux-pci-fixup.s | 41 ++++++++ 2 files changed, 134 insertions(+), 33 deletions(-) create mode 100644 bolt/test/X86/linux-pci-fixup.s diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp index 331a61e7c3c2..a2bfd45a64e3 100644 --- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -55,6 +55,11 @@ static cl::opt DumpParavirtualPatchSites( "dump-para-sites", cl::desc("dump Linux kernel paravitual patch sites"), cl::init(false), cl::Hidden, cl::cat(BoltCategory)); +static cl::opt + DumpPCIFixups("dump-pci-fixups", + cl::desc("dump Linux kernel PCI fixup table"), + cl::init(false), cl::Hidden, cl::cat(BoltCategory)); + static cl::opt DumpStaticCalls("dump-static-calls", cl::desc("dump Linux kernel static calls"), cl::init(false), cl::Hidden, @@ -181,6 +186,10 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Size of bug_entry struct. static constexpr size_t BUG_TABLE_ENTRY_SIZE = 12; + /// .pci_fixup section. + ErrorOr PCIFixupSection = std::errc::bad_address; + static constexpr size_t PCI_FIXUP_ENTRY_SIZE = 16; + /// Insert an LKMarker for a given code pointer \p PC from a non-code section /// \p SectionName. void insertLKMarker(uint64_t PC, uint64_t SectionOffset, @@ -190,9 +199,6 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Process linux kernel special sections and their relocations. void processLKSections(); - /// Process special linux kernel section, .pci_fixup. - void processLKPCIFixup(); - /// Process __ksymtab and __ksymtab_gpl. void processLKKSymtab(bool IsGPL = false); @@ -226,6 +232,9 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Read alternative instruction info from .altinstructions. Error readAltInstructions(); + /// Read .pci_fixup + Error readPCIFixupTable(); + /// Mark instructions referenced by kernel metadata. Error markInstructions(); @@ -256,6 +265,9 @@ public: if (Error E = readAltInstructions()) return E; + if (Error E = readPCIFixupTable()) + return E; + return Error::success(); } @@ -318,41 +330,11 @@ void LinuxKernelRewriter::insertLKMarker(uint64_t PC, uint64_t SectionOffset, } void LinuxKernelRewriter::processLKSections() { - processLKPCIFixup(); processLKKSymtab(); processLKKSymtab(true); processLKSMPLocks(); } -/// Process .pci_fixup section of Linux Kernel. -/// This section contains a list of entries for different PCI devices and their -/// corresponding hook handler (code pointer where the fixup -/// code resides, usually on x86_64 it is an entry PC relative 32 bit offset). -/// Documentation is in include/linux/pci.h. -void LinuxKernelRewriter::processLKPCIFixup() { - ErrorOr SectionOrError = - BC.getUniqueSectionByName(".pci_fixup"); - if (!SectionOrError) - return; - - const uint64_t SectionSize = SectionOrError->getSize(); - const uint64_t SectionAddress = SectionOrError->getAddress(); - assert((SectionSize % 16) == 0 && ".pci_fixup size is not a multiple of 16"); - - for (uint64_t I = 12; I + 4 <= SectionSize; I += 16) { - const uint64_t PC = SectionAddress + I; - ErrorOr Offset = BC.getSignedValueAtAddress(PC, 4); - assert(Offset && "cannot read value from .pci_fixup"); - const int32_t SignedOffset = *Offset; - const uint64_t HookupAddress = PC + SignedOffset; - BinaryFunction *HookupFunction = - BC.getBinaryFunctionAtAddress(HookupAddress); - assert(HookupFunction && "expected function for entry in .pci_fixup"); - BC.addRelocation(PC, HookupFunction->getSymbol(), Relocation::getPC32(), 0, - *Offset); - } -} - /// Process __ksymtab[_gpl] sections of Linux Kernel. /// This section lists all the vmlinux symbols that kernel modules can access. /// @@ -1283,6 +1265,84 @@ Error LinuxKernelRewriter::readAltInstructions() { return Error::success(); } +/// When the Linux kernel needs to handle an error associated with a given PCI +/// device, it uses a table stored in .pci_fixup section to locate a fixup code +/// specific to the vendor and the problematic device. The section contains a +/// list of the following structures defined in include/linux/pci.h: +/// +/// struct pci_fixup { +/// u16 vendor; /* Or PCI_ANY_ID */ +/// u16 device; /* Or PCI_ANY_ID */ +/// u32 class; /* Or PCI_ANY_ID */ +/// unsigned int class_shift; /* should be 0, 8, 16 */ +/// int hook_offset; +/// }; +/// +/// Normally, the hook will point to a function start and we don't have to +/// update the pointer if we are not relocating functions. Hence, while reading +/// the table we validate this assumption. If a function has a fixup code in the +/// middle of its body, we issue a warning and ignore it. +Error LinuxKernelRewriter::readPCIFixupTable() { + PCIFixupSection = BC.getUniqueSectionByName(".pci_fixup"); + if (!PCIFixupSection) + return Error::success(); + + if (PCIFixupSection->getSize() % PCI_FIXUP_ENTRY_SIZE) + return createStringError(errc::executable_format_error, + "PCI fixup table size error"); + + const uint64_t Address = PCIFixupSection->getAddress(); + DataExtractor DE = DataExtractor(PCIFixupSection->getContents(), + BC.AsmInfo->isLittleEndian(), + BC.AsmInfo->getCodePointerSize()); + uint64_t EntryID = 0; + DataExtractor::Cursor Cursor(0); + while (Cursor && !DE.eof(Cursor)) { + const uint16_t Vendor = DE.getU16(Cursor); + const uint16_t Device = DE.getU16(Cursor); + const uint32_t Class = DE.getU32(Cursor); + const uint32_t ClassShift = DE.getU32(Cursor); + const uint64_t HookAddress = + Address + Cursor.tell() + (int32_t)DE.getU32(Cursor); + + if (!Cursor) + return createStringError(errc::executable_format_error, + "out of bounds while reading .pci_fixup: %s", + toString(Cursor.takeError()).c_str()); + + ++EntryID; + + if (opts::DumpPCIFixups) { + BC.outs() << "PCI fixup entry: " << EntryID << "\n\tVendor 0x" + << Twine::utohexstr(Vendor) << "\n\tDevice: 0x" + << Twine::utohexstr(Device) << "\n\tClass: 0x" + << Twine::utohexstr(Class) << "\n\tClassShift: 0x" + << Twine::utohexstr(ClassShift) << "\n\tHookAddress: 0x" + << Twine::utohexstr(HookAddress) << '\n'; + } + + BinaryFunction *BF = BC.getBinaryFunctionContainingAddress(HookAddress); + if (!BF && opts::Verbosity) { + BC.outs() << "BOLT-INFO: no function matches address 0x" + << Twine::utohexstr(HookAddress) + << " of hook from .pci_fixup\n"; + } + + if (!BF || !BC.shouldEmit(*BF)) + continue; + + if (const uint64_t Offset = HookAddress - BF->getAddress()) { + BC.errs() << "BOLT-WARNING: PCI fixup detected in the middle of function " + << *BF << " at offset 0x" << Twine::utohexstr(Offset) << '\n'; + BF->setSimple(false); + } + } + + BC.outs() << "BOLT-INFO: parsed " << EntryID << " PCI fixup entries\n"; + + return Error::success(); +} + } // namespace std::unique_ptr diff --git a/bolt/test/X86/linux-pci-fixup.s b/bolt/test/X86/linux-pci-fixup.s new file mode 100644 index 000000000000..a574ba84c4df --- /dev/null +++ b/bolt/test/X86/linux-pci-fixup.s @@ -0,0 +1,41 @@ +# REQUIRES: system-linux + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o +# RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ +# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie +# RUN: llvm-bolt %t.exe --print-normalized -o %t.out |& FileCheck %s + +## Check that BOLT correctly parses the Linux kernel .pci_fixup section and +## verify that PCI fixup hook in the middle of a function is detected. + +# CHECK: BOLT-INFO: Linux kernel binary detected +# CHECK: BOLT-WARNING: PCI fixup detected in the middle of function _start +# CHECK: BOLT-INFO: parsed 2 PCI fixup entries + + .text + .globl _start + .type _start, %function +_start: + nop +.L0: + ret + .size _start, .-_start + +## PCI fixup table. + .section .pci_fixup,"a",@progbits + + .short 0x8086 # vendor + .short 0xbeef # device + .long 0xffffffff # class + .long 0x0 # class shift + .long _start - . # fixup + + .short 0x8086 # vendor + .short 0xbad # device + .long 0xffffffff # class + .long 0x0 # class shift + .long .L0 - . # fixup + +## Fake Linux Kernel sections. + .section __ksymtab,"a",@progbits + .section __ksymtab_gpl,"a",@progbits -- GitLab From 422d240dc9b4b36f505c43e6fe650af4f4cf4f98 Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Tue, 12 Mar 2024 16:10:38 -0700 Subject: [PATCH 0055/1407] Relax tests to also work with newer versions of lldb. - result variables are optional - static members may print their values - public/protected shows up in ptype output --- .../debuginfo-tests/llgdb-tests/static-member-2.cpp | 10 +++++----- .../debuginfo-tests/llgdb-tests/static-member.cpp | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cross-project-tests/debuginfo-tests/llgdb-tests/static-member-2.cpp b/cross-project-tests/debuginfo-tests/llgdb-tests/static-member-2.cpp index 5b6647c0631c..c9b416dace92 100644 --- a/cross-project-tests/debuginfo-tests/llgdb-tests/static-member-2.cpp +++ b/cross-project-tests/debuginfo-tests/llgdb-tests/static-member-2.cpp @@ -4,19 +4,19 @@ // XFAIL: gdb-clang-incompatibility // DEBUGGER: delete breakpoints -// DEBUGGER: break static-member.cpp:33 +// DEBUGGER: break static-member-2.cpp:36 // DEBUGGER: r // DEBUGGER: ptype C // CHECK: {{struct|class}} C { -// CHECK: static const int a; +// CHECK: static const int a // CHECK-NEXT: static int b; // CHECK-NEXT: static int c; -// CHECK-NEXT: int d; +// CHECK: int d; // CHECK-NEXT: } // DEBUGGER: p C::a -// CHECK: ${{[0-9]}} = 4 +// CHECK: 4 // DEBUGGER: p C::c -// CHECK: ${{[0-9]}} = 15 +// CHECK: 15 // PR14471, PR14734 diff --git a/cross-project-tests/debuginfo-tests/llgdb-tests/static-member.cpp b/cross-project-tests/debuginfo-tests/llgdb-tests/static-member.cpp index 29dd84dc8325..492e0ca08420 100644 --- a/cross-project-tests/debuginfo-tests/llgdb-tests/static-member.cpp +++ b/cross-project-tests/debuginfo-tests/llgdb-tests/static-member.cpp @@ -3,14 +3,14 @@ // RUN: %test_debuginfo %s %t.out // XFAIL: !system-darwin && gdb-clang-incompatibility // DEBUGGER: delete breakpoints -// DEBUGGER: break static-member.cpp:33 +// DEBUGGER: break static-member.cpp:35 // DEBUGGER: r // DEBUGGER: ptype MyClass // CHECK: {{struct|class}} MyClass { -// CHECK: static const int a; +// CHECK: static const int a // CHECK-NEXT: static int b; // CHECK-NEXT: static int c; -// CHECK-NEXT: int d; +// CHECK: int d; // CHECK-NEXT: } // DEBUGGER: p MyClass::a // CHECK: ${{[0-9]}} = 4 -- GitLab From 418f0066ebfcde8cd72d33cc103fc1c3e36a1205 Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Tue, 12 Mar 2024 16:11:45 -0700 Subject: [PATCH 0056/1407] Modernize llgdb script and make it easier to debug. --- .../debuginfo-tests/llgdb-tests/test_debuginfo.pl | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cross-project-tests/debuginfo-tests/llgdb-tests/test_debuginfo.pl b/cross-project-tests/debuginfo-tests/llgdb-tests/test_debuginfo.pl index 6dbc3b9b8632..fa52a5037c21 100755 --- a/cross-project-tests/debuginfo-tests/llgdb-tests/test_debuginfo.pl +++ b/cross-project-tests/debuginfo-tests/llgdb-tests/test_debuginfo.pl @@ -56,7 +56,7 @@ my $my_debugger = $ENV{'DEBUGGER'}; if (!$my_debugger) { if ($use_lldb) { my $path = dirname(Cwd::abs_path($0)); - $my_debugger = "/usr/bin/env python3 $path/llgdb.py"; + $my_debugger = "/usr/bin/xcrun python3 $path/llgdb.py"; } else { $my_debugger = "gdb"; } @@ -66,12 +66,18 @@ if (!$my_debugger) { my $debugger_options = "-q -batch -n -x"; # run debugger and capture output. +print("Running debugger\n"); system("$my_debugger $debugger_options $debugger_script_file $executable_file > $output_file 2>&1"); - +if ($?) { + print("Debugger output was:\n"); + system("cat", "$output_file"); + exit 1; +} # validate output. +print("Running FileCheck\n"); system("FileCheck", "-input-file", "$output_file", "$testcase_file"); -if ($?>>8 == 1) { - print "Debugger output was:\n"; +if ($?) { + print("Debugger output was:\n"); system("cat", "$output_file"); exit 1; } -- GitLab From 65f07b804c2c05cf49bd043f2a6e9a0020198165 Mon Sep 17 00:00:00 2001 From: anbbna <117081688+anbbna@users.noreply.github.com> Date: Wed, 13 Mar 2024 07:27:18 +0800 Subject: [PATCH 0057/1407] [MIPS] Introduce NAL instruction support for Mipsr6 and prer6 (#84429) NAL is an assembly idiom on Pre-R6 instruction sets (which is implemented in binutils), or an actual instruction on Release 6 instruction set, and is used to read the PC, due to the nature of the MIPS architecture. Since we can't read the PC directly, on pre-R6 we use a always-not-taken Branch and Link operation to the address of the next instruction, which effectively writes the address to $31, thus PC is read with offset +8. MIPS Release 6 removed the conventional Branch and Link instructions, but kept NAL as an actual instruction for compatibility on the assembly level. The instruction has the same encoding of the pre-R6 ones, and with the same behavior: PC + 8 -> $31. --- llvm/lib/Target/Mips/Mips32r6InstrFormats.td | 11 ++++++++++ llvm/lib/Target/Mips/Mips32r6InstrInfo.td | 16 ++++++++++++++- llvm/lib/Target/Mips/MipsInstrInfo.td | 3 +++ llvm/lib/Target/Mips/MipsScheduleGeneric.td | 2 +- llvm/test/MC/Mips/mips32/nal.s | 14 +++++++++++++ llvm/test/MC/Mips/mips32r6/nal.s | 21 ++++++++++++++++++++ 6 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 llvm/test/MC/Mips/mips32/nal.s create mode 100644 llvm/test/MC/Mips/mips32r6/nal.s diff --git a/llvm/lib/Target/Mips/Mips32r6InstrFormats.td b/llvm/lib/Target/Mips/Mips32r6InstrFormats.td index ccb6d1df777a..8536c52028c3 100644 --- a/llvm/lib/Target/Mips/Mips32r6InstrFormats.td +++ b/llvm/lib/Target/Mips/Mips32r6InstrFormats.td @@ -86,6 +86,7 @@ def OPCODE5_BC1NEZ : OPCODE5<0b01101>; def OPCODE5_BC2EQZ : OPCODE5<0b01001>; def OPCODE5_BC2NEZ : OPCODE5<0b01101>; def OPCODE5_BGEZAL : OPCODE5<0b10001>; +def OPCODE5_NAL : OPCODE5<0b10000>; def OPCODE5_SIGRIE : OPCODE5<0b10111>; // The next four constants are unnamed in the spec. These names are taken from // the OPGROUP names they are used with. @@ -201,6 +202,16 @@ class BAL_FM : MipsR6Inst { let Inst{15-0} = offset; } +// NAL for Release 6 +class NAL_FM : MipsR6Inst { + bits<32> Inst; + + let Inst{31-26} = OPGROUP_REGIMM.Value; + let Inst{25-21} = 0b00000; + let Inst{20-16} = OPCODE5_NAL.Value; + let Inst{15-0} = 0x00; +} + class COP0_EVP_DVP_FM sc> : MipsR6Inst { bits<5> rt; diff --git a/llvm/lib/Target/Mips/Mips32r6InstrInfo.td b/llvm/lib/Target/Mips/Mips32r6InstrInfo.td index 854563ab32bd..9c29acbd0d8a 100644 --- a/llvm/lib/Target/Mips/Mips32r6InstrInfo.td +++ b/llvm/lib/Target/Mips/Mips32r6InstrInfo.td @@ -73,6 +73,7 @@ class AUI_ENC : AUI_FM; class AUIPC_ENC : PCREL16_FM; class BAL_ENC : BAL_FM; +class NAL_ENC : NAL_FM; class BALC_ENC : BRANCH_OFF26_FM<0b111010>; class BC_ENC : BRANCH_OFF26_FM<0b110010>; class BEQC_ENC : CMP_BRANCH_2R_OFF16_FM, @@ -381,6 +382,12 @@ class BC_DESC_BASE : BRANCH_DESC_BASE, bit isCTI = 1; } +class NAL_DESC_BASE : BRANCH_DESC_BASE, + MipsR6Arch { + string AsmString = instr_asm; + bit isCTI = 1; +} + class CMP_BC_DESC_BASE : BRANCH_DESC_BASE, MipsR6Arch { @@ -424,6 +431,12 @@ class BAL_DESC : BC_DESC_BASE<"bal", brtarget> { bit isCTI = 1; } +class NAL_DESC : NAL_DESC_BASE<"nal"> { + bit hasDelaySlot = 1; + list Defs = [RA]; + bit isCTI = 1; +} + class BALC_DESC : BC_DESC_BASE<"balc", brtarget26> { bit isCall = 1; list Defs = [RA]; @@ -868,6 +881,8 @@ def AUI : R6MMR6Rel, AUI_ENC, AUI_DESC, ISA_MIPS32R6; def AUIPC : R6MMR6Rel, AUIPC_ENC, AUIPC_DESC, ISA_MIPS32R6; def BAL : BAL_ENC, BAL_DESC, ISA_MIPS32R6; def BALC : R6MMR6Rel, BALC_ENC, BALC_DESC, ISA_MIPS32R6; +def NAL : NAL_ENC, NAL_DESC, ISA_MIPS32R6; + let AdditionalPredicates = [NotInMicroMips] in { def BC1EQZ : BC1EQZ_ENC, BC1EQZ_DESC, ISA_MIPS32R6, HARDFLOAT; def BC1NEZ : BC1NEZ_ENC, BC1NEZ_DESC, ISA_MIPS32R6, HARDFLOAT; @@ -948,7 +963,6 @@ let AdditionalPredicates = [NotInMicroMips] in { def MUL_R6 : R6MMR6Rel, MUL_R6_ENC, MUL_R6_DESC, ISA_MIPS32R6; def MULU : R6MMR6Rel, MULU_ENC, MULU_DESC, ISA_MIPS32R6; } -def NAL; // BAL with rd=0 let AdditionalPredicates = [NotInMicroMips] in { def PREF_R6 : R6MMR6Rel, PREF_ENC, PREF_DESC, ISA_MIPS32R6; def RINT_D : RINT_D_ENC, RINT_D_DESC, ISA_MIPS32R6, HARDFLOAT; diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.td b/llvm/lib/Target/Mips/MipsInstrInfo.td index 4b6f4b22e71b..23e04c442bf6 100644 --- a/llvm/lib/Target/Mips/MipsInstrInfo.td +++ b/llvm/lib/Target/Mips/MipsInstrInfo.td @@ -3049,6 +3049,9 @@ def : MipsInstAlias<"divu $rd, $imm", (UDivIMacro GPR32Opnd:$rd, GPR32Opnd:$rd, simm32:$imm), 0>, ISA_MIPS1_NOT_32R6_64R6; + +def : MipsInstAlias<"nal", (BLTZAL ZERO, 0), 1>, ISA_MIPS1_NOT_32R6_64R6; + def SRemMacro : MipsAsmPseudoInst<(outs GPR32Opnd:$rd), (ins GPR32Opnd:$rs, GPR32Opnd:$rt), "rem\t$rd, $rs, $rt">, diff --git a/llvm/lib/Target/Mips/MipsScheduleGeneric.td b/llvm/lib/Target/Mips/MipsScheduleGeneric.td index a3df88a93cfb..6771a897eea7 100644 --- a/llvm/lib/Target/Mips/MipsScheduleGeneric.td +++ b/llvm/lib/Target/Mips/MipsScheduleGeneric.td @@ -285,7 +285,7 @@ def GenericWriteJumpAndLink : SchedWriteRes<[GenericIssueCTISTD]> { // jalr, jr.hb, jr, jalr.hb, jarlc, jialc def : InstRW<[GenericWriteJump], (instrs B, BAL, BAL_BR, BEQ, BNE, BGTZ, BGEZ, BLEZ, BLTZ, BLTZAL, J, JALX, JR, JR_HB, ERET, - ERet, ERETNC, DERET)>; + ERet, ERETNC, DERET, NAL)>; def : InstRW<[GenericWriteJump], (instrs BEQL, BNEL, BGEZL, BGTZL, BLEZL, BLTZL)>; diff --git a/llvm/test/MC/Mips/mips32/nal.s b/llvm/test/MC/Mips/mips32/nal.s new file mode 100644 index 000000000000..72c723108586 --- /dev/null +++ b/llvm/test/MC/Mips/mips32/nal.s @@ -0,0 +1,14 @@ +# RUN: llvm-mc %s -triple=mipsel-linux-gnu -filetype=obj -o - | \ +# RUN: llvm-objdump --no-print-imm-hex -d - | FileCheck %s --check-prefix=MIPS32-EL +# RUN: llvm-mc %s -triple=mips-linux-gnu -filetype=obj -o - | \ +# RUN: llvm-objdump --no-print-imm-hex -d - | FileCheck %s --check-prefix=MIPS32-EB + +# Whether it is a macro or an actual instruction, it always has a delay slot. +# Ensure the delay slot is filled correctly. +# MIPS32-EL: 00 00 10 04 bltzal $zero, 0x4 +# MIPS32-EL-NEXT: 00 00 00 00 nop +# MIPS32-EB: 04 10 00 00 bltzal $zero, 0x4 +# MIPS32-EB-NEXT: 00 00 00 00 nop + +nal_test: + nal diff --git a/llvm/test/MC/Mips/mips32r6/nal.s b/llvm/test/MC/Mips/mips32r6/nal.s new file mode 100644 index 000000000000..94c1a774f47e --- /dev/null +++ b/llvm/test/MC/Mips/mips32r6/nal.s @@ -0,0 +1,21 @@ +# RUN: llvm-mc %s -triple=mipsisa32r6el-linux-gnu -filetype=obj -o - | \ +# RUN: llvm-objdump --no-print-imm-hex -d - | FileCheck %s --check-prefix=MIPS32R6-EL +# RUN: llvm-mc %s -triple=mipsisa32r6-linux-gnu -filetype=obj -o - | \ +# RUN: llvm-objdump --no-print-imm-hex -d - | FileCheck %s --check-prefix=MIPS32R6-EB + +# Whether it is a macro or an actual instruction, it always has a delay slot. +# Ensure the delay slot is filled correctly. +# Also ensure that NAL does not reside in a forbidden slot. +# MIPS32R6-EL: 00 00 80 f8 bnezc $4, 0x4 +# MIPS32R6-EL-NEXT: 00 00 00 00 nop +# MIPS32R6-EL: 00 00 10 04 nal +# MIPS32R6-EL-NEXT: 00 00 00 00 nop +# MIPS32R6-EB: f8 80 00 00 bnezc $4, 0x4 +# MIPS32R6-EB-NEXT: 00 00 00 00 nop +# MIPS32R6-EB: 04 10 00 00 nal +# MIPS32R6-EB-NEXT: 00 00 00 00 nop + +nal_test: + # We generate a fobidden solt just for testing. + bnezc $a0, 0 + nal -- GitLab From 8003f553a01a9a2a7eb09fe07e88f1ba9ee7d3a7 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Tue, 12 Mar 2024 16:29:34 -0700 Subject: [PATCH 0058/1407] Reland "[llvm-exegesis] Add thread IDs to subprocess memory names (#84451)" This reverts commit aefad27096bba513f06162fac2763089578f3de4. This relands commit 6bbe8a296ee91754d423c59c35727eaa624f7140. This patch was casuing build failures on non-Linux platforms due to the default implementations for the functions not being updated. This ended up causing out-of-line definition errors. Fixed for the relanding. --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 9 +++--- .../llvm-exegesis/lib/SubprocessMemory.cpp | 30 +++++++++++++------ .../llvm-exegesis/lib/SubprocessMemory.h | 5 +++- .../X86/SubprocessMemoryTest.cpp | 5 +++- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 5c9848f3c688..4e97d188d172 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -301,6 +301,7 @@ private: if (AddMemDefError) return AddMemDefError; + long ParentTID = SubprocessMemory::getCurrentTID(); pid_t ParentOrChildPID = fork(); if (ParentOrChildPID == -1) { @@ -314,7 +315,7 @@ private: // Unregister handlers, signal handling is now handled through ptrace in // the host process. sys::unregisterHandlers(); - prepareAndRunBenchmark(PipeFiles[0], Key); + prepareAndRunBenchmark(PipeFiles[0], Key, ParentTID); // The child process terminates in the above function, so we should never // get to this point. llvm_unreachable("Child process didn't exit when expected."); @@ -415,8 +416,8 @@ private: setrlimit(RLIMIT_CORE, &rlim); } - [[noreturn]] void prepareAndRunBenchmark(int Pipe, - const BenchmarkKey &Key) const { + [[noreturn]] void prepareAndRunBenchmark(int Pipe, const BenchmarkKey &Key, + long ParentTID) const { // Disable core dumps in the child process as otherwise everytime we // encounter an execution failure like a segmentation fault, we will create // a core dump. We report the information directly rather than require the @@ -473,7 +474,7 @@ private: Expected AuxMemFDOrError = SubprocessMemory::setupAuxiliaryMemoryInSubprocess( - Key.MemoryValues, ParentPID, CounterFileDescriptor); + Key.MemoryValues, ParentPID, ParentTID, CounterFileDescriptor); if (!AuxMemFDOrError) exit(ChildProcessExitCodeE::AuxiliaryMemorySetupFailed); diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp index a49fa077257d..1fd81bd407be 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp @@ -9,11 +9,13 @@ #include "SubprocessMemory.h" #include "Error.h" #include "llvm/Support/Error.h" +#include "llvm/Support/FormatVariadic.h" #include #ifdef __linux__ #include #include +#include #include #endif @@ -22,12 +24,21 @@ namespace exegesis { #if defined(__linux__) && !defined(__ANDROID__) +long SubprocessMemory::getCurrentTID() { + // We're using the raw syscall here rather than the gettid() function provided + // by most libcs for compatibility as gettid() was only added to glibc in + // version 2.30. + return syscall(SYS_gettid); +} + Error SubprocessMemory::initializeSubprocessMemory(pid_t ProcessID) { // Add the PID to the shared memory name so that if we're running multiple // processes at the same time, they won't interfere with each other. // This comes up particularly often when running the exegesis tests with - // llvm-lit - std::string AuxiliaryMemoryName = "/auxmem" + std::to_string(ProcessID); + // llvm-lit. Additionally add the TID so that downstream consumers + // using multiple threads don't run into conflicts. + std::string AuxiliaryMemoryName = + formatv("/{0}auxmem{1}", getCurrentTID(), ProcessID); int AuxiliaryMemoryFD = shm_open(AuxiliaryMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryFD == -1) @@ -47,8 +58,8 @@ Error SubprocessMemory::addMemoryDefinition( pid_t ProcessPID) { SharedMemoryNames.reserve(MemoryDefinitions.size()); for (auto &[Name, MemVal] : MemoryDefinitions) { - std::string SharedMemoryName = "/" + std::to_string(ProcessPID) + "memdef" + - std::to_string(MemVal.Index); + std::string SharedMemoryName = + formatv("/{0}t{1}memdef{2}", ProcessPID, getCurrentTID(), MemVal.Index); SharedMemoryNames.push_back(SharedMemoryName); int SharedMemoryFD = shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); @@ -82,8 +93,9 @@ Error SubprocessMemory::addMemoryDefinition( Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, int CounterFileDescriptor) { - std::string AuxiliaryMemoryName = "/auxmem" + std::to_string(ParentPID); + pid_t ParentPID, long ParentTID, int CounterFileDescriptor) { + std::string AuxiliaryMemoryName = + formatv("/{0}auxmem{1}", ParentTID, ParentPID); int AuxiliaryMemoryFileDescriptor = shm_open(AuxiliaryMemoryName.c_str(), O_RDWR, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryFileDescriptor == -1) @@ -97,8 +109,8 @@ Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( return make_error("Mapping auxiliary memory failed"); AuxiliaryMemoryMapping[0] = CounterFileDescriptor; for (auto &[Name, MemVal] : MemoryDefinitions) { - std::string MemoryValueName = "/" + std::to_string(ParentPID) + "memdef" + - std::to_string(MemVal.Index); + std::string MemoryValueName = + formatv("/{0}t{1}memdef{2}", ParentPID, ParentTID, MemVal.Index); AuxiliaryMemoryMapping[AuxiliaryMemoryOffset + MemVal.Index] = shm_open(MemoryValueName.c_str(), O_RDWR, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryMapping[AuxiliaryMemoryOffset + MemVal.Index] == -1) @@ -133,7 +145,7 @@ Error SubprocessMemory::addMemoryDefinition( Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, int CounterFileDescriptor) { + pid_t ParentPID, long ParentTID, int CounterFileDescriptor) { return make_error( "setupAuxiliaryMemoryInSubprocess is only supported on Linux"); } diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h index e20b50cdc811..572d1085d9cf 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h @@ -35,6 +35,9 @@ public: static constexpr const size_t AuxiliaryMemoryOffset = 1; static constexpr const size_t AuxiliaryMemorySize = 4096; + // Gets the thread ID for the calling thread. + static long getCurrentTID(); + Error initializeSubprocessMemory(pid_t ProcessID); // The following function sets up memory definitions. It creates shared @@ -54,7 +57,7 @@ public: // section. static Expected setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, int CounterFileDescriptor); + pid_t ParentPID, long ParentTID, int CounterFileDescriptor); ~SubprocessMemory(); diff --git a/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp index c07ec188a602..7c23e7b7e9c5 100644 --- a/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp +++ b/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #endif // __linux__ @@ -49,7 +50,9 @@ protected: std::string getSharedMemoryName(const unsigned TestNumber, const unsigned DefinitionNumber) { - return "/" + std::to_string(getSharedMemoryNumber(TestNumber)) + "memdef" + + long CurrentTID = syscall(SYS_gettid); + return "/" + std::to_string(getSharedMemoryNumber(TestNumber)) + "t" + + std::to_string(CurrentTID) + "memdef" + std::to_string(DefinitionNumber); } -- GitLab From 1c3b15e9f5bc671e40bcf5d3475f5425466754ce Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Tue, 12 Mar 2024 16:55:01 -0700 Subject: [PATCH 0059/1407] [llvm-exegesis] Use LLVM Support to get thread ID This patch switches from manually using the Linux syscall to get the current thread ID to using the relevant LLVM Support libraries that abstract over the low level system details. --- llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp | 2 +- .../tools/llvm-exegesis/lib/SubprocessMemory.cpp | 16 +++++----------- llvm/tools/llvm-exegesis/lib/SubprocessMemory.h | 5 +---- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 4e97d188d172..17ce0355ef4f 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -301,7 +301,7 @@ private: if (AddMemDefError) return AddMemDefError; - long ParentTID = SubprocessMemory::getCurrentTID(); + long ParentTID = get_threadid(); pid_t ParentOrChildPID = fork(); if (ParentOrChildPID == -1) { diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp index 1fd81bd407be..28b341c46180 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp @@ -10,6 +10,7 @@ #include "Error.h" #include "llvm/Support/Error.h" #include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/Threading.h" #include #ifdef __linux__ @@ -24,13 +25,6 @@ namespace exegesis { #if defined(__linux__) && !defined(__ANDROID__) -long SubprocessMemory::getCurrentTID() { - // We're using the raw syscall here rather than the gettid() function provided - // by most libcs for compatibility as gettid() was only added to glibc in - // version 2.30. - return syscall(SYS_gettid); -} - Error SubprocessMemory::initializeSubprocessMemory(pid_t ProcessID) { // Add the PID to the shared memory name so that if we're running multiple // processes at the same time, they won't interfere with each other. @@ -38,7 +32,7 @@ Error SubprocessMemory::initializeSubprocessMemory(pid_t ProcessID) { // llvm-lit. Additionally add the TID so that downstream consumers // using multiple threads don't run into conflicts. std::string AuxiliaryMemoryName = - formatv("/{0}auxmem{1}", getCurrentTID(), ProcessID); + formatv("/{0}auxmem{1}", get_threadid(), ProcessID); int AuxiliaryMemoryFD = shm_open(AuxiliaryMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryFD == -1) @@ -59,7 +53,7 @@ Error SubprocessMemory::addMemoryDefinition( SharedMemoryNames.reserve(MemoryDefinitions.size()); for (auto &[Name, MemVal] : MemoryDefinitions) { std::string SharedMemoryName = - formatv("/{0}t{1}memdef{2}", ProcessPID, getCurrentTID(), MemVal.Index); + formatv("/{0}t{1}memdef{2}", ProcessPID, get_threadid(), MemVal.Index); SharedMemoryNames.push_back(SharedMemoryName); int SharedMemoryFD = shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); @@ -93,7 +87,7 @@ Error SubprocessMemory::addMemoryDefinition( Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, long ParentTID, int CounterFileDescriptor) { + pid_t ParentPID, uint64_t ParentTID, int CounterFileDescriptor) { std::string AuxiliaryMemoryName = formatv("/{0}auxmem{1}", ParentTID, ParentPID); int AuxiliaryMemoryFileDescriptor = @@ -145,7 +139,7 @@ Error SubprocessMemory::addMemoryDefinition( Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, long ParentTID, int CounterFileDescriptor) { + pid_t ParentPID, uint64_t ParentTID, int CounterFileDescriptor) { return make_error( "setupAuxiliaryMemoryInSubprocess is only supported on Linux"); } diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h index 572d1085d9cf..807046e38ce6 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h @@ -35,9 +35,6 @@ public: static constexpr const size_t AuxiliaryMemoryOffset = 1; static constexpr const size_t AuxiliaryMemorySize = 4096; - // Gets the thread ID for the calling thread. - static long getCurrentTID(); - Error initializeSubprocessMemory(pid_t ProcessID); // The following function sets up memory definitions. It creates shared @@ -57,7 +54,7 @@ public: // section. static Expected setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, long ParentTID, int CounterFileDescriptor); + pid_t ParentPID, uint64_t ParentTID, int CounterFileDescriptor); ~SubprocessMemory(); -- GitLab From 94e27c265a9aeb3659175ecee81a68d1763e0180 Mon Sep 17 00:00:00 2001 From: Peiming Liu Date: Tue, 12 Mar 2024 16:59:17 -0700 Subject: [PATCH 0060/1407] =?UTF-8?q?[mlir][sparse]=20reuse=20tensor.inser?= =?UTF-8?q?t=20operation=20to=20insert=20elements=20into=20=E2=80=A6=20(#8?= =?UTF-8?q?4987)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …a sparse tensor. --- .../SparseTensor/IR/SparseTensorOps.td | 42 ------------------- .../SparseTensor/IR/SparseTensorDialect.cpp | 9 ---- .../BufferizableOpInterfaceImpl.cpp | 22 ---------- .../Transforms/SparseReinterpretMap.cpp | 4 +- .../Transforms/SparseTensorCodegen.cpp | 19 +++++---- .../Transforms/SparseTensorConversion.cpp | 21 ++++++---- .../Transforms/SparseTensorRewriting.cpp | 6 ++- .../Transforms/Sparsification.cpp | 5 ++- mlir/test/Dialect/SparseTensor/codegen.mlir | 6 +-- .../SparseTensor/constant_index_map.mlir | 2 +- .../test/Dialect/SparseTensor/conversion.mlir | 2 +- mlir/test/Dialect/SparseTensor/invalid.mlir | 18 -------- mlir/test/Dialect/SparseTensor/roundtrip.mlir | 4 +- mlir/test/Dialect/SparseTensor/sparse_2d.mlir | 10 ++--- .../SparseTensor/sparse_broadcast.mlir | 2 +- .../sparse_conv_2d_slice_based.mlir | 2 +- .../Dialect/SparseTensor/sparse_fp_ops.mlir | 4 +- .../Dialect/SparseTensor/sparse_index.mlir | 2 +- .../test/Dialect/SparseTensor/sparse_out.mlir | 4 +- .../SparseTensor/sparse_reinterpret_map.mlir | 2 +- .../Dialect/SparseTensor/sparse_reshape.mlir | 8 ++-- .../SparseTensor/sparse_tensor_reshape.mlir | 2 +- .../SparseTensor/sparse_transpose.mlir | 2 +- .../SparseTensor/CPU/sparse_insert_1d.mlir | 10 ++--- .../SparseTensor/CPU/sparse_insert_2d.mlir | 40 +++++++++--------- .../SparseTensor/CPU/sparse_insert_3d.mlir | 40 +++++++++--------- 26 files changed, 106 insertions(+), 182 deletions(-) diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td index feed15d6af05..0498576fcffc 100644 --- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td +++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td @@ -668,48 +668,6 @@ def SparseTensor_CrdTranslateOp : SparseTensor_Op<"crd_translate", [Pure]>, // refined over time as our sparse abstractions evolve. //===----------------------------------------------------------------------===// -def SparseTensor_InsertOp : SparseTensor_Op<"insert", - [TypesMatchWith<"value type matches element type of tensor", - "tensor", "value", - "::llvm::cast($_self).getElementType()">, - AllTypesMatch<["tensor", "result"]>]>, - Arguments<(ins AnyType:$value, - AnySparseTensor:$tensor, - Variadic:$lvlCoords)>, - Results<(outs AnySparseTensor:$result)> { - string summary = "Inserts a value into the sparse tensor"; - string description = [{ - Inserts the value into the underlying storage of the tensor at the - given level-coordinates. The arity of `lvlCoords` must match the - level-rank of the tensor. This operation can only be applied when - the tensor materializes unintialized from a `tensor.empty` operation - and the final tensor is constructed with a `load` operation which - has the `hasInserts` attribute set. - - The level-properties of the sparse tensor type fully describe what - kind of insertion order is allowed. When all levels have "unique" - and "ordered" properties, for example, insertions should occur in - strict lexicographical level-coordinate order. Other properties - define different insertion regimens. Inserting in a way contrary - to these properties results in undefined behavior. - - Note that this operation is "impure" in the sense that even though - the result is modeled through an SSA value, the insertion is eventually - done "in place", and referencing the old SSA value is undefined behavior. - This operation is scheduled to be unified with the dense counterpart - `tensor.insert` that has pure SSA semantics. - - Example: - - ```mlir - %result = sparse_tensor.insert %val into %tensor[%i,%j] : tensor<1024x1024xf64, #CSR> - ``` - }]; - let assemblyFormat = "$value `into` $tensor `[` $lvlCoords `]` attr-dict" - "`:` type($tensor)"; - let hasVerifier = 1; -} - def SparseTensor_PushBackOp : SparseTensor_Op<"push_back", [TypesMatchWith<"value type matches element type of inBuffer", "inBuffer", "value", diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp index c19907a945d3..7750efdd9add 100644 --- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp +++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp @@ -1741,15 +1741,6 @@ LogicalResult ConcatenateOp::verify() { return success(); } -LogicalResult InsertOp::verify() { - const auto stt = getSparseTensorType(getTensor()); - if (stt.getEncoding().getBatchLvlRank() > 0) - return emitOpError("batched sparse tensor insertion not implemented"); - if (stt.getLvlRank() != static_cast(getLvlCoords().size())) - return emitOpError("incorrect number of coordinates"); - return success(); -} - void PushBackOp::build(OpBuilder &builder, OperationState &result, Value curSize, Value inBuffer, Value value) { build(builder, result, curSize, inBuffer, value, Value()); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.cpp index 3f4ae1f67de1..a942a721e218 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.cpp @@ -187,27 +187,6 @@ struct DisassembleOpInterface } }; -struct InsertOpInterface : public SparseBufferizableOpInterfaceExternalModel< - InsertOpInterface, sparse_tensor::InsertOp> { - bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, - const AnalysisState &state) const { - return true; - } - - bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, - const AnalysisState &state) const { - // InsertOp writes to memory. - return true; - } - - AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand, - const AnalysisState &state) const { - // InsertOp returns an alias of its operand. - assert(op->getNumResults() == 1); - return {{op->getOpResult(0), BufferRelation::Equivalent}}; - } -}; - struct NumberOfEntriesOpInterface : public SparseBufferizableOpInterfaceExternalModel< NumberOfEntriesOpInterface, sparse_tensor::NumberOfEntriesOp> { @@ -324,7 +303,6 @@ void mlir::sparse_tensor::registerBufferizableOpInterfaceExternalModels( sparse_tensor::ConvertOp::attachInterface(*ctx); sparse_tensor::LoadOp::attachInterface(*ctx); sparse_tensor::NewOp::attachInterface(*ctx); - sparse_tensor::InsertOp::attachInterface(*ctx); sparse_tensor::NumberOfEntriesOp::attachInterface< NumberOfEntriesOpInterface>(*ctx); sparse_tensor::AssembleOp::attachInterface(*ctx); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp index fbe2fc31ab8b..f93b59de29e5 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp @@ -640,14 +640,14 @@ struct TensorInsertDemapper using DemapInsRewriter::DemapInsRewriter; LogicalResult rewriteOp(tensor::InsertOp op, OpAdaptor adaptor, PatternRewriter &rewriter) const { - if (!hasAnySparseResult(op)) + if (!hasAnySparseResult(op) || !hasAnyNonIdentityOperandsOrResults(op)) return failure(); Location loc = op.getLoc(); auto stt = getSparseTensorType(op.getResult()); ValueRange lvlCrd = stt.translateCrds(rewriter, loc, op.getIndices(), CrdTransDirectionKind::dim2lvl); - auto insertOp = rewriter.create( + auto insertOp = rewriter.create( loc, op.getScalar(), adaptor.getDest(), lvlCrd); Value out = genRemap(rewriter, stt.getEncoding(), insertOp.getResult()); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp index 44c5d4dbe485..7ff2fc25328a 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp @@ -1014,24 +1014,29 @@ public: }; /// Sparse codegen rule for the insert operator. -class SparseInsertConverter : public OpConversionPattern { +class SparseInsertConverter : public OpConversionPattern { public: using OpConversionPattern::OpConversionPattern; LogicalResult - matchAndRewrite(InsertOp op, OpAdaptor adaptor, + matchAndRewrite(tensor::InsertOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { + auto stt = getSparseTensorType(adaptor.getDest()); + if (!stt.hasEncoding()) + return failure(); + assert(stt.isIdentity() && "Run reinterpret-map before conversion."); + Location loc = op.getLoc(); - auto desc = getDescriptorFromTensorTuple(adaptor.getTensor()); + auto desc = getDescriptorFromTensorTuple(adaptor.getDest()); TypeRange flatSpTensorTps = desc.getFields().getTypes(); SmallVector params = llvm::to_vector(desc.getFields()); - params.append(adaptor.getLvlCoords().begin(), adaptor.getLvlCoords().end()); - params.push_back(adaptor.getValue()); - SparseInsertGenerator insertGen(op.getTensor().getType(), flatSpTensorTps, + params.append(adaptor.getIndices().begin(), adaptor.getIndices().end()); + params.push_back(adaptor.getScalar()); + SparseInsertGenerator insertGen(op.getDest().getType(), flatSpTensorTps, params, /*genCall=*/true); SmallVector ret = insertGen.genCallOrInline(rewriter, loc); // Replace operation with resulting memrefs. rewriter.replaceOp(op, - genTuple(rewriter, loc, op.getTensor().getType(), ret)); + genTuple(rewriter, loc, op.getDest().getType(), ret)); return success(); } }; diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp index 010c3aa58b72..0937c10f2572 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp @@ -580,17 +580,24 @@ public: }; /// Sparse conversion rule for the insertion operator. -class SparseTensorInsertConverter : public OpConversionPattern { +class SparseTensorInsertConverter + : public OpConversionPattern { public: using OpConversionPattern::OpConversionPattern; LogicalResult - matchAndRewrite(InsertOp op, OpAdaptor adaptor, + matchAndRewrite(tensor::InsertOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { // Note that the current regime only allows for strict lexicographic // coordinate order. All values are passed by reference through stack // allocated memrefs. Location loc = op->getLoc(); - const auto stt = getSparseTensorType(op.getTensor()); + const auto stt = getSparseTensorType(op.getDest()); + + // Dense tensor insertion. + if (!stt.hasEncoding()) + return failure(); + + assert(stt.isIdentity() && "Run reinterpret-map before conversion."); const auto elemTp = stt.getElementType(); const Level lvlRank = stt.getLvlRank(); Value lvlCoords, vref; @@ -608,12 +615,12 @@ public: lvlCoords = genAlloca(rewriter, loc, lvlRank, rewriter.getIndexType()); vref = genAllocaScalar(rewriter, loc, elemTp); } - storeAll(rewriter, loc, lvlCoords, adaptor.getLvlCoords()); - rewriter.create(loc, adaptor.getValue(), vref); + storeAll(rewriter, loc, lvlCoords, adaptor.getIndices()); + rewriter.create(loc, adaptor.getScalar(), vref); SmallString<12> name{"lexInsert", primaryTypeFunctionSuffix(elemTp)}; createFuncCall(rewriter, loc, name, {}, - {adaptor.getTensor(), lvlCoords, vref}, EmitCInterface::On); - rewriter.replaceOp(op, adaptor.getTensor()); + {adaptor.getDest(), lvlCoords, vref}, EmitCInterface::On); + rewriter.replaceOp(op, adaptor.getDest()); return success(); } }; diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp index a65bce78d095..17f70d0796cc 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp @@ -817,7 +817,8 @@ public: reshapeCvs(builder, loc, expandReass, collapsedSizes, collapsedDcvs, dstSizes, dstDcvs); - auto t = builder.create(loc, v, reduc.front(), dstDcvs); + auto t = + builder.create(loc, v, reduc.front(), dstDcvs); builder.create(loc, t); }); @@ -901,7 +902,8 @@ public: SmallVector dstDcvs; reshapeCvs(builder, loc, op.getReassociationIndices(), srcSizes, srcDcvs, dstSizes, dstDcvs); - auto t = builder.create(loc, v, reduc.front(), dstDcvs); + auto t = + builder.create(loc, v, reduc.front(), dstDcvs); builder.create(loc, t); }); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp index 1fb70ed5035c..cd046b670d9a 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp @@ -428,7 +428,7 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t, /*else=*/true); // True branch. builder.setInsertionPointToStart(ifValidLexInsert.thenBlock()); - Value res = builder.create(loc, rhs, chain, ivs); + Value res = builder.create(loc, rhs, chain, ivs); builder.create(loc, res); // False branch. builder.setInsertionPointToStart(ifValidLexInsert.elseBlock()); @@ -438,7 +438,8 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t, env.updateInsertionChain(ifValidLexInsert.getResult(0)); } else { // Generates regular insertion chain. - env.updateInsertionChain(builder.create(loc, rhs, chain, ivs)); + env.updateInsertionChain( + builder.create(loc, rhs, chain, ivs)); } return; } diff --git a/mlir/test/Dialect/SparseTensor/codegen.mlir b/mlir/test/Dialect/SparseTensor/codegen.mlir index b63762485c96..40bfa1e4e2a5 100644 --- a/mlir/test/Dialect/SparseTensor/codegen.mlir +++ b/mlir/test/Dialect/SparseTensor/codegen.mlir @@ -643,7 +643,7 @@ func.func @sparse_compression_unordered(%tensor: tensor<8x8xf64, #UCSR>, // CHECK: %[[R:.*]]:4 = call @_insert_compressed_128_f64_0_0(%[[A1]], %[[A2]], %[[A3]], %[[A4]], %[[A5]], %[[A6]]) // CHECK: return %[[R]]#0, %[[R]]#1, %[[R]]#2, %[[R]]#3 func.func @sparse_insert(%arg0: tensor<128xf64, #SV>, %arg1: index, %arg2: f64) -> tensor<128xf64, #SV> { - %0 = sparse_tensor.insert %arg2 into %arg0[%arg1] : tensor<128xf64, #SV> + %0 = tensor.insert %arg2 into %arg0[%arg1] : tensor<128xf64, #SV> %1 = sparse_tensor.load %0 hasInserts : tensor<128xf64, #SV> return %1 : tensor<128xf64, #SV> } @@ -666,7 +666,7 @@ func.func @sparse_insert(%arg0: tensor<128xf64, #SV>, %arg1: index, %arg2: f64) // CHECK: %[[R:.*]]:4 = call @_insert_compressed_128_f64_64_32(%[[A1]], %[[A2]], %[[A3]], %[[A4]], %[[A5]], %[[A6]]) // CHECK: return %[[R]]#0, %[[R]]#1, %[[R]]#2, %[[R]]#3 func.func @sparse_insert_typed(%arg0: tensor<128xf64, #SparseVector>, %arg1: index, %arg2: f64) -> tensor<128xf64, #SparseVector> { - %0 = sparse_tensor.insert %arg2 into %arg0[%arg1] : tensor<128xf64, #SparseVector> + %0 = tensor.insert %arg2 into %arg0[%arg1] : tensor<128xf64, #SparseVector> %1 = sparse_tensor.load %0 hasInserts : tensor<128xf64, #SparseVector> return %1 : tensor<128xf64, #SparseVector> } @@ -690,7 +690,7 @@ func.func @sparse_insert_typed(%arg0: tensor<128xf64, #SparseVector>, %arg1: ind // CHECK: %[[R:.*]]:4 = call @_insert_compressed_nonunique_singleton_5_6_f64_0_0(%[[A0]], %[[A1]], %[[A2]], %[[A3]], %[[A4]], %[[A4]], %[[A5]]) // CHECK: return %[[R]]#0, %[[R]]#1, %[[R]]#2, %[[R]]#3 func.func @sparse_insert_coo(%arg0: tensor<5x6xf64, #Coo>, %arg1: index, %arg2: f64) -> tensor<5x6xf64, #Coo> { - %0 = sparse_tensor.insert %arg2 into %arg0[%arg1, %arg1] : tensor<5x6xf64, #Coo> + %0 = tensor.insert %arg2 into %arg0[%arg1, %arg1] : tensor<5x6xf64, #Coo> %1 = sparse_tensor.load %0 hasInserts : tensor<5x6xf64, #Coo> return %1 : tensor<5x6xf64, #Coo> } diff --git a/mlir/test/Dialect/SparseTensor/constant_index_map.mlir b/mlir/test/Dialect/SparseTensor/constant_index_map.mlir index eaef6a315852..f9559ce648c7 100644 --- a/mlir/test/Dialect/SparseTensor/constant_index_map.mlir +++ b/mlir/test/Dialect/SparseTensor/constant_index_map.mlir @@ -20,7 +20,7 @@ // CHECK: %[[VAL_11:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_3]], %[[VAL_9]]] : memref<1x77xi1> // CHECK: %[[VAL_12:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_3]], %[[VAL_9]]] : memref<1x77xi1> // CHECK: %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i1 -// CHECK: %[[VAL_14:.*]] = sparse_tensor.insert %[[VAL_13]] into %[[VAL_10]]{{\[}}%[[VAL_9]]] : tensor<77xi1, #{{.*}}> +// CHECK: %[[VAL_14:.*]] = tensor.insert %[[VAL_13]] into %[[VAL_10]]{{\[}}%[[VAL_9]]] : tensor<77xi1, #{{.*}}> // CHECK: scf.yield %[[VAL_14]] : tensor<77xi1, #{{.*}}> // CHECK: } // CHECK: %[[VAL_15:.*]] = sparse_tensor.load %[[VAL_16:.*]] hasInserts : tensor<77xi1, #{{.*}}> diff --git a/mlir/test/Dialect/SparseTensor/conversion.mlir b/mlir/test/Dialect/SparseTensor/conversion.mlir index 465f21086266..f23f6ac4f181 100644 --- a/mlir/test/Dialect/SparseTensor/conversion.mlir +++ b/mlir/test/Dialect/SparseTensor/conversion.mlir @@ -318,7 +318,7 @@ func.func @sparse_reconstruct_ins(%arg0: tensor<128xf32, #SparseVector>) -> tens func.func @sparse_insert(%arg0: tensor<128xf32, #SparseVector>, %arg1: index, %arg2: f32) -> tensor<128xf32, #SparseVector> { - %0 = sparse_tensor.insert %arg2 into %arg0[%arg1] : tensor<128xf32, #SparseVector> + %0 = tensor.insert %arg2 into %arg0[%arg1] : tensor<128xf32, #SparseVector> return %0 : tensor<128xf32, #SparseVector> } diff --git a/mlir/test/Dialect/SparseTensor/invalid.mlir b/mlir/test/Dialect/SparseTensor/invalid.mlir index eac97f702f58..48f28ef390ed 100644 --- a/mlir/test/Dialect/SparseTensor/invalid.mlir +++ b/mlir/test/Dialect/SparseTensor/invalid.mlir @@ -290,24 +290,6 @@ func.func @sparse_unannotated_load(%arg0: tensor<16x32xf64>) -> tensor<16x32xf64 // ----- -func.func @sparse_unannotated_insert(%arg0: tensor<128xf64>, %arg1: index, %arg2: f64) { - // expected-error@+1 {{'sparse_tensor.insert' 'tensor' must be sparse tensor of any type values, but got 'tensor<128xf64>'}} - sparse_tensor.insert %arg2 into %arg0[%arg1] : tensor<128xf64> - return -} - -// ----- - -#CSR = #sparse_tensor.encoding<{map = (d0, d1) -> (d0 : dense, d1 : compressed)}> - -func.func @sparse_wrong_arity_insert(%arg0: tensor<128x64xf64, #CSR>, %arg1: index, %arg2: f64) { - // expected-error@+1 {{'sparse_tensor.insert' op incorrect number of coordinates}} - sparse_tensor.insert %arg2 into %arg0[%arg1] : tensor<128x64xf64, #CSR> - return -} - -// ----- - func.func @sparse_push_back(%arg0: index, %arg1: memref, %arg2: f32) -> (memref, index) { // expected-error@+1 {{'sparse_tensor.push_back' op failed to verify that value type matches element type of inBuffer}} %0:2 = sparse_tensor.push_back %arg0, %arg1, %arg2 : index, memref, f32 diff --git a/mlir/test/Dialect/SparseTensor/roundtrip.mlir b/mlir/test/Dialect/SparseTensor/roundtrip.mlir index 41094fbad921..e9e458e805ba 100644 --- a/mlir/test/Dialect/SparseTensor/roundtrip.mlir +++ b/mlir/test/Dialect/SparseTensor/roundtrip.mlir @@ -311,10 +311,10 @@ func.func @sparse_load_ins(%arg0: tensor<16x32xf64, #DenseMatrix>) -> tensor<16x // CHECK-SAME: %[[A:.*]]: tensor<128xf64, #sparse{{[0-9]*}}>, // CHECK-SAME: %[[B:.*]]: index, // CHECK-SAME: %[[C:.*]]: f64) -// CHECK: %[[T:.*]] = sparse_tensor.insert %[[C]] into %[[A]][%[[B]]] : tensor<128xf64, #{{.*}}> +// CHECK: %[[T:.*]] = tensor.insert %[[C]] into %[[A]][%[[B]]] : tensor<128xf64, #{{.*}}> // CHECK: return %[[T]] : tensor<128xf64, #{{.*}}> func.func @sparse_insert(%arg0: tensor<128xf64, #SparseVector>, %arg1: index, %arg2: f64) -> tensor<128xf64, #SparseVector> { - %0 = sparse_tensor.insert %arg2 into %arg0[%arg1] : tensor<128xf64, #SparseVector> + %0 = tensor.insert %arg2 into %arg0[%arg1] : tensor<128xf64, #SparseVector> return %0 : tensor<128xf64, #SparseVector> } diff --git a/mlir/test/Dialect/SparseTensor/sparse_2d.mlir b/mlir/test/Dialect/SparseTensor/sparse_2d.mlir index 85ae0db91689..4afa0a8ceccd 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_2d.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_2d.mlir @@ -1090,20 +1090,20 @@ func.func @cmp_ss_ss(%arga: tensor<32x16xf32, #Tss>, %argb: tensor<32x16xf32, #T // CHECK: %[[VAL_41:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_30]]] : memref // CHECK: %[[VAL_42:.*]] = memref.load %[[VAL_11]]{{\[}}%[[VAL_31]]] : memref // CHECK: %[[VAL_43:.*]] = arith.subf %[[VAL_41]], %[[VAL_42]] : f64 -// CHECK: %[[VAL_44:.*]] = sparse_tensor.insert %[[VAL_43]] into %[[VAL_32]]{{\[}}%[[VAL_13]], %[[VAL_36]]] : tensor<2x3xf64, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_44:.*]] = tensor.insert %[[VAL_43]] into %[[VAL_32]]{{\[}}%[[VAL_13]], %[[VAL_36]]] : tensor<2x3xf64, #sparse{{[0-9]*}}> // CHECK: scf.yield %[[VAL_44]] : tensor<2x3xf64, #sparse{{[0-9]*}}> // CHECK: } else { // CHECK: %[[VAL_45:.*]] = arith.cmpi eq, %[[VAL_33]], %[[VAL_36]] : index // CHECK: %[[VAL_46:.*]] = scf.if %[[VAL_45]] -> (tensor<2x3xf64, #sparse{{[0-9]*}}>) { // CHECK: %[[VAL_47:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_30]]] : memref -// CHECK: %[[VAL_48:.*]] = sparse_tensor.insert %[[VAL_47]] into %[[VAL_32]]{{\[}}%[[VAL_13]], %[[VAL_36]]] : tensor<2x3xf64, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_48:.*]] = tensor.insert %[[VAL_47]] into %[[VAL_32]]{{\[}}%[[VAL_13]], %[[VAL_36]]] : tensor<2x3xf64, #sparse{{[0-9]*}}> // CHECK: scf.yield %[[VAL_48]] : tensor<2x3xf64, #sparse{{[0-9]*}}> // CHECK: } else { // CHECK: %[[VAL_49:.*]] = arith.cmpi eq, %[[VAL_34]], %[[VAL_36]] : index // CHECK: %[[VAL_50:.*]] = scf.if %[[VAL_49]] -> (tensor<2x3xf64, #sparse{{[0-9]*}}>) { // CHECK: %[[VAL_51:.*]] = memref.load %[[VAL_11]]{{\[}}%[[VAL_31]]] : memref // CHECK: %[[VAL_52:.*]] = arith.negf %[[VAL_51]] : f64 -// CHECK: %[[VAL_53:.*]] = sparse_tensor.insert %[[VAL_52]] into %[[VAL_32]]{{\[}}%[[VAL_13]], %[[VAL_36]]] : tensor<2x3xf64, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_53:.*]] = tensor.insert %[[VAL_52]] into %[[VAL_32]]{{\[}}%[[VAL_13]], %[[VAL_36]]] : tensor<2x3xf64, #sparse{{[0-9]*}}> // CHECK: scf.yield %[[VAL_53]] : tensor<2x3xf64, #sparse{{[0-9]*}}> // CHECK: } else { // CHECK: scf.yield %[[VAL_32]] : tensor<2x3xf64, #sparse{{[0-9]*}}> @@ -1123,14 +1123,14 @@ func.func @cmp_ss_ss(%arga: tensor<32x16xf32, #Tss>, %argb: tensor<32x16xf32, #T // CHECK: %[[VAL_63:.*]] = scf.for %[[VAL_64:.*]] = %[[VAL_65:.*]]#0 to %[[VAL_18]] step %[[VAL_4]] iter_args(%[[VAL_66:.*]] = %[[VAL_65]]#2) // CHECK: %[[VAL_67:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_64]]] : memref // CHECK: %[[VAL_68:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_64]]] : memref -// CHECK: %[[VAL_69:.*]] = sparse_tensor.insert %[[VAL_68]] into %[[VAL_66]]{{\[}}%[[VAL_13]], %[[VAL_67]]] : tensor<2x3xf64, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_69:.*]] = tensor.insert %[[VAL_68]] into %[[VAL_66]]{{\[}}%[[VAL_13]], %[[VAL_67]]] : tensor<2x3xf64, #sparse{{[0-9]*}}> // CHECK: scf.yield %[[VAL_69]] : tensor<2x3xf64, #sparse{{[0-9]*}}> // CHECK: } // CHECK: %[[VAL_70:.*]] = scf.for %[[VAL_71:.*]] = %[[VAL_72:.*]]#1 to %[[VAL_22]] step %[[VAL_4]] iter_args(%[[VAL_73:.*]] = %[[VAL_74:.*]]) // CHECK: %[[VAL_75:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_71]]] : memref // CHECK: %[[VAL_76:.*]] = memref.load %[[VAL_11]]{{\[}}%[[VAL_71]]] : memref // CHECK: %[[VAL_77:.*]] = arith.negf %[[VAL_76]] : f64 -// CHECK: %[[VAL_78:.*]] = sparse_tensor.insert %[[VAL_77]] into %[[VAL_73]]{{\[}}%[[VAL_13]], %[[VAL_75]]] : tensor<2x3xf64, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_78:.*]] = tensor.insert %[[VAL_77]] into %[[VAL_73]]{{\[}}%[[VAL_13]], %[[VAL_75]]] : tensor<2x3xf64, #sparse{{[0-9]*}}> // CHECK: scf.yield %[[VAL_78]] : tensor<2x3xf64, #sparse{{[0-9]*}}> // CHECK: } // CHECK: scf.yield %[[VAL_79:.*]] : tensor<2x3xf64, #sparse{{[0-9]*}}> diff --git a/mlir/test/Dialect/SparseTensor/sparse_broadcast.mlir b/mlir/test/Dialect/SparseTensor/sparse_broadcast.mlir index 278450fabd74..a409329700ff 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_broadcast.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_broadcast.mlir @@ -33,7 +33,7 @@ // CHECK: %[[L2:.*]] = scf.for %[[TMP_arg3:.*]] = %[[TMP_10]] to %[[TMP_12]] step %[[TMP_c1]] {{.*}} { // CHECK: %[[TMP_13:.*]] = memref.load %[[TMP_4]][%[[TMP_arg3]]] : memref // CHECK: %[[TMP_14:.*]] = memref.load %[[TMP_5]][%[[TMP_arg3]]] : memref -// CHECK: %[[Y:.*]] = sparse_tensor.insert %[[TMP_14]] into %{{.*}}[%[[TMP_9]], %[[TMP_arg2]], %[[TMP_13]]] +// CHECK: %[[Y:.*]] = tensor.insert %[[TMP_14]] into %{{.*}}[%[[TMP_9]], %[[TMP_arg2]], %[[TMP_13]]] // CHECK: scf.yield %[[Y]] // CHECK: } // CHECK: scf.yield %[[L2]] diff --git a/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir b/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir index 6076c1fbe76f..bf3473ead204 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir @@ -41,7 +41,7 @@ // CHECK: scf.yield // CHECK: } // CHECK: scf.if {{.*}} { -// CHECK: sparse_tensor.insert %{{.*}} into %{{.*}}{{\[}}%[[D0]], %[[D1]]] +// CHECK: tensor.insert %{{.*}} into %{{.*}}{{\[}}%[[D0]], %[[D1]]] // CHECK: scf.yield // CHECK: } else { // CHECK: scf.yield diff --git a/mlir/test/Dialect/SparseTensor/sparse_fp_ops.mlir b/mlir/test/Dialect/SparseTensor/sparse_fp_ops.mlir index 8c09523174bd..07b2c3c22995 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_fp_ops.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_fp_ops.mlir @@ -371,7 +371,7 @@ func.func @divbyc(%arga: tensor<32xf64, #SV>, // CHECK: %[[VAL_17:.*]] = math.log1p %[[VAL_16]] : f64 // CHECK: %[[VAL_18:.*]] = math.sin %[[VAL_17]] : f64 // CHECK: %[[VAL_19:.*]] = math.tanh %[[VAL_18]] : f64 -// CHECK: %[[Y:.*]] = sparse_tensor.insert %[[VAL_19]] into %{{.*}}[%[[VAL_10]]] : tensor<32xf64, #sparse{{[0-9]*}}> +// CHECK: %[[Y:.*]] = tensor.insert %[[VAL_19]] into %{{.*}}[%[[VAL_10]]] : tensor<32xf64, #sparse{{[0-9]*}}> // CHECK: scf.yield %[[Y]] // CHECK: } // CHECK: %[[VAL_20:.*]] = sparse_tensor.load %[[T]] hasInserts : tensor<32xf64, #sparse{{[0-9]*}}> @@ -412,7 +412,7 @@ func.func @zero_preserving_math(%arga: tensor<32xf64, #SV>) -> tensor<32xf64, #S // CHECK: %[[VAL_11:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_10]]] : memref // CHECK: %[[VAL_12:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_10]]] : memref> // CHECK: %[[VAL_13:.*]] = complex.div %[[VAL_12]], %[[VAL_3]] : complex -// CHECK: %[[Y:.*]] = sparse_tensor.insert %[[VAL_13]] into %{{.*}}[%[[VAL_11]]] : tensor<32xcomplex, #sparse{{[0-9]*}}> +// CHECK: %[[Y:.*]] = tensor.insert %[[VAL_13]] into %{{.*}}[%[[VAL_11]]] : tensor<32xcomplex, #sparse{{[0-9]*}}> // CHECK: scf.yield %[[Y]] // CHECK: } // CHECK: %[[VAL_14:.*]] = sparse_tensor.load %[[T]] hasInserts : tensor<32xcomplex, #sparse{{[0-9]*}}> diff --git a/mlir/test/Dialect/SparseTensor/sparse_index.mlir b/mlir/test/Dialect/SparseTensor/sparse_index.mlir index 3e8b485f63df..7fe662893f4b 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_index.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_index.mlir @@ -95,7 +95,7 @@ func.func @dense_index(%arga: tensor) // CHECK: %[[VAL_22:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_18]]] : memref // CHECK: %[[VAL_23:.*]] = arith.muli %[[VAL_21]], %[[VAL_22]] : i64 // CHECK: %[[VAL_24:.*]] = arith.muli %[[VAL_20]], %[[VAL_23]] : i64 -// CHECK: %[[Y:.*]] = sparse_tensor.insert %[[VAL_24]] into %{{.*}}[%[[VAL_14]], %[[VAL_19]]] : tensor +// CHECK: %[[Y:.*]] = tensor.insert %[[VAL_24]] into %{{.*}}[%[[VAL_14]], %[[VAL_19]]] : tensor // CHECK: scf.yield %[[Y]] // CHECK: } // CHECK: scf.yield %[[L]] diff --git a/mlir/test/Dialect/SparseTensor/sparse_out.mlir b/mlir/test/Dialect/SparseTensor/sparse_out.mlir index b1795ff2e1a2..08b81b54a9e6 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_out.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_out.mlir @@ -114,7 +114,7 @@ func.func @sparse_simply_dynamic2(%argx: tensor<32x16xf32, #DCSR>) -> tensor<32x // CHECK: %[[VAL_18:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_16]]] : memref // CHECK: %[[VAL_19:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_16]]] : memref // CHECK: %[[VAL_20:.*]] = arith.mulf %[[VAL_19]], %[[VAL_4]] : f32 -// CHECK: %[[VAL_21:.*]] = sparse_tensor.insert %[[VAL_20]] into %[[VAL_17]]{{\[}}%[[VAL_10]], %[[VAL_18]]] : tensor<10x20xf32, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_21:.*]] = tensor.insert %[[VAL_20]] into %[[VAL_17]]{{\[}}%[[VAL_10]], %[[VAL_18]]] : tensor<10x20xf32, #sparse{{[0-9]*}}> // CHECK: scf.yield %[[VAL_21]] : tensor<10x20xf32, #sparse{{[0-9]*}}> // CHECK: } // CHECK: scf.yield %[[VAL_22:.*]] : tensor<10x20xf32, #sparse{{[0-9]*}}> @@ -248,7 +248,7 @@ func.func @sparse_truly_dynamic(%arga: tensor<10x20xf32, #CSR>) -> tensor<10x20x // CHECK: scf.yield %[[VAL_100]], %[[VAL_103]], %[[VAL_104:.*]]#0, %[[VAL_104]]#1, %[[VAL_104]]#2 : index, index, i32, i1, tensor // CHECK: } // CHECK: %[[VAL_202:.*]] = scf.if %[[VAL_74]]#3 -> (tensor) { -// CHECK: %[[VAL_105:.*]] = sparse_tensor.insert %[[VAL_74]]#2 into %[[VAL_74]]#4{{\[}}%[[VAL_39]], %[[VAL_63]]] : tensor +// CHECK: %[[VAL_105:.*]] = tensor.insert %[[VAL_74]]#2 into %[[VAL_74]]#4{{\[}}%[[VAL_39]], %[[VAL_63]]] : tensor // CHECK: scf.yield %[[VAL_105]] : tensor // CHECK: } else { // CHECK: scf.yield %[[VAL_74]]#4 : tensor diff --git a/mlir/test/Dialect/SparseTensor/sparse_reinterpret_map.mlir b/mlir/test/Dialect/SparseTensor/sparse_reinterpret_map.mlir index aa17261724db..97c668716eec 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_reinterpret_map.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_reinterpret_map.mlir @@ -63,7 +63,7 @@ func.func @mul(%arg0: tensor<32x32xf32>, // CHECK: %[[VAL_2:.*]] = sparse_tensor.reinterpret_map %[[VAL_0]] : tensor<2x4xf64, #[[$remap]]> to tensor<1x2x2x2xf64, #[[$demap]]> // CHECK: %[[VAL_4:.*]] = sparse_tensor.foreach in %[[VAL_2]] init(%[[VAL_1]]) // CHECK: ^bb0(%[[VAL_5:.*]]: index, %[[VAL_6:.*]]: index, %[[VAL_7:.*]]: index, %[[VAL_8:.*]]: index, %[[VAL_9:.*]]: f64, %[[VAL_10:.*]]: tensor<1x2x2x2xf64, #[[$demap]]> -// CHECK: %[[VAL_11:.*]] = sparse_tensor.insert %[[VAL_9]] into %[[VAL_10]]{{\[}}%[[VAL_5]], %[[VAL_6]], %[[VAL_7]], %[[VAL_8]]] : tensor<1x2x2x2xf64, #[[$demap]]> +// CHECK: %[[VAL_11:.*]] = tensor.insert %[[VAL_9]] into %[[VAL_10]]{{\[}}%[[VAL_5]], %[[VAL_6]], %[[VAL_7]], %[[VAL_8]]] : tensor<1x2x2x2xf64, #[[$demap]]> // CHECK: sparse_tensor.yield %[[VAL_11]] : tensor<1x2x2x2xf64, #sparse{{[0-9]*}}> // CHECK: } // CHECK: %[[VAL_12:.*]] = sparse_tensor.reinterpret_map %[[VAL_4]] : tensor<1x2x2x2xf64, #[[$demap]]> to tensor<2x4xf64, #[[$remap]]> diff --git a/mlir/test/Dialect/SparseTensor/sparse_reshape.mlir b/mlir/test/Dialect/SparseTensor/sparse_reshape.mlir index eea77c6e5a6c..edb53fa024c2 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_reshape.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_reshape.mlir @@ -31,7 +31,7 @@ // CHECK: %[[SV:.*]] = memref.load %[[V]]{{\[}}%[[I]]] : memref // CHECK: %[[DI0:.*]] = arith.divui %[[SI]], %[[C10]] : index // CHECK: %[[DI1:.*]] = arith.remui %[[SI]], %[[C10]] : index -// CHECK: %[[NT:.*]] = sparse_tensor.insert %[[SV]] into %[[R]]{{\[}}%[[DI0]], %[[DI1]]] +// CHECK: %[[NT:.*]] = tensor.insert %[[SV]] into %[[R]]{{\[}}%[[DI0]], %[[DI1]]] // CHECK: scf.yield %[[NT:.*]] // CHECK: } // CHECK: %[[NT1:.*]] = sparse_tensor.load %[[RET]] hasInserts @@ -75,7 +75,7 @@ func.func @sparse_expand(%arg0: tensor<100xf64, #SparseVector>) -> tensor<10x10x // CHECK: %[[SV:.*]] = memref.load %[[V]]{{\[}}%[[J]]] : memref // CHECK: %[[T:.*]] = arith.muli %[[SI0]], %[[C10]] : index // CHECK: %[[DI:.*]] = arith.addi %[[T]], %[[SI1]] : index -// CHECK: %[[R1:.*]] = sparse_tensor.insert %[[SV]] into %[[A1]]{{\[}}%[[DI]]] +// CHECK: %[[R1:.*]] = tensor.insert %[[SV]] into %[[A1]]{{\[}}%[[DI]]] // CHECK scf.yield %[[R1]] // CHECK } // CHECK scf.yield %[[RET_1]] @@ -120,7 +120,7 @@ func.func @sparse_collapse(%arg0: tensor<10x10xf64, #SparseMatrix>) -> tensor<10 // CHECK: %[[T3:.*]] = arith.remui %[[SI]], %[[T2]] : index // CHECK: %[[T4:.*]] = arith.divui %[[T2]], %[[C10]] : index // CHECK: %[[DI1:.*]] = arith.divui %[[T3]], %[[T4]] : index -// CHECK: %[[NT:.*]] = sparse_tensor.insert %[[SV]] into %[[R]]{{\[}}%[[DI0]], %[[DI1]]] +// CHECK: %[[NT:.*]] = tensor.insert %[[SV]] into %[[R]]{{\[}}%[[DI0]], %[[DI1]]] // CHECK: scf.yield %[[NT]] // CHECK: } // CHECK: %[[NT1:.*]] = sparse_tensor.load %[[RET]] hasInserts @@ -169,7 +169,7 @@ func.func @dynamic_sparse_expand(%arg0: tensor) -> tensor< // CHECK: %[[T3:.*]] = arith.divui %[[T1]], %[[SD1]] : index // CHECK: %[[T4:.*]] = arith.muli %[[SI1]], %[[T3]] : index // CHECK: %[[DI:.*]] = arith.addi %[[T2]], %[[T4]] : index -// CHECK: %[[NT:.*]] = sparse_tensor.insert %[[SV]] into %[[R1]]{{\[}}%[[DI]]] +// CHECK: %[[NT:.*]] = tensor.insert %[[SV]] into %[[R1]]{{\[}}%[[DI]]] // CHECK scf.yield %[[NT]] // CHECK } // CHECK scf.yield %[[RET_1]] diff --git a/mlir/test/Dialect/SparseTensor/sparse_tensor_reshape.mlir b/mlir/test/Dialect/SparseTensor/sparse_tensor_reshape.mlir index 47d24ebf24fc..89826ebfe14d 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_tensor_reshape.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_tensor_reshape.mlir @@ -29,7 +29,7 @@ // CHECK: %[[DI:.*]] = arith.addi %[[T]], %[[SI1]] : index // CHECK: %[[D:.*]] = arith.divui %[[DI]], %[[C10]] : index // CHECK: %[[R:.*]] = arith.remui %[[DI]], %[[C10]] : index -// CHECK: %[[R1:.*]] = sparse_tensor.insert %[[SV]] into %[[A1]]{{\[}}%[[D]], %[[R]]] +// CHECK: %[[R1:.*]] = tensor.insert %[[SV]] into %[[A1]]{{\[}}%[[D]], %[[R]]] // CHECK: scf.yield %[[R1]] // CHECK: } // CHECK: scf.yield %[[RET_1]] diff --git a/mlir/test/Dialect/SparseTensor/sparse_transpose.mlir b/mlir/test/Dialect/SparseTensor/sparse_transpose.mlir index 80a989d789df..5038e977688d 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_transpose.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_transpose.mlir @@ -37,7 +37,7 @@ // CHECK: %[[VAL_19:.*]] = scf.for %[[VAL_20:.*]] = %[[VAL_16]] to %[[VAL_18]] step %[[VAL_2]] iter_args(%[[VAL_21:.*]] = %[[VAL_14]]) -> (tensor<4x3xf64, #sparse{{[0-9]*}}>) { // CHECK: %[[VAL_22:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_20]]] : memref // CHECK: %[[VAL_23:.*]] = memref.load %[[VAL_9]]{{\[}}%[[VAL_20]]] : memref -// CHECK: %[[VAL_24:.*]] = sparse_tensor.insert %[[VAL_23]] into %[[VAL_21]]{{\[}}%[[VAL_15]], %[[VAL_22]]] : tensor<4x3xf64, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_24:.*]] = tensor.insert %[[VAL_23]] into %[[VAL_21]]{{\[}}%[[VAL_15]], %[[VAL_22]]] : tensor<4x3xf64, #sparse{{[0-9]*}}> // CHECK: scf.yield %[[VAL_24]] : tensor<4x3xf64, #sparse{{[0-9]*}}> // CHECK: } // CHECK: scf.yield %[[VAL_25:.*]] : tensor<4x3xf64, #sparse{{[0-9]*}}> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir index 61c68507ea51..12e0d2267a26 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir @@ -54,10 +54,10 @@ module { // Build the sparse vector from straightline code. %0 = tensor.empty() : tensor<1024xf32, #SparseVector> - %1 = sparse_tensor.insert %f1 into %0[%c0] : tensor<1024xf32, #SparseVector> - %2 = sparse_tensor.insert %f2 into %1[%c1] : tensor<1024xf32, #SparseVector> - %3 = sparse_tensor.insert %f3 into %2[%c3] : tensor<1024xf32, #SparseVector> - %4 = sparse_tensor.insert %f4 into %3[%c1023] : tensor<1024xf32, #SparseVector> + %1 = tensor.insert %f1 into %0[%c0] : tensor<1024xf32, #SparseVector> + %2 = tensor.insert %f2 into %1[%c1] : tensor<1024xf32, #SparseVector> + %3 = tensor.insert %f3 into %2[%c3] : tensor<1024xf32, #SparseVector> + %4 = tensor.insert %f4 into %3[%c1023] : tensor<1024xf32, #SparseVector> %5 = sparse_tensor.load %4 hasInserts : tensor<1024xf32, #SparseVector> // @@ -76,7 +76,7 @@ module { %6 = tensor.empty() : tensor<1024xf32, #SparseVector> %7 = scf.for %i = %c0 to %c8 step %c1 iter_args(%vin = %6) -> tensor<1024xf32, #SparseVector> { %ii = arith.muli %i, %c3 : index - %vout = sparse_tensor.insert %f1 into %vin[%ii] : tensor<1024xf32, #SparseVector> + %vout = tensor.insert %f1 into %vin[%ii] : tensor<1024xf32, #SparseVector> scf.yield %vout : tensor<1024xf32, #SparseVector> } %8 = sparse_tensor.load %7 hasInserts : tensor<1024xf32, #SparseVector> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir index d51b67792337..883109150653 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir @@ -72,10 +72,10 @@ module { // CHECK-NEXT: ---- // %densea = tensor.empty() : tensor<4x3xf64, #Dense> - %dense1 = sparse_tensor.insert %f1 into %densea[%c0, %c0] : tensor<4x3xf64, #Dense> - %dense2 = sparse_tensor.insert %f2 into %dense1[%c2, %c2] : tensor<4x3xf64, #Dense> - %dense3 = sparse_tensor.insert %f3 into %dense2[%c3, %c0] : tensor<4x3xf64, #Dense> - %dense4 = sparse_tensor.insert %f4 into %dense3[%c3, %c2] : tensor<4x3xf64, #Dense> + %dense1 = tensor.insert %f1 into %densea[%c0, %c0] : tensor<4x3xf64, #Dense> + %dense2 = tensor.insert %f2 into %dense1[%c2, %c2] : tensor<4x3xf64, #Dense> + %dense3 = tensor.insert %f3 into %dense2[%c3, %c0] : tensor<4x3xf64, #Dense> + %dense4 = tensor.insert %f4 into %dense3[%c3, %c2] : tensor<4x3xf64, #Dense> %densem = sparse_tensor.load %dense4 hasInserts : tensor<4x3xf64, #Dense> sparse_tensor.print %densem : tensor<4x3xf64, #Dense> @@ -93,10 +93,10 @@ module { // CHECK-NEXT: ---- // %cooa = tensor.empty() : tensor<4x3xf64, #SortedCOO> - %coo1 = sparse_tensor.insert %f1 into %cooa[%c0, %c0] : tensor<4x3xf64, #SortedCOO> - %coo2 = sparse_tensor.insert %f2 into %coo1[%c2, %c2] : tensor<4x3xf64, #SortedCOO> - %coo3 = sparse_tensor.insert %f3 into %coo2[%c3, %c0] : tensor<4x3xf64, #SortedCOO> - %coo4 = sparse_tensor.insert %f4 into %coo3[%c3, %c2] : tensor<4x3xf64, #SortedCOO> + %coo1 = tensor.insert %f1 into %cooa[%c0, %c0] : tensor<4x3xf64, #SortedCOO> + %coo2 = tensor.insert %f2 into %coo1[%c2, %c2] : tensor<4x3xf64, #SortedCOO> + %coo3 = tensor.insert %f3 into %coo2[%c3, %c0] : tensor<4x3xf64, #SortedCOO> + %coo4 = tensor.insert %f4 into %coo3[%c3, %c2] : tensor<4x3xf64, #SortedCOO> %coom = sparse_tensor.load %coo4 hasInserts : tensor<4x3xf64, #SortedCOO> sparse_tensor.print %coom : tensor<4x3xf64, #SortedCOO> @@ -113,10 +113,10 @@ module { // CHECK-NEXT: ---- // %csra = tensor.empty() : tensor<4x3xf64, #CSR> - %csr1 = sparse_tensor.insert %f1 into %csra[%c0, %c0] : tensor<4x3xf64, #CSR> - %csr2 = sparse_tensor.insert %f2 into %csr1[%c2, %c2] : tensor<4x3xf64, #CSR> - %csr3 = sparse_tensor.insert %f3 into %csr2[%c3, %c0] : tensor<4x3xf64, #CSR> - %csr4 = sparse_tensor.insert %f4 into %csr3[%c3, %c2] : tensor<4x3xf64, #CSR> + %csr1 = tensor.insert %f1 into %csra[%c0, %c0] : tensor<4x3xf64, #CSR> + %csr2 = tensor.insert %f2 into %csr1[%c2, %c2] : tensor<4x3xf64, #CSR> + %csr3 = tensor.insert %f3 into %csr2[%c3, %c0] : tensor<4x3xf64, #CSR> + %csr4 = tensor.insert %f4 into %csr3[%c3, %c2] : tensor<4x3xf64, #CSR> %csrm = sparse_tensor.load %csr4 hasInserts : tensor<4x3xf64, #CSR> sparse_tensor.print %csrm : tensor<4x3xf64, #CSR> @@ -135,10 +135,10 @@ module { // CHECK-NEXT: ---- // %dcsra = tensor.empty() : tensor<4x3xf64, #DCSR> - %dcsr1 = sparse_tensor.insert %f1 into %dcsra[%c0, %c0] : tensor<4x3xf64, #DCSR> - %dcsr2 = sparse_tensor.insert %f2 into %dcsr1[%c2, %c2] : tensor<4x3xf64, #DCSR> - %dcsr3 = sparse_tensor.insert %f3 into %dcsr2[%c3, %c0] : tensor<4x3xf64, #DCSR> - %dcsr4 = sparse_tensor.insert %f4 into %dcsr3[%c3, %c2] : tensor<4x3xf64, #DCSR> + %dcsr1 = tensor.insert %f1 into %dcsra[%c0, %c0] : tensor<4x3xf64, #DCSR> + %dcsr2 = tensor.insert %f2 into %dcsr1[%c2, %c2] : tensor<4x3xf64, #DCSR> + %dcsr3 = tensor.insert %f3 into %dcsr2[%c3, %c0] : tensor<4x3xf64, #DCSR> + %dcsr4 = tensor.insert %f4 into %dcsr3[%c3, %c2] : tensor<4x3xf64, #DCSR> %dcsrm = sparse_tensor.load %dcsr4 hasInserts : tensor<4x3xf64, #DCSR> sparse_tensor.print %dcsrm : tensor<4x3xf64, #DCSR> @@ -155,10 +155,10 @@ module { // CHECK-NEXT: ---- // %rowa = tensor.empty() : tensor<4x3xf64, #Row> - %row1 = sparse_tensor.insert %f1 into %rowa[%c0, %c0] : tensor<4x3xf64, #Row> - %row2 = sparse_tensor.insert %f2 into %row1[%c2, %c2] : tensor<4x3xf64, #Row> - %row3 = sparse_tensor.insert %f3 into %row2[%c3, %c0] : tensor<4x3xf64, #Row> - %row4 = sparse_tensor.insert %f4 into %row3[%c3, %c2] : tensor<4x3xf64, #Row> + %row1 = tensor.insert %f1 into %rowa[%c0, %c0] : tensor<4x3xf64, #Row> + %row2 = tensor.insert %f2 into %row1[%c2, %c2] : tensor<4x3xf64, #Row> + %row3 = tensor.insert %f3 into %row2[%c3, %c0] : tensor<4x3xf64, #Row> + %row4 = tensor.insert %f4 into %row3[%c3, %c2] : tensor<4x3xf64, #Row> %rowm = sparse_tensor.load %row4 hasInserts : tensor<4x3xf64, #Row> sparse_tensor.print %rowm : tensor<4x3xf64, #Row> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir index 1917fd987c5d..364a188cf37c 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir @@ -71,11 +71,11 @@ module { // CHECK-NEXT: values : ( 1.1, 2.2, 3.3, 4.4, 5.5 // CHECK-NEXT: ---- %tensora = tensor.empty() : tensor<5x4x3xf64, #TensorCSR> - %tensor1 = sparse_tensor.insert %f1 into %tensora[%c3, %c0, %c1] : tensor<5x4x3xf64, #TensorCSR> - %tensor2 = sparse_tensor.insert %f2 into %tensor1[%c3, %c0, %c2] : tensor<5x4x3xf64, #TensorCSR> - %tensor3 = sparse_tensor.insert %f3 into %tensor2[%c3, %c3, %c1] : tensor<5x4x3xf64, #TensorCSR> - %tensor4 = sparse_tensor.insert %f4 into %tensor3[%c4, %c2, %c2] : tensor<5x4x3xf64, #TensorCSR> - %tensor5 = sparse_tensor.insert %f5 into %tensor4[%c4, %c3, %c2] : tensor<5x4x3xf64, #TensorCSR> + %tensor1 = tensor.insert %f1 into %tensora[%c3, %c0, %c1] : tensor<5x4x3xf64, #TensorCSR> + %tensor2 = tensor.insert %f2 into %tensor1[%c3, %c0, %c2] : tensor<5x4x3xf64, #TensorCSR> + %tensor3 = tensor.insert %f3 into %tensor2[%c3, %c3, %c1] : tensor<5x4x3xf64, #TensorCSR> + %tensor4 = tensor.insert %f4 into %tensor3[%c4, %c2, %c2] : tensor<5x4x3xf64, #TensorCSR> + %tensor5 = tensor.insert %f5 into %tensor4[%c4, %c3, %c2] : tensor<5x4x3xf64, #TensorCSR> %tensorm = sparse_tensor.load %tensor5 hasInserts : tensor<5x4x3xf64, #TensorCSR> sparse_tensor.print %tensorm : tensor<5x4x3xf64, #TensorCSR> @@ -90,11 +90,11 @@ module { // CHECK-NEXT: values : ( 0, 1.1, 2.2, 0, 3.3, 0, 0, 0, 4.4, 0, 0, 5.5 // CHECK-NEXT: ---- %rowa = tensor.empty() : tensor<5x4x3xf64, #TensorRow> - %row1 = sparse_tensor.insert %f1 into %rowa[%c3, %c0, %c1] : tensor<5x4x3xf64, #TensorRow> - %row2 = sparse_tensor.insert %f2 into %row1[%c3, %c0, %c2] : tensor<5x4x3xf64, #TensorRow> - %row3 = sparse_tensor.insert %f3 into %row2[%c3, %c3, %c1] : tensor<5x4x3xf64, #TensorRow> - %row4 = sparse_tensor.insert %f4 into %row3[%c4, %c2, %c2] : tensor<5x4x3xf64, #TensorRow> - %row5 = sparse_tensor.insert %f5 into %row4[%c4, %c3, %c2] : tensor<5x4x3xf64, #TensorRow> + %row1 = tensor.insert %f1 into %rowa[%c3, %c0, %c1] : tensor<5x4x3xf64, #TensorRow> + %row2 = tensor.insert %f2 into %row1[%c3, %c0, %c2] : tensor<5x4x3xf64, #TensorRow> + %row3 = tensor.insert %f3 into %row2[%c3, %c3, %c1] : tensor<5x4x3xf64, #TensorRow> + %row4 = tensor.insert %f4 into %row3[%c4, %c2, %c2] : tensor<5x4x3xf64, #TensorRow> + %row5 = tensor.insert %f5 into %row4[%c4, %c3, %c2] : tensor<5x4x3xf64, #TensorRow> %rowm = sparse_tensor.load %row5 hasInserts : tensor<5x4x3xf64, #TensorRow> sparse_tensor.print %rowm : tensor<5x4x3xf64, #TensorRow> @@ -109,11 +109,11 @@ module { // CHECK-NEXT: values : ( 1.1, 2.2, 3.3, 4.4, 5.5 // CHECK-NEXT: ---- %ccoo = tensor.empty() : tensor<5x4x3xf64, #CCoo> - %ccoo1 = sparse_tensor.insert %f1 into %ccoo[%c3, %c0, %c1] : tensor<5x4x3xf64, #CCoo> - %ccoo2 = sparse_tensor.insert %f2 into %ccoo1[%c3, %c0, %c2] : tensor<5x4x3xf64, #CCoo> - %ccoo3 = sparse_tensor.insert %f3 into %ccoo2[%c3, %c3, %c1] : tensor<5x4x3xf64, #CCoo> - %ccoo4 = sparse_tensor.insert %f4 into %ccoo3[%c4, %c2, %c2] : tensor<5x4x3xf64, #CCoo> - %ccoo5 = sparse_tensor.insert %f5 into %ccoo4[%c4, %c3, %c2] : tensor<5x4x3xf64, #CCoo> + %ccoo1 = tensor.insert %f1 into %ccoo[%c3, %c0, %c1] : tensor<5x4x3xf64, #CCoo> + %ccoo2 = tensor.insert %f2 into %ccoo1[%c3, %c0, %c2] : tensor<5x4x3xf64, #CCoo> + %ccoo3 = tensor.insert %f3 into %ccoo2[%c3, %c3, %c1] : tensor<5x4x3xf64, #CCoo> + %ccoo4 = tensor.insert %f4 into %ccoo3[%c4, %c2, %c2] : tensor<5x4x3xf64, #CCoo> + %ccoo5 = tensor.insert %f5 into %ccoo4[%c4, %c3, %c2] : tensor<5x4x3xf64, #CCoo> %ccoom = sparse_tensor.load %ccoo5 hasInserts : tensor<5x4x3xf64, #CCoo> sparse_tensor.print %ccoom : tensor<5x4x3xf64, #CCoo> @@ -126,11 +126,11 @@ module { // CHECK-NEXT: values : ( 1.1, 2.2, 3.3, 4.4, 5.5 // CHECK-NEXT: ---- %dcoo = tensor.empty() : tensor<5x4x3xf64, #DCoo> - %dcoo1 = sparse_tensor.insert %f1 into %dcoo[%c3, %c0, %c1] : tensor<5x4x3xf64, #DCoo> - %dcoo2 = sparse_tensor.insert %f2 into %dcoo1[%c3, %c0, %c2] : tensor<5x4x3xf64, #DCoo> - %dcoo3 = sparse_tensor.insert %f3 into %dcoo2[%c3, %c3, %c1] : tensor<5x4x3xf64, #DCoo> - %dcoo4 = sparse_tensor.insert %f4 into %dcoo3[%c4, %c2, %c2] : tensor<5x4x3xf64, #DCoo> - %dcoo5 = sparse_tensor.insert %f5 into %dcoo4[%c4, %c3, %c2] : tensor<5x4x3xf64, #DCoo> + %dcoo1 = tensor.insert %f1 into %dcoo[%c3, %c0, %c1] : tensor<5x4x3xf64, #DCoo> + %dcoo2 = tensor.insert %f2 into %dcoo1[%c3, %c0, %c2] : tensor<5x4x3xf64, #DCoo> + %dcoo3 = tensor.insert %f3 into %dcoo2[%c3, %c3, %c1] : tensor<5x4x3xf64, #DCoo> + %dcoo4 = tensor.insert %f4 into %dcoo3[%c4, %c2, %c2] : tensor<5x4x3xf64, #DCoo> + %dcoo5 = tensor.insert %f5 into %dcoo4[%c4, %c3, %c2] : tensor<5x4x3xf64, #DCoo> %dcoom = sparse_tensor.load %dcoo5 hasInserts : tensor<5x4x3xf64, #DCoo> sparse_tensor.print %dcoom : tensor<5x4x3xf64, #DCoo> -- GitLab From 64111831ed46b9e82b8626356c087ce2202f029b Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Tue, 12 Mar 2024 17:00:51 -0700 Subject: [PATCH 0061/1407] Relax test to work with newer versions of lldb --- .../debuginfo-tests/llgdb-tests/forward-declare-class.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cross-project-tests/debuginfo-tests/llgdb-tests/forward-declare-class.cpp b/cross-project-tests/debuginfo-tests/llgdb-tests/forward-declare-class.cpp index 132420009bd1..850eed6ad95d 100644 --- a/cross-project-tests/debuginfo-tests/llgdb-tests/forward-declare-class.cpp +++ b/cross-project-tests/debuginfo-tests/llgdb-tests/forward-declare-class.cpp @@ -6,9 +6,8 @@ // Work around a gdb bug where it believes that a class is a // struct if there aren't any methods - even though it's tagged // as a class. -// CHECK: type = {{struct|class}} A { -// CHECK-NEXT: {{(public:){0,1}}} -// CHECK-NEXT: int MyData; +// CHECK: {{struct|class}} A { +// CHECK: int MyData; // CHECK-NEXT: } class A; class B { -- GitLab From bb5921e2a2da4a87016e623deb8c2eaed7f1f5a8 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Tue, 12 Mar 2024 17:01:29 -0700 Subject: [PATCH 0062/1407] [libc] Include FP_* macros in math.h (#84996) These are used unconditionally by libc++ math.h. This is related to issue #84879. --- libc/include/llvm-libc-macros/math-macros.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libc/include/llvm-libc-macros/math-macros.h b/libc/include/llvm-libc-macros/math-macros.h index e67fe4d11b44..db8a4ea65bd6 100644 --- a/libc/include/llvm-libc-macros/math-macros.h +++ b/libc/include/llvm-libc-macros/math-macros.h @@ -11,6 +11,12 @@ #include "limits-macros.h" +#define FP_NAN 0 +#define FP_INFINITE 1 +#define FP_ZERO 2 +#define FP_SUBNORMAL 3 +#define FP_NORMAL 4 + #define MATH_ERRNO 1 #define MATH_ERREXCEPT 2 -- GitLab From accfbf4e4959957db0993a15bab7316001131df3 Mon Sep 17 00:00:00 2001 From: Zahi Moudallal <128723247+zahimoud@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:07:16 -0700 Subject: [PATCH 0063/1407] [MLIR][ROCDL] Add BallotOp and lit test (#84856) --- mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td | 10 ++++++++++ mlir/test/Target/LLVMIR/rocdl.mlir | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td index 32b5a1c016b6..abb38a3df806 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td @@ -158,6 +158,16 @@ Arguments<(ins I32:$index, }]; } +def ROCDL_BallotOp : + ROCDL_Op<"ballot">, + Results<(outs LLVM_Type:$res)>, + Arguments<(ins I1:$pred)> { + string llvmBuilder = [{ + $res = createIntrinsicCall(builder, + llvm::Intrinsic::amdgcn_ballot, {$pred}, {llvm::Type::getInt32Ty(moduleTranslation.getLLVMContext())}); + }]; + let assemblyFormat = "$pred attr-dict `:` type($res)"; +} //===----------------------------------------------------------------------===// // Thread index and Block index diff --git a/mlir/test/Target/LLVMIR/rocdl.mlir b/mlir/test/Target/LLVMIR/rocdl.mlir index d35acb0475e6..93550f5c7bd5 100644 --- a/mlir/test/Target/LLVMIR/rocdl.mlir +++ b/mlir/test/Target/LLVMIR/rocdl.mlir @@ -88,6 +88,13 @@ llvm.func @rocdl.bpermute(%src : i32) -> i32 { llvm.return %0 : i32 } +llvm.func @rocdl.ballot(%pred : i1) -> i32 { + // CHECK-LABEL: rocdl.ballot + // CHECK: call i32 @llvm.amdgcn.ballot + %0 = rocdl.ballot %pred : i32 + llvm.return %0 : i32 +} + llvm.func @rocdl.waitcnt() { // CHECK-LABEL: rocdl.waitcnt // CHECK-NEXT: call void @llvm.amdgcn.s.waitcnt(i32 0) -- GitLab From e2468bf16a0c1f63a39aa417c15c03ebd77fab9e Mon Sep 17 00:00:00 2001 From: Jason Molenda Date: Tue, 12 Mar 2024 17:19:46 -0700 Subject: [PATCH 0064/1407] [lldb][debugserver] Update flags past to app launch request rdar://117421999 --- lldb/tools/debugserver/source/MacOSX/MachProcess.mm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lldb/tools/debugserver/source/MacOSX/MachProcess.mm b/lldb/tools/debugserver/source/MacOSX/MachProcess.mm index 87bdbf835bfd..70b4564a027b 100644 --- a/lldb/tools/debugserver/source/MacOSX/MachProcess.mm +++ b/lldb/tools/debugserver/source/MacOSX/MachProcess.mm @@ -472,6 +472,8 @@ FBSCreateOptionsDictionary(const char *app_bundle_path, // And there are some other options at the top level in this dictionary: [options setObject:[NSNumber numberWithBool:YES] forKey:FBSOpenApplicationOptionKeyUnlockDevice]; + [options setObject:[NSNumber numberWithBool:YES] + forKey:FBSOpenApplicationOptionKeyPromptUnlockDevice]; // We have to get the "sequence ID & UUID" for this app bundle path and send // them to FBS: -- GitLab From 2f400a2fd77b44d34281792aca84c42e149095e7 Mon Sep 17 00:00:00 2001 From: Michael Maitland Date: Tue, 12 Mar 2024 20:22:49 -0400 Subject: [PATCH 0065/1407] [GISEL] Add G_VSCALE instruction (#84542) --- llvm/docs/GlobalISel/GenericOpcode.rst | 11 ++++++++++ .../CodeGen/GlobalISel/MachineIRBuilder.h | 22 +++++++++++++++++++ llvm/include/llvm/Support/TargetOpcodes.def | 3 +++ llvm/include/llvm/Target/GenericOpcodes.td | 9 ++++++++ .../CodeGen/GlobalISel/MachineIRBuilder.cpp | 18 +++++++++++++++ llvm/lib/CodeGen/MachineVerifier.cpp | 11 ++++++++++ .../GlobalISel/legalizer-info-validation.mir | 3 +++ llvm/test/MachineVerifier/test_g_vscale.mir | 15 +++++++++++++ 8 files changed, 92 insertions(+) create mode 100644 llvm/test/MachineVerifier/test_g_vscale.mir diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst index f9f9e1186460..bf1b3cb30a52 100644 --- a/llvm/docs/GlobalISel/GenericOpcode.rst +++ b/llvm/docs/GlobalISel/GenericOpcode.rst @@ -607,6 +607,17 @@ See the LLVM LangRef entry on '``llvm.lround.*'`` for details on behaviour. Vector Specific Operations -------------------------- +G_VSCALE +^^^^^^^^ + +Puts the value of the runtime ``vscale`` multiplied by the value in the source +operand into the destination register. This can be useful in determining the +actual runtime number of elements in a vector. + +.. code-block:: + + %0:_(s32) = G_VSCALE 4 + G_INSERT_SUBVECTOR ^^^^^^^^^^^^^^^^^^ diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 4732eaf4ee27..aaa81342845b 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -1143,6 +1143,28 @@ public: MachineInstrBuilder buildInsert(const DstOp &Res, const SrcOp &Src, const SrcOp &Op, unsigned Index); + /// Build and insert \p Res = G_VSCALE \p MinElts + /// + /// G_VSCALE puts the value of the runtime vscale multiplied by \p MinElts + /// into \p Res. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res must be a generic virtual register with scalar type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildVScale(const DstOp &Res, unsigned MinElts); + + /// Build and insert \p Res = G_VSCALE \p MinElts + /// + /// G_VSCALE puts the value of the runtime vscale multiplied by \p MinElts + /// into \p Res. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res must be a generic virtual register with scalar type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildVScale(const DstOp &Res, const ConstantInt &MinElts); + /// Build and insert a G_INTRINSIC instruction. /// /// There are four different opcodes based on combinations of whether the diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 3dade14f043b..899eaad5842a 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -727,6 +727,9 @@ HANDLE_TARGET_OPCODE(G_BR) /// Generic branch to jump table entry. HANDLE_TARGET_OPCODE(G_BRJT) +/// Generic vscale. +HANDLE_TARGET_OPCODE(G_VSCALE) + /// Generic insert subvector. HANDLE_TARGET_OPCODE(G_INSERT_SUBVECTOR) diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index 8dc84fb0ba05..67d405ba96fa 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1289,6 +1289,15 @@ def G_MERGE_VALUES : GenericInstruction { let variadicOpsType = type1; } +// Generic vscale. +// Puts the value of the runtime vscale multiplied by the value in the source +// operand into the destination register. +def G_VSCALE : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins unknown:$src); + let hasSideEffects = false; +} + /// Create a vector from multiple scalar registers. No implicit /// conversion is performed (i.e. the result element type must be the /// same as all source operands) diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 9b12d443c96e..f7aaa0f02efc 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -793,6 +793,24 @@ MachineInstrBuilder MachineIRBuilder::buildInsert(const DstOp &Res, return buildInstr(TargetOpcode::G_INSERT, Res, {Src, Op, uint64_t(Index)}); } +MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res, + unsigned MinElts) { + + auto IntN = IntegerType::get(getMF().getFunction().getContext(), + Res.getLLTTy(*getMRI()).getScalarSizeInBits()); + ConstantInt *CI = ConstantInt::get(IntN, MinElts); + return buildVScale(Res, *CI); +} + +MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res, + const ConstantInt &MinElts) { + auto VScale = buildInstr(TargetOpcode::G_VSCALE); + VScale->setDebugLoc(DebugLoc()); + Res.addDefToMIB(*getMRI(), VScale); + VScale.addCImm(&MinElts); + return VScale; +} + static unsigned getIntrinsicOpcode(bool HasSideEffects, bool IsConvergent) { if (HasSideEffects && IsConvergent) return TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS; diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index 90cbf097370d..c2d6dd35e1cb 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1613,6 +1613,17 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { report("G_BSWAP size must be a multiple of 16 bits", MI); break; } + case TargetOpcode::G_VSCALE: { + if (!MI->getOperand(1).isCImm()) { + report("G_VSCALE operand must be cimm", MI); + break; + } + if (MI->getOperand(1).getCImm()->isZero()) { + report("G_VSCALE immediate cannot be zero", MI); + break; + } + break; + } case TargetOpcode::G_INSERT_SUBVECTOR: { const MachineOperand &Src0Op = MI->getOperand(1); if (!Src0Op.isReg()) { diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index ac330918b430..c9e5f8924f8a 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -616,6 +616,9 @@ # DEBUG-NEXT: G_BRJT (opcode {{[0-9]+}}): 2 type indices # DEBUG-NEXT: .. the first uncovered type index: 2, OK # DEBUG-NEXT: .. the first uncovered imm index: 0, OK +# DEBUG-NEXT: G_VSCALE (opcode {{[0-9]+}}): 1 type index, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined # DEBUG-NEXT: G_INSERT_SUBVECTOR (opcode {{[0-9]+}}): 2 type indices, 1 imm index # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined diff --git a/llvm/test/MachineVerifier/test_g_vscale.mir b/llvm/test/MachineVerifier/test_g_vscale.mir new file mode 100644 index 000000000000..78854620913a --- /dev/null +++ b/llvm/test/MachineVerifier/test_g_vscale.mir @@ -0,0 +1,15 @@ +# RUN: not --crash llc -verify-machineinstrs -run-pass none -o /dev/null %s 2>&1 | FileCheck %s + +--- +name: g_vscale +body: | + bb.0: + + %1:_(s32) = G_CONSTANT i32 4 + + ; CHECK: G_VSCALE operand must be cimm + %2:_(s32) = G_VSCALE %1 + + ; CHECK: G_VSCALE immediate cannot be zero + %3:_(s32) = G_VSCALE i32 0 +... -- GitLab From f467cc9caf37fcf6b3523271f977585c39372d55 Mon Sep 17 00:00:00 2001 From: Dave Clausen Date: Tue, 12 Mar 2024 17:21:00 -0700 Subject: [PATCH 0066/1407] [tsan] Add missing link option to tsan test after #84923 Pull Request: https://github.com/llvm/llvm-project/pull/85003 --- compiler-rt/test/tsan/compare_exchange_acquire_fence.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler-rt/test/tsan/compare_exchange_acquire_fence.cpp b/compiler-rt/test/tsan/compare_exchange_acquire_fence.cpp index b9fd0c5ad21f..404fd2fbb3c2 100644 --- a/compiler-rt/test/tsan/compare_exchange_acquire_fence.cpp +++ b/compiler-rt/test/tsan/compare_exchange_acquire_fence.cpp @@ -1,4 +1,4 @@ -// RUN: %clangxx_tsan -O1 %s -o %t && %run %t 2>&1 +// RUN: %clangxx_tsan -O1 %s %link_libcxx_tsan -o %t && %run %t 2>&1 // This is a correct program and tsan should not report a race. // // Verify that there is a happens-before relationship between a -- GitLab From 2aacb56e8361213c1bd69c2ceafdea3aa0ca9125 Mon Sep 17 00:00:00 2001 From: 4ast Date: Tue, 12 Mar 2024 17:27:25 -0700 Subject: [PATCH 0067/1407] BPF address space insn (#84410) This commit aims to support BPF arena kernel side [feature](https://lore.kernel.org/bpf/20240209040608.98927-1-alexei.starovoitov@gmail.com/): - arena is a memory region accessible from both BPF program and userspace; - base pointers for this memory region differ between kernel and user spaces; - `dst_reg = addr_space_cast(src_reg, dst_addr_space, src_addr_space)` translates src_reg, a pointer in src_addr_space to dst_reg, equivalent pointer in dst_addr_space, {src,dst}_addr_space are immediate constants; - number 0 is assigned to kernel address space; - number 1 is assigned to user address space. On the LLVM side, the goal is to make load and store operations on arena pointers "transparent" for BPF programs: - assume that pointers with non-zero address space are pointers to arena memory; - assume that arena is identified by address space number; - assume that address space zero corresponds to kernel address space; - assume that every BPF-side load or store from arena is done via pointer in user address space, thus convert base pointers using `addr_space_cast(src_reg, 0, 1)`; Only load, store, cmpxchg and atomicrmw IR instructions are handled by this transformation. For example, the following C code: ```c #define __as __attribute__((address_space(1))) void copy(int __as *from, int __as *to) { *to = *from; } ``` Compiled to the following IR: ```llvm define void @copy(ptr addrspace(1) %from, ptr addrspace(1) %to) { entry: %0 = load i32, ptr addrspace(1) %from, align 4 store i32 %0, ptr addrspace(1) %to, align 4 ret void } ``` Is transformed to: ```llvm %to2 = addrspacecast ptr addrspace(1) %to to ptr ;; ! %from1 = addrspacecast ptr addrspace(1) %from to ptr ;; ! %0 = load i32, ptr %from1, align 4, !tbaa !3 store i32 %0, ptr %to2, align 4, !tbaa !3 ret void ``` And compiled as: ```asm r2 = addr_space_cast(r2, 0, 1) r1 = addr_space_cast(r1, 0, 1) r1 = *(u32 *)(r1 + 0) *(u32 *)(r2 + 0) = r1 exit ``` Co-authored-by: Eduard Zingerman --- clang/lib/Basic/Targets/BPF.cpp | 3 + .../test/Preprocessor/bpf-predefined-macros.c | 8 ++ .../lib/Target/BPF/AsmParser/BPFAsmParser.cpp | 1 + llvm/lib/Target/BPF/BPF.h | 8 ++ .../Target/BPF/BPFASpaceCastSimplifyPass.cpp | 92 ++++++++++++++ llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp | 116 ++++++++++++++++++ llvm/lib/Target/BPF/BPFInstrInfo.td | 29 +++++ llvm/lib/Target/BPF/BPFTargetMachine.cpp | 5 + llvm/lib/Target/BPF/CMakeLists.txt | 1 + .../test/CodeGen/BPF/addr-space-auto-casts.ll | 78 ++++++++++++ llvm/test/CodeGen/BPF/addr-space-cast.ll | 22 ++++ llvm/test/CodeGen/BPF/addr-space-gep-chain.ll | 25 ++++ llvm/test/CodeGen/BPF/addr-space-globals.ll | 30 +++++ llvm/test/CodeGen/BPF/addr-space-globals2.ll | 25 ++++ llvm/test/CodeGen/BPF/addr-space-phi.ll | 53 ++++++++ .../test/CodeGen/BPF/addr-space-simplify-1.ll | 19 +++ .../test/CodeGen/BPF/addr-space-simplify-2.ll | 21 ++++ .../test/CodeGen/BPF/addr-space-simplify-3.ll | 26 ++++ .../test/CodeGen/BPF/addr-space-simplify-4.ll | 21 ++++ .../test/CodeGen/BPF/addr-space-simplify-5.ll | 25 ++++ .../test/CodeGen/BPF/assembler-disassembler.s | 7 ++ 21 files changed, 615 insertions(+) create mode 100644 llvm/lib/Target/BPF/BPFASpaceCastSimplifyPass.cpp create mode 100644 llvm/test/CodeGen/BPF/addr-space-auto-casts.ll create mode 100644 llvm/test/CodeGen/BPF/addr-space-cast.ll create mode 100644 llvm/test/CodeGen/BPF/addr-space-gep-chain.ll create mode 100644 llvm/test/CodeGen/BPF/addr-space-globals.ll create mode 100644 llvm/test/CodeGen/BPF/addr-space-globals2.ll create mode 100644 llvm/test/CodeGen/BPF/addr-space-phi.ll create mode 100644 llvm/test/CodeGen/BPF/addr-space-simplify-1.ll create mode 100644 llvm/test/CodeGen/BPF/addr-space-simplify-2.ll create mode 100644 llvm/test/CodeGen/BPF/addr-space-simplify-3.ll create mode 100644 llvm/test/CodeGen/BPF/addr-space-simplify-4.ll create mode 100644 llvm/test/CodeGen/BPF/addr-space-simplify-5.ll diff --git a/clang/lib/Basic/Targets/BPF.cpp b/clang/lib/Basic/Targets/BPF.cpp index e3fbbb720d06..26a54f631fcf 100644 --- a/clang/lib/Basic/Targets/BPF.cpp +++ b/clang/lib/Basic/Targets/BPF.cpp @@ -35,6 +35,9 @@ void BPFTargetInfo::getTargetDefines(const LangOptions &Opts, Builder.defineMacro("__BPF_CPU_VERSION__", "0"); return; } + + Builder.defineMacro("__BPF_FEATURE_ARENA_CAST"); + if (CPU.empty() || CPU == "generic" || CPU == "v1") { Builder.defineMacro("__BPF_CPU_VERSION__", "1"); return; diff --git a/clang/test/Preprocessor/bpf-predefined-macros.c b/clang/test/Preprocessor/bpf-predefined-macros.c index ff4d00ac3bcf..fea24d1ea0ff 100644 --- a/clang/test/Preprocessor/bpf-predefined-macros.c +++ b/clang/test/Preprocessor/bpf-predefined-macros.c @@ -61,6 +61,9 @@ int r; #ifdef __BPF_FEATURE_ST int s; #endif +#ifdef __BPF_FEATURE_ARENA_CAST +int t; +#endif // CHECK: int b; // CHECK: int c; @@ -90,6 +93,11 @@ int s; // CPU_V4: int r; // CPU_V4: int s; +// CPU_V1: int t; +// CPU_V2: int t; +// CPU_V3: int t; +// CPU_V4: int t; + // CPU_GENERIC: int g; // CPU_PROBE: int f; diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp index 0d1eef60c3b5..3145bc3d19f5 100644 --- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp +++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp @@ -271,6 +271,7 @@ public: .Case("xchg32_32", true) .Case("cmpxchg_64", true) .Case("cmpxchg32_32", true) + .Case("addr_space_cast", true) .Default(false); } }; diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h index 5c77d183e1ef..bbdbdbbde532 100644 --- a/llvm/lib/Target/BPF/BPF.h +++ b/llvm/lib/Target/BPF/BPF.h @@ -66,6 +66,14 @@ public: static bool isRequired() { return true; } }; +class BPFASpaceCastSimplifyPass + : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + + static bool isRequired() { return true; } +}; + class BPFAdjustOptPass : public PassInfoMixin { public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); diff --git a/llvm/lib/Target/BPF/BPFASpaceCastSimplifyPass.cpp b/llvm/lib/Target/BPF/BPFASpaceCastSimplifyPass.cpp new file mode 100644 index 000000000000..f87b299bbba6 --- /dev/null +++ b/llvm/lib/Target/BPF/BPFASpaceCastSimplifyPass.cpp @@ -0,0 +1,92 @@ +//===-- BPFASpaceCastSimplifyPass.cpp - BPF addrspacecast simplications --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "BPF.h" +#include + +#define DEBUG_TYPE "bpf-aspace-simplify" + +using namespace llvm; + +namespace { + +struct CastGEPCast { + AddrSpaceCastInst *OuterCast; + + // Match chain of instructions: + // %inner = addrspacecast N->M + // %gep = getelementptr %inner, ... + // %outer = addrspacecast M->N %gep + // Where I is %outer. + static std::optional match(Value *I) { + auto *OuterCast = dyn_cast(I); + if (!OuterCast) + return std::nullopt; + auto *GEP = dyn_cast(OuterCast->getPointerOperand()); + if (!GEP) + return std::nullopt; + auto *InnerCast = dyn_cast(GEP->getPointerOperand()); + if (!InnerCast) + return std::nullopt; + if (InnerCast->getSrcAddressSpace() != OuterCast->getDestAddressSpace()) + return std::nullopt; + if (InnerCast->getDestAddressSpace() != OuterCast->getSrcAddressSpace()) + return std::nullopt; + return CastGEPCast{OuterCast}; + } + + static PointerType *changeAddressSpace(PointerType *Ty, unsigned AS) { + return Ty->get(Ty->getContext(), AS); + } + + // Assuming match(this->OuterCast) is true, convert: + // (addrspacecast M->N (getelementptr (addrspacecast N->M ptr) ...)) + // To: + // (getelementptr ptr ...) + GetElementPtrInst *rewrite() { + auto *GEP = cast(OuterCast->getPointerOperand()); + auto *InnerCast = cast(GEP->getPointerOperand()); + unsigned AS = OuterCast->getDestAddressSpace(); + auto *NewGEP = cast(GEP->clone()); + NewGEP->setName(GEP->getName()); + NewGEP->insertAfter(OuterCast); + NewGEP->setOperand(0, InnerCast->getPointerOperand()); + auto *GEPTy = cast(GEP->getType()); + NewGEP->mutateType(changeAddressSpace(GEPTy, AS)); + OuterCast->replaceAllUsesWith(NewGEP); + OuterCast->eraseFromParent(); + if (GEP->use_empty()) + GEP->eraseFromParent(); + if (InnerCast->use_empty()) + InnerCast->eraseFromParent(); + return NewGEP; + } +}; + +} // anonymous namespace + +PreservedAnalyses BPFASpaceCastSimplifyPass::run(Function &F, + FunctionAnalysisManager &AM) { + SmallVector WorkList; + bool Changed = false; + for (BasicBlock &BB : F) { + for (Instruction &I : BB) + if (auto It = CastGEPCast::match(&I)) + WorkList.push_back(It.value()); + Changed |= !WorkList.empty(); + + while (!WorkList.empty()) { + CastGEPCast InsnChain = WorkList.pop_back_val(); + GetElementPtrInst *NewGEP = InsnChain.rewrite(); + for (User *U : NewGEP->users()) + if (auto It = CastGEPCast::match(U)) + WorkList.push_back(It.value()); + } + } + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp index 81effc9b1db4..edd59aaa6d01 100644 --- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp +++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp @@ -14,6 +14,8 @@ // optimizations are done and those builtins can be removed. // - remove llvm.bpf.getelementptr.and.load builtins. // - remove llvm.bpf.getelementptr.and.store builtins. +// - for loads and stores with base addresses from non-zero address space +// cast base address to zero address space (support for BPF arenas). // //===----------------------------------------------------------------------===// @@ -55,6 +57,7 @@ private: bool removeCompareBuiltin(Module &M); bool sinkMinMax(Module &M); bool removeGEPBuiltins(Module &M); + bool insertASpaceCasts(Module &M); }; } // End anonymous namespace @@ -416,11 +419,124 @@ bool BPFCheckAndAdjustIR::removeGEPBuiltins(Module &M) { return Changed; } +// Wrap ToWrap with cast to address space zero: +// - if ToWrap is a getelementptr, +// wrap it's base pointer instead and return a copy; +// - if ToWrap is Instruction, insert address space cast +// immediately after ToWrap; +// - if ToWrap is not an Instruction (function parameter +// or a global value), insert address space cast at the +// beginning of the Function F; +// - use Cache to avoid inserting too many casts; +static Value *aspaceWrapValue(DenseMap &Cache, Function *F, + Value *ToWrap) { + auto It = Cache.find(ToWrap); + if (It != Cache.end()) + return It->getSecond(); + + if (auto *GEP = dyn_cast(ToWrap)) { + Value *Ptr = GEP->getPointerOperand(); + Value *WrappedPtr = aspaceWrapValue(Cache, F, Ptr); + auto *GEPTy = cast(GEP->getType()); + auto *NewGEP = GEP->clone(); + NewGEP->insertAfter(GEP); + NewGEP->mutateType(GEPTy->getPointerTo(0)); + NewGEP->setOperand(GEP->getPointerOperandIndex(), WrappedPtr); + NewGEP->setName(GEP->getName()); + Cache[ToWrap] = NewGEP; + return NewGEP; + } + + IRBuilder IB(F->getContext()); + if (Instruction *InsnPtr = dyn_cast(ToWrap)) + IB.SetInsertPoint(*InsnPtr->getInsertionPointAfterDef()); + else + IB.SetInsertPoint(F->getEntryBlock().getFirstInsertionPt()); + auto *PtrTy = cast(ToWrap->getType()); + auto *ASZeroPtrTy = PtrTy->getPointerTo(0); + auto *ACast = IB.CreateAddrSpaceCast(ToWrap, ASZeroPtrTy, ToWrap->getName()); + Cache[ToWrap] = ACast; + return ACast; +} + +// Wrap a pointer operand OpNum of instruction I +// with cast to address space zero +static void aspaceWrapOperand(DenseMap &Cache, Instruction *I, + unsigned OpNum) { + Value *OldOp = I->getOperand(OpNum); + if (OldOp->getType()->getPointerAddressSpace() == 0) + return; + + Value *NewOp = aspaceWrapValue(Cache, I->getFunction(), OldOp); + I->setOperand(OpNum, NewOp); + // Check if there are any remaining users of old GEP, + // delete those w/o users + for (;;) { + auto *OldGEP = dyn_cast(OldOp); + if (!OldGEP) + break; + if (!OldGEP->use_empty()) + break; + OldOp = OldGEP->getPointerOperand(); + OldGEP->eraseFromParent(); + } +} + +// Support for BPF arenas: +// - for each function in the module M, update pointer operand of +// each memory access instruction (load/store/cmpxchg/atomicrmw) +// by casting it from non-zero address space to zero address space, e.g: +// +// (load (ptr addrspace (N) %p) ...) +// -> (load (addrspacecast ptr addrspace (N) %p to ptr)) +// +// - assign section with name .arena.N for globals defined in +// non-zero address space N +bool BPFCheckAndAdjustIR::insertASpaceCasts(Module &M) { + bool Changed = false; + for (Function &F : M) { + DenseMap CastsCache; + for (BasicBlock &BB : F) { + for (Instruction &I : BB) { + unsigned PtrOpNum; + + if (auto *LD = dyn_cast(&I)) + PtrOpNum = LD->getPointerOperandIndex(); + else if (auto *ST = dyn_cast(&I)) + PtrOpNum = ST->getPointerOperandIndex(); + else if (auto *CmpXchg = dyn_cast(&I)) + PtrOpNum = CmpXchg->getPointerOperandIndex(); + else if (auto *RMW = dyn_cast(&I)) + PtrOpNum = RMW->getPointerOperandIndex(); + else + continue; + + aspaceWrapOperand(CastsCache, &I, PtrOpNum); + } + } + Changed |= !CastsCache.empty(); + } + // Merge all globals within same address space into single + // .arena. section + for (GlobalVariable &G : M.globals()) { + if (G.getAddressSpace() == 0 || G.hasSection()) + continue; + SmallString<16> SecName; + raw_svector_ostream OS(SecName); + OS << ".arena." << G.getAddressSpace(); + G.setSection(SecName); + // Prevent having separate section for constants + G.setConstant(false); + } + return Changed; +} + bool BPFCheckAndAdjustIR::adjustIR(Module &M) { bool Changed = removePassThroughBuiltin(M); Changed = removeCompareBuiltin(M) || Changed; Changed = sinkMinMax(M) || Changed; Changed = removeGEPBuiltins(M) || Changed; + Changed = insertASpaceCasts(M) || Changed; return Changed; } diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td index 82d347023106..7198e9499bc3 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.td +++ b/llvm/lib/Target/BPF/BPFInstrInfo.td @@ -420,6 +420,35 @@ let Predicates = [BPFHasMovsx] in { } } +def ADDR_SPACE_CAST + : ALU_RR { + bits<64> dst_as; + bits<64> src_as; + + let Inst{47-32} = 1; + let Inst{31-16} = dst_as{15-0}; + let Inst{15-0} = src_as{15-0}; +} + +def SrcAddrSpace : SDNodeXFormgetTargetConstant( + cast(N)->getSrcAddressSpace(), + SDLoc(N), MVT::i64); +}]>; + +def DstAddrSpace : SDNodeXFormgetTargetConstant( + cast(N)->getDestAddressSpace(), + SDLoc(N), MVT::i64); +}]>; + +def : Pat<(addrspacecast:$this GPR:$src), + (ADDR_SPACE_CAST $src, (DstAddrSpace $this), (SrcAddrSpace $this))>; + def FI_ri : TYPE_LD_ST 42) +; a = magic1(); +; else +; a = magic2(); +; a[5] = 7; +; } +; +; Using the following command: +; +; clang --target=bpf -O2 -S -emit-llvm -o t.ll t.c + +define void @test(i64 noundef %i) { +; CHECK: if.end: +; CHECK-NEXT: [[A_0:%.*]] = phi ptr addrspace(1) +; CHECK-NEXT: [[A_01:%.*]] = addrspacecast ptr addrspace(1) [[A_0]] to ptr +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A_01]], i64 5 +; CHECK-NEXT: store i32 7, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: ret void +; +entry: + %cmp = icmp sgt i64 %i, 42 + br i1 %cmp, label %if.then, label %if.else + +if.then: ; preds = %entry + %call = tail call ptr addrspace(1) @magic1() + br label %if.end + +if.else: ; preds = %entry + %call1 = tail call ptr addrspace(1) @magic2() + br label %if.end + +if.end: ; preds = %if.else, %if.then + %a.0 = phi ptr addrspace(1) [ %call, %if.then ], [ %call1, %if.else ] + %arrayidx = getelementptr inbounds i32, ptr addrspace(1) %a.0, i64 5 + store i32 7, ptr addrspace(1) %arrayidx, align 4 + ret void +} + +declare ptr addrspace(1) @magic1(...) +declare ptr addrspace(1) @magic2(...) diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-1.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-1.ll new file mode 100644 index 000000000000..32d67284d1c1 --- /dev/null +++ b/llvm/test/CodeGen/BPF/addr-space-simplify-1.ll @@ -0,0 +1,19 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s + +; Check that bpf-aspace-simplify pass removes unnecessary (for BPF) +; address space casts for cast M->N -> GEP -> cast N->M chain. + +define dso_local ptr addrspace(1) @test (ptr addrspace(1) %p) { +; CHECK-LABEL: define dso_local ptr addrspace(1) @test( +; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[B1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 8 +; CHECK-NEXT: ret ptr addrspace(1) [[B1]] +; + entry: + %a = addrspacecast ptr addrspace(1) %p to ptr + %b = getelementptr inbounds i8, ptr %a, i64 8 + %c = addrspacecast ptr %b to ptr addrspace(1) + ret ptr addrspace(1) %c +} diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-2.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-2.ll new file mode 100644 index 000000000000..a2965554a973 --- /dev/null +++ b/llvm/test/CodeGen/BPF/addr-space-simplify-2.ll @@ -0,0 +1,21 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s + +; Check that bpf-aspace-simplify pass does not change +; chain 'cast M->N -> GEP -> cast N->K'. + +define dso_local ptr addrspace(2) @test (ptr addrspace(1) %p) { +; CHECK-LABEL: define dso_local ptr addrspace(2) @test( +; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A:%.*]] = addrspacecast ptr addrspace(1) [[P]] to ptr +; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8 +; CHECK-NEXT: [[C:%.*]] = addrspacecast ptr [[B]] to ptr addrspace(2) +; CHECK-NEXT: ret ptr addrspace(2) [[C]] +; + entry: + %a = addrspacecast ptr addrspace(1) %p to ptr + %b = getelementptr inbounds i8, ptr %a, i64 8 + %c = addrspacecast ptr %b to ptr addrspace(2) + ret ptr addrspace(2) %c +} diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-3.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-3.ll new file mode 100644 index 000000000000..a7736c462b44 --- /dev/null +++ b/llvm/test/CodeGen/BPF/addr-space-simplify-3.ll @@ -0,0 +1,26 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s + +; Check that when bpf-aspace-simplify pass modifies chain +; 'cast M->N -> GEP -> cast N->M' it does not remove GEP, +; when that GEP is used by some other instruction. + +define dso_local ptr addrspace(1) @test (ptr addrspace(1) %p) { +; CHECK-LABEL: define dso_local ptr addrspace(1) @test( +; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A:%.*]] = addrspacecast ptr addrspace(1) [[P]] to ptr +; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8 +; CHECK-NEXT: [[B1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 8 +; CHECK-NEXT: call void @sink(ptr [[B]]) +; CHECK-NEXT: ret ptr addrspace(1) [[B1]] +; + entry: + %a = addrspacecast ptr addrspace(1) %p to ptr + %b = getelementptr inbounds i8, ptr %a, i64 8 + %c = addrspacecast ptr %b to ptr addrspace(1) + call void @sink(ptr %b) + ret ptr addrspace(1) %c +} + +declare dso_local void @sink(ptr) diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-4.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-4.ll new file mode 100644 index 000000000000..b2c384bbb6ab --- /dev/null +++ b/llvm/test/CodeGen/BPF/addr-space-simplify-4.ll @@ -0,0 +1,21 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s + +; Check that bpf-aspace-simplify pass simplifies chain +; 'cast K->M -> cast M->N -> GEP -> cast N->M -> cast M->K'. + +define dso_local ptr addrspace(2) @test (ptr addrspace(2) %p) { +; CHECK-LABEL: define dso_local ptr addrspace(2) @test( +; CHECK-SAME: ptr addrspace(2) [[P:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C12:%.*]] = getelementptr inbounds i8, ptr addrspace(2) [[P]], i64 8 +; CHECK-NEXT: ret ptr addrspace(2) [[C12]] +; + entry: + %a = addrspacecast ptr addrspace(2) %p to ptr addrspace(1) + %b = addrspacecast ptr addrspace(1) %a to ptr + %c = getelementptr inbounds i8, ptr %b, i64 8 + %d = addrspacecast ptr %c to ptr addrspace(1) + %e = addrspacecast ptr addrspace (1) %d to ptr addrspace(2) + ret ptr addrspace(2) %e +} diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-5.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-5.ll new file mode 100644 index 000000000000..b62d25384d95 --- /dev/null +++ b/llvm/test/CodeGen/BPF/addr-space-simplify-5.ll @@ -0,0 +1,25 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s + +; Check that bpf-aspace-simplify pass removes unnecessary (for BPF) +; address space casts for cast M->N -> GEP -> cast N->M chain, +; where chain is split between several BBs. + +define dso_local ptr addrspace(1) @test (ptr addrspace(1) %p) { +; CHECK-LABEL: define dso_local ptr addrspace(1) @test( +; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[EXIT:%.*]] +; CHECK: exit: +; CHECK-NEXT: [[B1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 8 +; CHECK-NEXT: ret ptr addrspace(1) [[B1]] +; +entry: + %a = addrspacecast ptr addrspace(1) %p to ptr + %b = getelementptr inbounds i8, ptr %a, i64 8 + br label %exit + +exit: + %c = addrspacecast ptr %b to ptr addrspace(1) + ret ptr addrspace(1) %c +} diff --git a/llvm/test/CodeGen/BPF/assembler-disassembler.s b/llvm/test/CodeGen/BPF/assembler-disassembler.s index 2bc7421c2471..991d6edc683a 100644 --- a/llvm/test/CodeGen/BPF/assembler-disassembler.s +++ b/llvm/test/CodeGen/BPF/assembler-disassembler.s @@ -289,3 +289,10 @@ r0 = *(u32*)skb[42] r0 = *(u8*)skb[r1] r0 = *(u16*)skb[r1] r0 = *(u32*)skb[r1] + +// CHECK: bf 10 01 00 01 00 00 00 r0 = addr_space_cast(r1, 0x0, 0x1) +// CHECK: bf 21 01 00 00 00 01 00 r1 = addr_space_cast(r2, 0x1, 0x0) +// CHECK: bf 43 01 00 2a 00 07 00 r3 = addr_space_cast(r4, 0x7, 0x2a) +r0 = addr_space_cast(r1, 0, 1) +r1 = addr_space_cast(r2, 1, 0) +r3 = addr_space_cast(r4, 7, 42) -- GitLab From d014708a217beaef04f9533d311c68bda71a52f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Ferrer=20Ib=C3=A1=C3=B1ez?= Date: Wed, 13 Mar 2024 01:30:26 +0100 Subject: [PATCH 0068/1407] [llvm][Mips] Use a Target ISD opcode for PseudoD_SELECT (#84294) The Mips target uses two TargetOpcode enumerators called `PseudoD_SELECT_I` and `PseudoD_SELECT_I64`. A SDAG node is created using these enumerators which is manually selected in `MipsSEISelDAGToDAG.cpp` and ultimately expanded in `EmitInstrWithCustomInserter` in `MipsISelLowering.cpp`. This is not causing any upstream build to fail at the moment but it is not guaranteed that these enumerators do not clash with Target ISD nodes (i.e. those in the `MipsISD` namespace). We have seen this happening in our downstream builds in which `Mips::PseudoD_SELECT_I` ends having the same integer value as `MipsISD::VEXTRACT_ZEXT_ELT`. This confuses the function `trySelect` in `MipsSEISelDAGToDAG.cpp` and causes a crash in 3 tests. This change adds a new Target ISD opcode for these two cases and uses them for the SDAG nodes. No test is included because this is a potential error in the future not one that can be demonstrated in the current codebase. --- llvm/lib/Target/Mips/MipsISelLowering.cpp | 6 ++++-- llvm/lib/Target/Mips/MipsISelLowering.h | 4 ++++ llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp index 97e830cec27c..7e5f148e7bf4 100644 --- a/llvm/lib/Target/Mips/MipsISelLowering.cpp +++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp @@ -239,6 +239,8 @@ const char *MipsTargetLowering::getTargetNodeName(unsigned Opcode) const { case MipsISD::MAQ_S_W_PHR: return "MipsISD::MAQ_S_W_PHR"; case MipsISD::MAQ_SA_W_PHL: return "MipsISD::MAQ_SA_W_PHL"; case MipsISD::MAQ_SA_W_PHR: return "MipsISD::MAQ_SA_W_PHR"; + case MipsISD::DOUBLE_SELECT_I: return "MipsISD::DOUBLE_SELECT_I"; + case MipsISD::DOUBLE_SELECT_I64: return "MipsISD::DOUBLE_SELECT_I64"; case MipsISD::DPAU_H_QBL: return "MipsISD::DPAU_H_QBL"; case MipsISD::DPAU_H_QBR: return "MipsISD::DPAU_H_QBR"; case MipsISD::DPSU_H_QBL: return "MipsISD::DPSU_H_QBL"; @@ -2652,8 +2654,8 @@ SDValue MipsTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG, if (!(Subtarget.hasMips4() || Subtarget.hasMips32())) { SDVTList VTList = DAG.getVTList(VT, VT); - return DAG.getNode(Subtarget.isGP64bit() ? Mips::PseudoD_SELECT_I64 - : Mips::PseudoD_SELECT_I, + return DAG.getNode(Subtarget.isGP64bit() ? MipsISD::DOUBLE_SELECT_I64 + : MipsISD::DOUBLE_SELECT_I, DL, VTList, Cond, ShiftRightHi, IsSRA ? Ext : DAG.getConstant(0, DL, VT), Or, ShiftRightHi); diff --git a/llvm/lib/Target/Mips/MipsISelLowering.h b/llvm/lib/Target/Mips/MipsISelLowering.h index 7d243eeca5c6..84ad40d6bbbe 100644 --- a/llvm/lib/Target/Mips/MipsISelLowering.h +++ b/llvm/lib/Target/Mips/MipsISelLowering.h @@ -242,6 +242,10 @@ class TargetRegisterClass; VEXTRACT_SEXT_ELT, VEXTRACT_ZEXT_ELT, + // Double select nodes for machines without conditional-move. + DOUBLE_SELECT_I, + DOUBLE_SELECT_I64, + // Load/Store Left/Right nodes. LWL = ISD::FIRST_TARGET_MEMORY_OPCODE, LWR, diff --git a/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp b/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp index c0e978018919..ab39d1b661ef 100644 --- a/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp +++ b/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp @@ -741,8 +741,8 @@ bool MipsSEDAGToDAGISel::trySelect(SDNode *Node) { switch(Opcode) { default: break; - case Mips::PseudoD_SELECT_I: - case Mips::PseudoD_SELECT_I64: { + case MipsISD::DOUBLE_SELECT_I: + case MipsISD::DOUBLE_SELECT_I64: { MVT VT = Subtarget->isGP64bit() ? MVT::i64 : MVT::i32; SDValue cond = Node->getOperand(0); SDValue Hi1 = Node->getOperand(1); -- GitLab From 97c0cad388e5d3f5089a05001149ea7eeabbd777 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Wed, 13 Mar 2024 00:35:08 +0000 Subject: [PATCH 0069/1407] [gn build] Port 2aacb56e8361 --- llvm/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn index 668512ecba88..aa594df8c164 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn @@ -60,6 +60,7 @@ static_library("LLVMBPFCodeGen") { ] include_dirs = [ "." ] sources = [ + "BPFASpaceCastSimplifyPass.cpp", "BPFAbstractMemberAccess.cpp", "BPFAdjustOpt.cpp", "BPFAsmPrinter.cpp", -- GitLab From 8bda5657332c7a94900d3eb2891d2b86e60b0e68 Mon Sep 17 00:00:00 2001 From: Qizhi Hu <836744285@qq.com> Date: Wed, 13 Mar 2024 08:42:22 +0800 Subject: [PATCH 0070/1407] [Clang][Sema] Allow access to a public template alias declaration that refers to friend's private nested type (#83847) This patch attempts to fix https://github.com/llvm/llvm-project/issues/25708 Current access check missed qualifier(`NestedNameSpecifier`) in friend class checking. Add it to `Records` of `EffectiveContext` by changing the `DeclContext` makes `MatchesFriend` work. Co-authored-by: huqizhi <836744285@qq.com> --- clang/docs/ReleaseNotes.rst | 2 ++ clang/lib/Sema/SemaTemplate.cpp | 10 +++++++--- clang/test/SemaTemplate/PR25708.cpp | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 clang/test/SemaTemplate/PR25708.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index e14c92eae0af..64a9fe0d8bcc 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -360,6 +360,8 @@ Bug Fixes to C++ Support when one of the function had more specialized templates. Fixes (`#82509 `_) and (`#74494 `_) +- Allow access to a public template alias declaration that refers to friend's + private nested type. (#GH25708). Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index d62095558d0f..d8c9a5c09944 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -4343,9 +4343,13 @@ QualType Sema::CheckTemplateIdType(TemplateName Name, if (Inst.isInvalid()) return QualType(); - CanonType = SubstType(Pattern->getUnderlyingType(), - TemplateArgLists, AliasTemplate->getLocation(), - AliasTemplate->getDeclName()); + std::optional SavedContext; + if (!AliasTemplate->getDeclContext()->isFileContext()) + SavedContext.emplace(*this, AliasTemplate->getDeclContext()); + + CanonType = + SubstType(Pattern->getUnderlyingType(), TemplateArgLists, + AliasTemplate->getLocation(), AliasTemplate->getDeclName()); if (CanonType.isNull()) { // If this was enable_if and we failed to find the nested type // within enable_if in a SFINAE context, dig out the specific diff --git a/clang/test/SemaTemplate/PR25708.cpp b/clang/test/SemaTemplate/PR25708.cpp new file mode 100644 index 000000000000..6a214fc6b43b --- /dev/null +++ b/clang/test/SemaTemplate/PR25708.cpp @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -std=c++11 -verify %s +// expected-no-diagnostics + +struct FooAccessor +{ + template + using Foo = typename T::Foo; +}; + +class Type +{ + friend struct FooAccessor; + + using Foo = int; +}; + +int main() +{ + FooAccessor::Foo t; +} -- GitLab From 0fae4530e8476328f9a19b3ea4338b3a1e2187a8 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Tue, 12 Mar 2024 17:44:04 -0700 Subject: [PATCH 0071/1407] [LangRef] Fix mistake in example (#84849) --- llvm/docs/LangRef.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 77ec72f176d6..ecedd3a32c7b 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -27553,12 +27553,12 @@ in example below: .. code-block:: text %cond = call i1 @llvm.experimental.widenable.condition() - br i1 %cond, label %solution_1, label %solution_2 + br i1 %cond, label %fast_path, label %slow_path - label %fast_path: + fast_path: ; Apply memory-consuming but fast solution for a task. - label %slow_path: + slow_path: ; Cheap in memory but slow solution. Whether the result of intrinsic's call is `true` or `false`, -- GitLab From dcd9f49c2214891e3e0faffa70cf1a082434592a Mon Sep 17 00:00:00 2001 From: Charlie Barto Date: Tue, 12 Mar 2024 17:45:00 -0700 Subject: [PATCH 0072/1407] [sanitizer][windows] report symbols in clang_rt. or \compiler-rt\lib\ as internal. (#84971) This is the windows equivalent to the existing filters. Work from https://github.com/llvm/llvm-project/pull/81677 that can be applied separately (and is actually not critical for that PR) --- .../lib/sanitizer_common/sanitizer_symbolizer_report.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp index f6b157c07c65..ffbaf1468ec8 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp @@ -39,8 +39,12 @@ static bool FrameIsInternal(const SymbolizedStack *frame) { internal_strstr(file, "/include/c++/") || internal_strstr(file, "/include/g++"))) return true; + if (file && internal_strstr(file, "\\compiler-rt\\lib\\")) + return true; if (module && (internal_strstr(module, "libclang_rt."))) return true; + if (module && (internal_strstr(module, "clang_rt."))) + return true; return false; } -- GitLab From c1ac9a09d04e9be8f8f4860416528baf3691848d Mon Sep 17 00:00:00 2001 From: Yinying Li Date: Tue, 12 Mar 2024 20:57:21 -0400 Subject: [PATCH 0073/1407] [mlir][sparse] Finish migrating integration tests to use sparse_tensor.print (#84997) --- .../SparseTensor/CPU/sparse_conversion.mlir | 367 ++++++++---------- .../CPU/sparse_conversion_block.mlir | 70 ++-- .../CPU/sparse_conversion_dyn.mlir | 122 +++--- .../CPU/sparse_conversion_element.mlir | 4 +- .../CPU/sparse_conversion_ptr.mlir | 144 ++++--- .../CPU/sparse_conversion_sparse2dense.mlir | 4 +- .../CPU/sparse_conversion_sparse2sparse.mlir | 4 +- .../SparseTensor/CPU/sparse_coo_test.mlir | 47 ++- .../CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir | 4 +- .../Dialect/SparseTensor/CPU/sparse_dot.mlir | 38 +- .../SparseTensor/CPU/sparse_expand.mlir | 32 +- .../SparseTensor/CPU/sparse_expand_shape.mlir | 123 ++++-- .../CPU/sparse_filter_conv2d.mlir | 32 +- .../SparseTensor/CPU/sparse_flatten.mlir | 4 +- .../CPU/sparse_foreach_slices.mlir | 4 +- .../SparseTensor/CPU/sparse_generate.mlir | 20 +- .../SparseTensor/CPU/sparse_index.mlir | 176 +++++---- .../SparseTensor/CPU/sparse_index_dense.mlir | 4 +- .../SparseTensor/CPU/sparse_insert_3d.mlir | 4 +- 19 files changed, 670 insertions(+), 533 deletions(-) diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir index f13c1c66df6d..8024c1281895 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -46,28 +46,10 @@ // Integration test that tests conversions between sparse tensors. // module { - // - // Output utilities. - // - func.func @dumpf64(%arg0: memref) { - %c0 = arith.constant 0 : index - %d0 = arith.constant -1.0 : f64 - %0 = vector.transfer_read %arg0[%c0], %d0: memref, vector<24xf64> - vector.print %0 : vector<24xf64> - return - } - func.func @dumpidx(%arg0: memref) { - %c0 = arith.constant 0 : index - %d0 = arith.constant 0 : index - %0 = vector.transfer_read %arg0[%c0], %d0: memref, vector<25xindex> - vector.print %0 : vector<25xindex> - return - } - // // Main driver. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index %c2 = arith.constant 2 : index @@ -110,195 +92,176 @@ module { %i = sparse_tensor.convert %3 : tensor<2x3x4xf64, #Tensor3> to tensor<2x3x4xf64, #Tensor3> // - // Check number_of_entries. + // Verify the outputs. // - // CHECK-COUNT-12: 24 - %nv1 = sparse_tensor.number_of_entries %1 : tensor<2x3x4xf64, #Tensor1> - %nv2 = sparse_tensor.number_of_entries %2 : tensor<2x3x4xf64, #Tensor2> - %nv3 = sparse_tensor.number_of_entries %3 : tensor<2x3x4xf64, #Tensor3> - %nav = sparse_tensor.number_of_entries %a : tensor<2x3x4xf64, #Tensor1> - %nbv = sparse_tensor.number_of_entries %b : tensor<2x3x4xf64, #Tensor1> - %ncv = sparse_tensor.number_of_entries %c : tensor<2x3x4xf64, #Tensor1> - %ndv = sparse_tensor.number_of_entries %d : tensor<2x3x4xf64, #Tensor2> - %nev = sparse_tensor.number_of_entries %e : tensor<2x3x4xf64, #Tensor2> - %nfv = sparse_tensor.number_of_entries %f : tensor<2x3x4xf64, #Tensor2> - %ngv = sparse_tensor.number_of_entries %g : tensor<2x3x4xf64, #Tensor3> - %nhv = sparse_tensor.number_of_entries %h : tensor<2x3x4xf64, #Tensor3> - %niv = sparse_tensor.number_of_entries %i : tensor<2x3x4xf64, #Tensor3> - vector.print %nv1 : index - vector.print %nv2 : index - vector.print %nv3 : index - vector.print %nav : index - vector.print %nbv : index - vector.print %ncv : index - vector.print %ndv : index - vector.print %nev : index - vector.print %nfv : index - vector.print %ngv : index - vector.print %nhv : index - vector.print %niv : index - + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 2, 3, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 1 + // CHECK-NEXT: pos[1] : ( 0, 3, 6 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 + // CHECK-NEXT: ---- // - // Check values. + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 3, 4, 2 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: values : ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24 + // CHECK-NEXT: ---- // - // CHECK: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 ) - // CHECK-NEXT: ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24 ) - // CHECK-NEXT: ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24 ) - // CHECK-NEXT: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 ) - // CHECK-NEXT: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 ) - // CHECK-NEXT: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 ) - // CHECK-NEXT: ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24 ) - // CHECK-NEXT: ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24 ) - // CHECK-NEXT: ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24 ) - // CHECK-NEXT: ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24 ) - // CHECK-NEXT: ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24 ) - // CHECK-NEXT: ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 4, 2, 3 ) + // CHECK-NEXT: pos[0] : ( 0, 4 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8 + // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9, 12, 15, 18, 21, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: values : ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24 + // CHECK-NEXT: ---- // - %v1 = sparse_tensor.values %1 : tensor<2x3x4xf64, #Tensor1> to memref - %v2 = sparse_tensor.values %2 : tensor<2x3x4xf64, #Tensor2> to memref - %v3 = sparse_tensor.values %3 : tensor<2x3x4xf64, #Tensor3> to memref - %av = sparse_tensor.values %a : tensor<2x3x4xf64, #Tensor1> to memref - %bv = sparse_tensor.values %b : tensor<2x3x4xf64, #Tensor1> to memref - %cv = sparse_tensor.values %c : tensor<2x3x4xf64, #Tensor1> to memref - %dv = sparse_tensor.values %d : tensor<2x3x4xf64, #Tensor2> to memref - %ev = sparse_tensor.values %e : tensor<2x3x4xf64, #Tensor2> to memref - %fv = sparse_tensor.values %f : tensor<2x3x4xf64, #Tensor2> to memref - %gv = sparse_tensor.values %g : tensor<2x3x4xf64, #Tensor3> to memref - %hv = sparse_tensor.values %h : tensor<2x3x4xf64, #Tensor3> to memref - %iv = sparse_tensor.values %i : tensor<2x3x4xf64, #Tensor3> to memref - - call @dumpf64(%v1) : (memref) -> () - call @dumpf64(%v2) : (memref) -> () - call @dumpf64(%v3) : (memref) -> () - call @dumpf64(%av) : (memref) -> () - call @dumpf64(%bv) : (memref) -> () - call @dumpf64(%cv) : (memref) -> () - call @dumpf64(%dv) : (memref) -> () - call @dumpf64(%ev) : (memref) -> () - call @dumpf64(%fv) : (memref) -> () - call @dumpf64(%gv) : (memref) -> () - call @dumpf64(%hv) : (memref) -> () - call @dumpf64(%iv) : (memref) -> () - + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 2, 3, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 1 + // CHECK-NEXT: pos[1] : ( 0, 3, 6 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 + // CHECK-NEXT: ---- // - // Check coordinates. + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 2, 3, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 1 + // CHECK-NEXT: pos[1] : ( 0, 3, 6 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 + // CHECK-NEXT: ---- // - // CHECK-NEXT: ( 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0 ) - // CHECK-NEXT: ( 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0 ) - // CHECK-NEXT: ( 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0 ) - // CHECK-NEXT: ( 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 2, 3, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 1 + // CHECK-NEXT: pos[1] : ( 0, 3, 6 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 + // CHECK-NEXT: ---- // - %v10 = sparse_tensor.coordinates %1 { level = 0 : index } : tensor<2x3x4xf64, #Tensor1> to memref - %v11 = sparse_tensor.coordinates %1 { level = 1 : index } : tensor<2x3x4xf64, #Tensor1> to memref - %v12 = sparse_tensor.coordinates %1 { level = 2 : index } : tensor<2x3x4xf64, #Tensor1> to memref - %v20 = sparse_tensor.coordinates %2 { level = 0 : index } : tensor<2x3x4xf64, #Tensor2> to memref - %v21 = sparse_tensor.coordinates %2 { level = 1 : index } : tensor<2x3x4xf64, #Tensor2> to memref - %v22 = sparse_tensor.coordinates %2 { level = 2 : index } : tensor<2x3x4xf64, #Tensor2> to memref - %v30 = sparse_tensor.coordinates %3 { level = 0 : index } : tensor<2x3x4xf64, #Tensor3> to memref - %v31 = sparse_tensor.coordinates %3 { level = 1 : index } : tensor<2x3x4xf64, #Tensor3> to memref - %v32 = sparse_tensor.coordinates %3 { level = 2 : index } : tensor<2x3x4xf64, #Tensor3> to memref - - %a10 = sparse_tensor.coordinates %a { level = 0 : index } : tensor<2x3x4xf64, #Tensor1> to memref - %a11 = sparse_tensor.coordinates %a { level = 1 : index } : tensor<2x3x4xf64, #Tensor1> to memref - %a12 = sparse_tensor.coordinates %a { level = 2 : index } : tensor<2x3x4xf64, #Tensor1> to memref - %b10 = sparse_tensor.coordinates %b { level = 0 : index } : tensor<2x3x4xf64, #Tensor1> to memref - %b11 = sparse_tensor.coordinates %b { level = 1 : index } : tensor<2x3x4xf64, #Tensor1> to memref - %b12 = sparse_tensor.coordinates %b { level = 2 : index } : tensor<2x3x4xf64, #Tensor1> to memref - %c10 = sparse_tensor.coordinates %c { level = 0 : index } : tensor<2x3x4xf64, #Tensor1> to memref - %c11 = sparse_tensor.coordinates %c { level = 1 : index } : tensor<2x3x4xf64, #Tensor1> to memref - %c12 = sparse_tensor.coordinates %c { level = 2 : index } : tensor<2x3x4xf64, #Tensor1> to memref - - %d20 = sparse_tensor.coordinates %d { level = 0 : index } : tensor<2x3x4xf64, #Tensor2> to memref - %d21 = sparse_tensor.coordinates %d { level = 1 : index } : tensor<2x3x4xf64, #Tensor2> to memref - %d22 = sparse_tensor.coordinates %d { level = 2 : index } : tensor<2x3x4xf64, #Tensor2> to memref - %e20 = sparse_tensor.coordinates %e { level = 0 : index } : tensor<2x3x4xf64, #Tensor2> to memref - %e21 = sparse_tensor.coordinates %e { level = 1 : index } : tensor<2x3x4xf64, #Tensor2> to memref - %e22 = sparse_tensor.coordinates %e { level = 2 : index } : tensor<2x3x4xf64, #Tensor2> to memref - %f20 = sparse_tensor.coordinates %f { level = 0 : index } : tensor<2x3x4xf64, #Tensor2> to memref - %f21 = sparse_tensor.coordinates %f { level = 1 : index } : tensor<2x3x4xf64, #Tensor2> to memref - %f22 = sparse_tensor.coordinates %f { level = 2 : index } : tensor<2x3x4xf64, #Tensor2> to memref - - %g30 = sparse_tensor.coordinates %g { level = 0 : index } : tensor<2x3x4xf64, #Tensor3> to memref - %g31 = sparse_tensor.coordinates %g { level = 1 : index } : tensor<2x3x4xf64, #Tensor3> to memref - %g32 = sparse_tensor.coordinates %g { level = 2 : index } : tensor<2x3x4xf64, #Tensor3> to memref - %h30 = sparse_tensor.coordinates %h { level = 0 : index } : tensor<2x3x4xf64, #Tensor3> to memref - %h31 = sparse_tensor.coordinates %h { level = 1 : index } : tensor<2x3x4xf64, #Tensor3> to memref - %h32 = sparse_tensor.coordinates %h { level = 2 : index } : tensor<2x3x4xf64, #Tensor3> to memref - %i30 = sparse_tensor.coordinates %i { level = 0 : index } : tensor<2x3x4xf64, #Tensor3> to memref - %i31 = sparse_tensor.coordinates %i { level = 1 : index } : tensor<2x3x4xf64, #Tensor3> to memref - %i32 = sparse_tensor.coordinates %i { level = 2 : index } : tensor<2x3x4xf64, #Tensor3> to memref - - call @dumpidx(%v10) : (memref) -> () - call @dumpidx(%v11) : (memref) -> () - call @dumpidx(%v12) : (memref) -> () - call @dumpidx(%v20) : (memref) -> () - call @dumpidx(%v21) : (memref) -> () - call @dumpidx(%v22) : (memref) -> () - call @dumpidx(%v30) : (memref) -> () - call @dumpidx(%v31) : (memref) -> () - call @dumpidx(%v32) : (memref) -> () - - call @dumpidx(%a10) : (memref) -> () - call @dumpidx(%a11) : (memref) -> () - call @dumpidx(%a12) : (memref) -> () - call @dumpidx(%b10) : (memref) -> () - call @dumpidx(%b11) : (memref) -> () - call @dumpidx(%b12) : (memref) -> () - call @dumpidx(%c10) : (memref) -> () - call @dumpidx(%c11) : (memref) -> () - call @dumpidx(%c12) : (memref) -> () - - call @dumpidx(%d20) : (memref) -> () - call @dumpidx(%d21) : (memref) -> () - call @dumpidx(%d22) : (memref) -> () - call @dumpidx(%e20) : (memref) -> () - call @dumpidx(%e21) : (memref) -> () - call @dumpidx(%e22) : (memref) -> () - call @dumpidx(%f20) : (memref) -> () - call @dumpidx(%f21) : (memref) -> () - call @dumpidx(%f22) : (memref) -> () - - call @dumpidx(%g30) : (memref) -> () - call @dumpidx(%g31) : (memref) -> () - call @dumpidx(%g32) : (memref) -> () - call @dumpidx(%h30) : (memref) -> () - call @dumpidx(%h31) : (memref) -> () - call @dumpidx(%h32) : (memref) -> () - call @dumpidx(%i30) : (memref) -> () - call @dumpidx(%i31) : (memref) -> () - call @dumpidx(%i32) : (memref) -> () + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 3, 4, 2 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: values : ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 3, 4, 2 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: values : ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 3, 4, 2 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: values : ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 4, 2, 3 ) + // CHECK-NEXT: pos[0] : ( 0, 4 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8 + // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9, 12, 15, 18, 21, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: values : ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 4, 2, 3 ) + // CHECK-NEXT: pos[0] : ( 0, 4 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8 + // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9, 12, 15, 18, 21, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: values : ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 4, 2, 3 ) + // CHECK-NEXT: pos[0] : ( 0, 4 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8 + // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9, 12, 15, 18, 21, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: values : ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24 + // CHECK-NEXT: ---- + // + sparse_tensor.print %1 : tensor<2x3x4xf64, #Tensor1> + sparse_tensor.print %2 : tensor<2x3x4xf64, #Tensor2> + sparse_tensor.print %3 : tensor<2x3x4xf64, #Tensor3> + sparse_tensor.print %a : tensor<2x3x4xf64, #Tensor1> + sparse_tensor.print %b : tensor<2x3x4xf64, #Tensor1> + sparse_tensor.print %c : tensor<2x3x4xf64, #Tensor1> + sparse_tensor.print %d : tensor<2x3x4xf64, #Tensor2> + sparse_tensor.print %e : tensor<2x3x4xf64, #Tensor2> + sparse_tensor.print %f : tensor<2x3x4xf64, #Tensor2> + sparse_tensor.print %g : tensor<2x3x4xf64, #Tensor3> + sparse_tensor.print %h : tensor<2x3x4xf64, #Tensor3> + sparse_tensor.print %i : tensor<2x3x4xf64, #Tensor3> // Release the resources. bufferization.dealloc_tensor %1 : tensor<2x3x4xf64, #Tensor1> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir index 809414ba977d..ff22283f43a7 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -52,21 +52,10 @@ // Integration test that tests conversions between sparse tensors. // module { - // - // Output utilities. - // - func.func @dumpf64(%arg0: memref) { - %c0 = arith.constant 0 : index - %d0 = arith.constant -1.0 : f64 - %0 = vector.transfer_read %arg0[%c0], %d0: memref, vector<8xf64> - vector.print %0 : vector<8xf64> - return - } - // // Main driver. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index %c2 = arith.constant 2 : index @@ -88,20 +77,47 @@ module { %3 = sparse_tensor.convert %1 : tensor<2x4xf64, #BSR> to tensor<2x4xf64, #CSR> %4 = sparse_tensor.convert %1 : tensor<2x4xf64, #BSR> to tensor<2x4xf64, #CSC> - %v1 = sparse_tensor.values %1 : tensor<2x4xf64, #BSR> to memref - %v2 = sparse_tensor.values %2 : tensor<2x4xf64, #BSR> to memref - %v3 = sparse_tensor.values %3 : tensor<2x4xf64, #CSR> to memref - %v4 = sparse_tensor.values %4 : tensor<2x4xf64, #CSC> to memref - - - // CHECK: ( 1, 2, 5, 6, 3, 4, 7, 8 ) - // CHECK-NEXT: ( 1, 2, 5, 6, 3, 4, 7, 8 ) - // CHECK-NEXT: ( 1, 2, 3, 4, 5, 6, 7, 8 ) - // CHECK-NEXT: ( 1, 5, 2, 6, 3, 7, 4, 8 ) - call @dumpf64(%v1) : (memref) -> () - call @dumpf64(%v2) : (memref) -> () - call @dumpf64(%v3) : (memref) -> () - call @dumpf64(%v4) : (memref) -> () + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 8 + // CHECK-NEXT: dim = ( 2, 4 ) + // CHECK-NEXT: lvl = ( 1, 2, 2, 2 ) + // CHECK-NEXT: pos[1] : ( 0, 2 + // CHECK-NEXT: crd[1] : ( 0, 1 + // CHECK-NEXT: values : ( 1, 2, 5, 6, 3, 4, 7, 8 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 8 + // CHECK-NEXT: dim = ( 2, 4 ) + // CHECK-NEXT: lvl = ( 1, 2, 2, 2 ) + // CHECK-NEXT: pos[1] : ( 0, 2 + // CHECK-NEXT: crd[1] : ( 0, 1 + // CHECK-NEXT: values : ( 1, 2, 5, 6, 3, 4, 7, 8 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 8 + // CHECK-NEXT: dim = ( 2, 4 ) + // CHECK-NEXT: lvl = ( 2, 4 ) + // CHECK-NEXT: pos[1] : ( 0, 4, 8 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 8 + // CHECK-NEXT: dim = ( 2, 4 ) + // CHECK-NEXT: lvl = ( 4, 2 ) + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8 + // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: values : ( 1, 5, 2, 6, 3, 7, 4, 8 + // CHECK-NEXT: ---- + // + sparse_tensor.print %1 : tensor<2x4xf64, #BSR> + sparse_tensor.print %2 : tensor<2x4xf64, #BSR> + sparse_tensor.print %3 : tensor<2x4xf64, #CSR> + sparse_tensor.print %4 : tensor<2x4xf64, #CSC> // TODO: Fix memory leaks. bufferization.dealloc_tensor %1 : tensor<2x4xf64, #BSR> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir index f658457fa673..11baf65e6350 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -44,19 +44,7 @@ // may change (the actual underlying sizes obviously never change). // module { - - func.func private @printMemref1dF64(%ptr : memref) attributes { llvm.emit_c_interface } - - // - // Helper method to print values array. The transfer actually - // reads more than required to verify size of buffer as well. - // - func.func @dump(%arg0: memref) { - call @printMemref1dF64(%arg0) : (memref) -> () - return - } - - func.func @entry() { + func.func @main() { %t1 = arith.constant sparse< [ [0,0], [0,1], [0,63], [1,0], [1,1], [31,0], [31,63] ], [ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0 ]> : tensor<32x64xf64> @@ -72,45 +60,81 @@ module { %5 = sparse_tensor.convert %3 : tensor to tensor %6 = sparse_tensor.convert %4 : tensor to tensor -// - // Check number_of_entries. // - // CHECK-COUNT-6: 7 - %n1 = sparse_tensor.number_of_entries %1 : tensor - %n2 = sparse_tensor.number_of_entries %2 : tensor - %n3 = sparse_tensor.number_of_entries %3 : tensor - %n4 = sparse_tensor.number_of_entries %4 : tensor - %n5 = sparse_tensor.number_of_entries %5 : tensor - %n6 = sparse_tensor.number_of_entries %6 : tensor - vector.print %n1 : index - vector.print %n2 : index - vector.print %n3 : index - vector.print %n4 : index - vector.print %n5 : index - vector.print %n6 : index - + // Verify the outputs. + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 7 + // CHECK-NEXT: dim = ( 32, 64 ) + // CHECK-NEXT: lvl = ( 32, 64 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 31 + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 + // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 7 + // CHECK-NEXT: dim = ( 32, 64 ) + // CHECK-NEXT: lvl = ( 64, 32 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 63 + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 + // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31 + // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 7 + // CHECK-NEXT: dim = ( 32, 64 ) + // CHECK-NEXT: lvl = ( 32, 64 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 31 + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 + // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 7 + // CHECK-NEXT: dim = ( 32, 64 ) + // CHECK-NEXT: lvl = ( 64, 32 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 63 + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 + // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31 + // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7 + // CHECK-NEXT: ---- // - // All proper row-/column-wise? + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 7 + // CHECK-NEXT: dim = ( 32, 64 ) + // CHECK-NEXT: lvl = ( 64, 32 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 63 + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 + // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31 + // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7 + // CHECK-NEXT: ---- // - // CHECK: [1, 2, 3, 4, 5, 6, 7 - // CHECK: [1, 4, 6, 2, 5, 3, 7 - // CHECK: [1, 2, 3, 4, 5, 6, 7 - // CHECK: [1, 4, 6, 2, 5, 3, 7 - // CHECK: [1, 4, 6, 2, 5, 3, 7 - // CHECK: [1, 2, 3, 4, 5, 6, 7 + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 7 + // CHECK-NEXT: dim = ( 32, 64 ) + // CHECK-NEXT: lvl = ( 32, 64 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 31 + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 + // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 + // CHECK-NEXT: ---- // - %m1 = sparse_tensor.values %1 : tensor to memref - %m2 = sparse_tensor.values %2 : tensor to memref - %m3 = sparse_tensor.values %3 : tensor to memref - %m4 = sparse_tensor.values %4 : tensor to memref - %m5 = sparse_tensor.values %5 : tensor to memref - %m6 = sparse_tensor.values %6 : tensor to memref - call @dump(%m1) : (memref) -> () - call @dump(%m2) : (memref) -> () - call @dump(%m3) : (memref) -> () - call @dump(%m4) : (memref) -> () - call @dump(%m5) : (memref) -> () - call @dump(%m6) : (memref) -> () + sparse_tensor.print %1 : tensor + sparse_tensor.print %2 : tensor + sparse_tensor.print %3 : tensor + sparse_tensor.print %4 : tensor + sparse_tensor.print %5 : tensor + sparse_tensor.print %6 : tensor // Release the resources. bufferization.dealloc_tensor %1 : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir index 81f68366be28..a2ec6df392aa 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -55,7 +55,7 @@ module { // // The first test suite (for non-singleton LevelTypes). // - func.func @entry() { + func.func @main() { // // Initialize a 3-dim dense tensor. // diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir index 3ebe3be757d2..6005aa6cfeae 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -54,41 +54,7 @@ // in addition to layout. // module { - - // - // Helper method to print values and indices arrays. The transfer actually - // reads more than required to verify size of buffer as well. - // - func.func @dumpf64(%arg0: memref) { - %c = arith.constant 0 : index - %d = arith.constant 0.0 : f64 - %0 = vector.transfer_read %arg0[%c], %d: memref, vector<8xf64> - vector.print %0 : vector<8xf64> - return - } - func.func @dumpi08(%arg0: memref) { - %c = arith.constant 0 : index - %d = arith.constant 0 : i8 - %0 = vector.transfer_read %arg0[%c], %d: memref, vector<8xi8> - vector.print %0 : vector<8xi8> - return - } - func.func @dumpi32(%arg0: memref) { - %c = arith.constant 0 : index - %d = arith.constant 0 : i32 - %0 = vector.transfer_read %arg0[%c], %d: memref, vector<8xi32> - vector.print %0 : vector<8xi32> - return - } - func.func @dumpi64(%arg0: memref) { - %c = arith.constant 0 : index - %d = arith.constant 0 : i64 - %0 = vector.transfer_read %arg0[%c], %d: memref, vector<8xi64> - vector.print %0 : vector<8xi64> - return - } - - func.func @entry() { + func.func @main() { %c1 = arith.constant 1 : index %t1 = arith.constant sparse< [ [0,0], [0,1], [0,63], [1,0], [1,1], [31,0], [31,63] ], @@ -106,50 +72,78 @@ module { %6 = sparse_tensor.convert %3 : tensor<32x64xf64, #CSC> to tensor<32x64xf64, #DCSR> // - // All proper row-/column-wise? + // Verify the outputs. // - // CHECK: ( 1, 2, 3, 4, 5, 6, 7, 0 ) - // CHECK-NEXT: ( 1, 4, 6, 2, 5, 3, 7, 0 ) - // CHECK-NEXT: ( 1, 4, 6, 2, 5, 3, 7, 0 ) - // CHECK-NEXT: ( 1, 4, 6, 2, 5, 3, 7, 0 ) - // CHECK-NEXT: ( 1, 2, 3, 4, 5, 6, 7, 0 ) - // CHECK-NEXT: ( 1, 2, 3, 4, 5, 6, 7, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 7 + // CHECK-NEXT: dim = ( 32, 64 ) + // CHECK-NEXT: lvl = ( 32, 64 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 31 + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 + // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 + // CHECK-NEXT: ---- // - %m1 = sparse_tensor.values %1 : tensor<32x64xf64, #DCSR> to memref - %m2 = sparse_tensor.values %2 : tensor<32x64xf64, #DCSC> to memref - %m3 = sparse_tensor.values %3 : tensor<32x64xf64, #CSC> to memref - %m4 = sparse_tensor.values %4 : tensor<32x64xf64, #DCSC> to memref - %m5 = sparse_tensor.values %5 : tensor<32x64xf64, #DCSR> to memref - %m6 = sparse_tensor.values %6 : tensor<32x64xf64, #DCSR> to memref - call @dumpf64(%m1) : (memref) -> () - call @dumpf64(%m2) : (memref) -> () - call @dumpf64(%m3) : (memref) -> () - call @dumpf64(%m4) : (memref) -> () - call @dumpf64(%m5) : (memref) -> () - call @dumpf64(%m6) : (memref) -> () - + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 7 + // CHECK-NEXT: dim = ( 32, 64 ) + // CHECK-NEXT: lvl = ( 64, 32 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 63 + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 + // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31 + // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 7 + // CHECK-NEXT: dim = ( 32, 64 ) + // CHECK-NEXT: lvl = ( 64, 32 ) + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 7 + // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31 + // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 7 + // CHECK-NEXT: dim = ( 32, 64 ) + // CHECK-NEXT: lvl = ( 64, 32 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 63 + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 + // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31 + // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7 + // CHECK-NEXT: ---- // - // Sanity check on indices. + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 7 + // CHECK-NEXT: dim = ( 32, 64 ) + // CHECK-NEXT: lvl = ( 32, 64 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 31 + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 + // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 + // CHECK-NEXT: ---- // - // CHECK-NEXT: ( 0, 1, 63, 0, 1, 0, 63, 0 ) - // CHECK-NEXT: ( 0, 1, 31, 0, 1, 0, 31, 0 ) - // CHECK-NEXT: ( 0, 1, 31, 0, 1, 0, 31, 0 ) - // CHECK-NEXT: ( 0, 1, 31, 0, 1, 0, 31, 0 ) - // CHECK-NEXT: ( 0, 1, 63, 0, 1, 0, 63, 0 ) - // CHECK-NEXT: ( 0, 1, 63, 0, 1, 0, 63, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 7 + // CHECK-NEXT: dim = ( 32, 64 ) + // CHECK-NEXT: lvl = ( 32, 64 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 31 + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 + // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 + // CHECK-NEXT: ---- // - %i1 = sparse_tensor.coordinates %1 { level = 1 : index } : tensor<32x64xf64, #DCSR> to memref - %i2 = sparse_tensor.coordinates %2 { level = 1 : index } : tensor<32x64xf64, #DCSC> to memref - %i3 = sparse_tensor.coordinates %3 { level = 1 : index } : tensor<32x64xf64, #CSC> to memref - %i4 = sparse_tensor.coordinates %4 { level = 1 : index } : tensor<32x64xf64, #DCSC> to memref - %i5 = sparse_tensor.coordinates %5 { level = 1 : index } : tensor<32x64xf64, #DCSR> to memref - %i6 = sparse_tensor.coordinates %6 { level = 1 : index } : tensor<32x64xf64, #DCSR> to memref - call @dumpi08(%i1) : (memref) -> () - call @dumpi64(%i2) : (memref) -> () - call @dumpi32(%i3) : (memref) -> () - call @dumpi64(%i4) : (memref) -> () - call @dumpi08(%i5) : (memref) -> () - call @dumpi08(%i6) : (memref) -> () + sparse_tensor.print %1 : tensor<32x64xf64, #DCSR> + sparse_tensor.print %2 : tensor<32x64xf64, #DCSC> + sparse_tensor.print %3 : tensor<32x64xf64, #CSC> + sparse_tensor.print %4 : tensor<32x64xf64, #DCSC> + sparse_tensor.print %5 : tensor<32x64xf64, #DCSR> + sparse_tensor.print %6 : tensor<32x64xf64, #DCSR> // Release the resources. bufferization.dealloc_tensor %1 : tensor<32x64xf64, #DCSR> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir index 1655d6a03a62..9b05f9bf3a29 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -107,7 +107,7 @@ module { // // Main driver. // - func.func @entry() { + func.func @main() { // // Initialize a 3-dim dense tensor. // diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir index 2ace317554a0..0f9dfb9da720 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -180,7 +180,7 @@ module { // // Main driver. // - func.func @entry() { + func.func @main() { call @testNonSingleton() : () -> () call @testSingleton() : () -> () return diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir index 16252c1005eb..16813e0aa707 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -93,17 +93,17 @@ module { func.func @add_coo_coo_out_coo(%arga: tensor<8x8xf32, #SortedCOO>, %argb: tensor<8x8xf32, #SortedCOOSoA>) - -> tensor<8x8xf32, #SortedCOO> { - %init = tensor.empty() : tensor<8x8xf32, #SortedCOO> + -> tensor<8x8xf32, #SortedCOOSoA> { + %init = tensor.empty() : tensor<8x8xf32, #SortedCOOSoA> %0 = linalg.generic #trait ins(%arga, %argb: tensor<8x8xf32, #SortedCOO>, tensor<8x8xf32, #SortedCOOSoA>) - outs(%init: tensor<8x8xf32, #SortedCOO>) { + outs(%init: tensor<8x8xf32, #SortedCOOSoA>) { ^bb(%a: f32, %b: f32, %x: f32): %0 = arith.addf %a, %b : f32 linalg.yield %0 : f32 - } -> tensor<8x8xf32, #SortedCOO> - return %0 : tensor<8x8xf32, #SortedCOO> + } -> tensor<8x8xf32, #SortedCOOSoA> + return %0 : tensor<8x8xf32, #SortedCOOSoA> } @@ -126,7 +126,7 @@ module { return %0 : tensor<8x8xf32> } - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index %c8 = arith.constant 8 : index @@ -171,8 +171,9 @@ module { -> tensor<8x8xf32> %COO_RET = call @add_coo_coo_out_coo(%COO_A, %COO_B) : (tensor<8x8xf32, #SortedCOO>, tensor<8x8xf32, #SortedCOOSoA>) - -> tensor<8x8xf32, #SortedCOO> - %C4 = sparse_tensor.convert %COO_RET : tensor<8x8xf32, #SortedCOO> to tensor<8x8xf32> + -> tensor<8x8xf32, #SortedCOOSoA> + %C4 = sparse_tensor.convert %COO_RET : tensor<8x8xf32, #SortedCOOSoA> to tensor<8x8xf32> + // // Verify computed matrix C. // @@ -201,6 +202,32 @@ module { vector.print %v4 : vector<8xf32> } + // + // Ensure that COO-SoA output has the same values. + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 64 + // CHECK-NEXT: dim = ( 8, 8 ) + // CHECK-NEXT: lvl = ( 8, 8 ) + // CHECK-NEXT: pos[0] : ( 0, 64 + // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, + // CHECK-SAME: 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, + // CHECK-SAME: 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, + // CHECK-SAME: 7, 7, 7, 7 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, + // CHECK-SAME: 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, + // CHECK-SAME: 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, + // CHECK-SAME: 4, 5, 6, 7 + // CHECK-NEXT: values : ( 8.8, 4.8, 6.8, 4.8, 8.8, 6.1, 14.8, 16.8, 4.4, 4.4, 4.4, 8.4, + // CHECK-SAME: 8.4, 12.4, 16.4, 16.4, 8.8, 4.8, 6.8, 8.8, 8.8, 12.8, 14.8, + // CHECK-SAME: 15.8, 4.3, 5.3, 6.3, 8.3, 8.3, 12.3, 14.3, 16.3, 4.5, 4.5, + // CHECK-SAME: 6.5, 8.5, 8.5, 12.5, 14.5, 16.5, 9.9, 4.9, 6.9, 8.9, 8.9, + // CHECK-SAME: 12.9, 15.9, 16.9, 12.1, 6.1, 5.1, 9.1, 9.1, 13.1, 15.1, 17.1, + // CHECK-SAME: 15.4, 5.4, 7.4, 5.4, 11.4, 10.4, 11.4, 9.4 + // CHECK-NEXT: ---- + // + sparse_tensor.print %COO_RET : tensor<8x8xf32, #SortedCOOSoA> + // Release resources. bufferization.dealloc_tensor %C1 : tensor<8x8xf32> bufferization.dealloc_tensor %C2 : tensor<8x8xf32> @@ -209,7 +236,7 @@ module { bufferization.dealloc_tensor %CSR_A : tensor<8x8xf32, #CSR> bufferization.dealloc_tensor %COO_A : tensor<8x8xf32, #SortedCOO> bufferization.dealloc_tensor %COO_B : tensor<8x8xf32, #SortedCOOSoA> - bufferization.dealloc_tensor %COO_RET : tensor<8x8xf32, #SortedCOO> + bufferization.dealloc_tensor %COO_RET : tensor<8x8xf32, #SortedCOOSoA> return diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir index b4d40ae08401..40738a9f7d7f 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -78,7 +78,7 @@ func.func @conv_2d_nhwc_hwcf_dual_CDCC(%arg0: tensor, %arg1: } -func.func @entry() { +func.func @main() { %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index %c3 = arith.constant 3 : index diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir index f7ba4daa2458..5451f2d957ad 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -49,7 +49,7 @@ module { // // Main driver. // - func.func @entry() { + func.func @main() { // Setup two sparse vectors. %d1 = arith.constant sparse< [ [0], [1], [22], [23], [1022] ], [1.0, 2.0, 3.0, 4.0, 5.0] @@ -60,6 +60,30 @@ module { %s1 = sparse_tensor.convert %d1 : tensor<1024xf32> to tensor<1024xf32, #SparseVector> %s2 = sparse_tensor.convert %d2 : tensor<1024xf32> to tensor<1024xf32, #SparseVector> + // + // Verify the inputs. + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 1024 ) + // CHECK-NEXT: lvl = ( 1024 ) + // CHECK-NEXT: pos[0] : ( 0, 5 + // CHECK-NEXT: crd[0] : ( 0, 1, 22, 23, 1022 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 3 + // CHECK-NEXT: dim = ( 1024 ) + // CHECK-NEXT: lvl = ( 1024 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 22, 1022, 1023 + // CHECK-NEXT: values : ( 6, 7, 8 + // CHECK-NEXT: ---- + // + sparse_tensor.print %s1 : tensor<1024xf32, #SparseVector> + sparse_tensor.print %s2 : tensor<1024xf32, #SparseVector> + // Call the kernel and verify the output. // // CHECK: 53 @@ -73,16 +97,6 @@ module { %1 = tensor.extract %0[] : tensor vector.print %1 : f32 - // Print number of entries in the sparse vectors. - // - // CHECK: 5 - // CHECK: 3 - // - %noe1 = sparse_tensor.number_of_entries %s1 : tensor<1024xf32, #SparseVector> - %noe2 = sparse_tensor.number_of_entries %s2 : tensor<1024xf32, #SparseVector> - vector.print %noe1 : index - vector.print %noe2 : index - // Release the resources. bufferization.dealloc_tensor %0 : tensor bufferization.dealloc_tensor %s1 : tensor<1024xf32, #SparseVector> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir index 93a7ea51ec9c..451195b2185b 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -35,8 +35,6 @@ }> module { - func.func private @printMemrefF64(%ptr : tensor<*xf64>) - // // Column-wise storage forces the ijk loop to permute into jki // so that access pattern expansion (workspace) needs to be @@ -54,7 +52,7 @@ module { // // Main driver. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %d1 = arith.constant -1.0 : f64 @@ -83,24 +81,26 @@ module { : (tensor<8x2xf64, #CSC>, tensor<2x4xf64, #CSC>) -> tensor<8x4xf64, #CSC> - // CHECK: {{\[}}[32.53, 35.73, 38.93, 42.13], - // CHECK-NEXT: [34.56, 37.96, 41.36, 44.76], - // CHECK-NEXT: [36.59, 40.19, 43.79, 47.39], - // CHECK-NEXT: [38.62, 42.42, 46.22, 50.02], - // CHECK-NEXT: [40.65, 44.65, 48.65, 52.65], - // CHECK-NEXT: [42.68, 46.88, 51.08, 55.28], - // CHECK-NEXT: [44.71, 49.11, 53.51, 57.91], - // CHECK-NEXT: [46.74, 51.34, 55.94, 60.54]] // - %xc = sparse_tensor.convert %x3 : tensor<8x4xf64, #CSC> to tensor<8x4xf64> - %xu = tensor.cast %xc : tensor<8x4xf64> to tensor<*xf64> - call @printMemrefF64(%xu) : (tensor<*xf64>) -> () + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 32 + // CHECK-NEXT: dim = ( 8, 4 ) + // CHECK-NEXT: lvl = ( 4, 8 ) + // CHECK-NEXT: pos[1] : ( 0, 8, 16, 24, 32 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, + // CHECK-SAME: 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 + // CHECK-NEXT: values : ( 32.53, 34.56, 36.59, 38.62, 40.65, 42.68, 44.71, 46.74, + // CHECK-SAME: 35.73, 37.96, 40.19, 42.42, 44.65, 46.88, 49.11, 51.34, + // CHECK-SAME: 38.93, 41.36, 43.79, 46.22, 48.65, 51.08, 53.51, 55.94, + // CHECK-SAME: 42.13, 44.76, 47.39, 50.02, 52.65, 55.28, 57.91, 60.54 + // CHECK-NEXT: ---- + // + sparse_tensor.print %x3 : tensor<8x4xf64, #CSC> // Release the resources. bufferization.dealloc_tensor %x1 : tensor<8x2xf64, #CSC> bufferization.dealloc_tensor %x2 : tensor<2x4xf64, #CSC> bufferization.dealloc_tensor %x3 : tensor<8x4xf64, #CSC> - bufferization.dealloc_tensor %xc : tensor<8x4xf64> return } diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir index c784375e0f3e..6679a81c7408 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -115,7 +115,7 @@ module { // // Main driver. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %df = arith.constant -1.0 : f64 @@ -147,60 +147,111 @@ module { %expand11 = call @expand_sparse2sparse_dyn(%sdm) : (tensor) -> tensor // - // Verify results of expand + // Verify results of expand with dense output. // // CHECK: ( ( 1, 0, 3, 0 ), ( 5, 0, 7, 0 ), ( 9, 0, 11, 0 ) ) // CHECK-NEXT: ( ( 1, 0, 3, 0 ), ( 5, 0, 7, 0 ), ( 9, 0, 11, 0 ) ) - // CHECK-NEXT: ( 1, 3, 5, 7, 9, - // CHECK-NEXT: ( 1, 3, 5, 7, 9, // CHECK-NEXT: ( ( ( 1.1, 1.2 ), ( 1.3, 1.4 ) ), ( ( 2.1, 2.2 ), ( 2.3, 2.4 ) ), ( ( 3.1, 3.2 ), ( 3.3, 3.4 ) ) ) // CHECK-NEXT: ( ( ( 1.1, 1.2 ), ( 1.3, 1.4 ) ), ( ( 2.1, 2.2 ), ( 2.3, 2.4 ) ), ( ( 3.1, 3.2 ), ( 3.3, 3.4 ) ) ) - // CHECK-NEXT: ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4 ) - // CHECK-NEXT: ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4 ) // CHECK-NEXT: ( ( ( 1.1, 1.2 ), ( 1.3, 1.4 ) ), ( ( 2.1, 2.2 ), ( 2.3, 2.4 ) ), ( ( 3.1, 3.2 ), ( 3.3, 3.4 ) ) ) // CHECK-NEXT: ( ( ( 1.1, 1.2 ), ( 1.3, 1.4 ) ), ( ( 2.1, 2.2 ), ( 2.3, 2.4 ) ), ( ( 3.1, 3.2 ), ( 3.3, 3.4 ) ) ) - // CHECK-NEXT: 12 - // CHECK-NEXT: ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4 ) - // CHECK-NEXT: 12 - // CHECK-NEXT: ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4 ) // - %m0 = vector.transfer_read %expand0[%c0, %c0], %df: tensor<3x4xf64>, vector<3x4xf64> vector.print %m0 : vector<3x4xf64> %m1 = vector.transfer_read %expand1[%c0, %c0], %df: tensor<3x4xf64>, vector<3x4xf64> vector.print %m1 : vector<3x4xf64> - %a2 = sparse_tensor.values %expand2 : tensor<3x4xf64, #SparseMatrix> to memref - %m2 = vector.transfer_read %a2[%c0], %df: memref, vector<12xf64> - vector.print %m2 : vector<12xf64> - %a3 = sparse_tensor.values %expand3 : tensor<3x4xf64, #SparseMatrix> to memref - %m3 = vector.transfer_read %a3[%c0], %df: memref, vector<12xf64> - vector.print %m3 : vector<12xf64> - %m4 = vector.transfer_read %expand4[%c0, %c0, %c0], %df: tensor<3x2x2xf64>, vector<3x2x2xf64> vector.print %m4 : vector<3x2x2xf64> %m5 = vector.transfer_read %expand5[%c0, %c0, %c0], %df: tensor<3x2x2xf64>, vector<3x2x2xf64> vector.print %m5 : vector<3x2x2xf64> - %a6 = sparse_tensor.values %expand6 : tensor<3x2x2xf64, #Sparse3dTensor> to memref - %m6 = vector.transfer_read %a6[%c0], %df: memref, vector<12xf64> - vector.print %m6 : vector<12xf64> - %a7 = sparse_tensor.values %expand7 : tensor<3x2x2xf64, #Sparse3dTensor> to memref - %m7 = vector.transfer_read %a7[%c0], %df: memref, vector<12xf64> - vector.print %m7 : vector<12xf64> - %m8 = vector.transfer_read %expand8[%c0, %c0, %c0], %df: tensor, vector<3x2x2xf64> vector.print %m8 : vector<3x2x2xf64> %m9 = vector.transfer_read %expand9[%c0, %c0, %c0], %df: tensor, vector<3x2x2xf64> vector.print %m9 : vector<3x2x2xf64> - %n10 = sparse_tensor.number_of_entries %expand10 : tensor - vector.print %n10 : index - %a10 = sparse_tensor.values %expand10 : tensor to memref - %m10 = vector.transfer_read %a10[%c0], %df: memref, vector<12xf64> - vector.print %m10 : vector<12xf64> - %n11 = sparse_tensor.number_of_entries %expand11 : tensor - vector.print %n11 : index - %a11 = sparse_tensor.values %expand11 : tensor to memref - %m11 = vector.transfer_read %a11[%c0], %df: memref, vector<12xf64> - vector.print %m11 : vector<12xf64> + + // + // Verify results of expand with sparse output. + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 3, 4 ) + // CHECK-NEXT: lvl = ( 3, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 + // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, 0, 2 + // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 11 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 3, 4 ) + // CHECK-NEXT: lvl = ( 3, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 + // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, 0, 2 + // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 11 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 3, 2, 2 ) + // CHECK-NEXT: lvl = ( 3, 2, 2 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 + // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12 + // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: values : ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 3, 2, 2 ) + // CHECK-NEXT: lvl = ( 3, 2, 2 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 + // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12 + // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: values : ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 3, 2, 2 ) + // CHECK-NEXT: lvl = ( 3, 2, 2 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 + // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12 + // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: values : ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 3, 2, 2 ) + // CHECK-NEXT: lvl = ( 3, 2, 2 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 + // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12 + // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + // CHECK-NEXT: values : ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4 + // CHECK-NEXT: ---- + // + sparse_tensor.print %expand2 : tensor<3x4xf64, #SparseMatrix> + sparse_tensor.print %expand3 : tensor<3x4xf64, #SparseMatrix> + sparse_tensor.print %expand6 : tensor<3x2x2xf64, #Sparse3dTensor> + sparse_tensor.print %expand7 : tensor<3x2x2xf64, #Sparse3dTensor> + sparse_tensor.print %expand10 : tensor + sparse_tensor.print %expand11 : tensor // Release sparse resources. diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir index 7478b604ff67..37ff2e3ffd3f 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -53,7 +53,7 @@ module { return %0 : tensor<6x6xi32, #DCSR> } - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %i0 = arith.constant 0 : i32 @@ -100,25 +100,27 @@ module { vector.print %v : vector<6x6xi32> // - // Should be the same as dense output - // CHECK: ( ( 0, 0, -1, -6, -1, 6 ), - // CHECK-SAME: ( -1, 0, 1, 0, 1, 0 ), - // CHECK-SAME: ( 0, -1, 1, 0, 0, 0 ), - // CHECK-SAME: ( -1, 0, 0, 0, 0, 0 ), - // CHECK-SAME: ( 0, 0, 3, 6, -3, -6 ), - // CHECK-SAME: ( 2, -1, 3, 0, -3, 0 ) ) + // Should be the same as dense output. // - %sparse_ret = sparse_tensor.convert %1 - : tensor<6x6xi32, #DCSR> to tensor<6x6xi32> - %v1 = vector.transfer_read %sparse_ret[%c0, %c0], %i0 - : tensor<6x6xi32>, vector<6x6xi32> - vector.print %v1 : vector<6x6xi32> + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 36 + // CHECK-NEXT: dim = ( 6, 6 ) + // CHECK-NEXT: lvl = ( 6, 6 ) + // CHECK-NEXT: pos[0] : ( 0, 6 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5 + // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, + // CHECK-SAME: 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 + // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, + // CHECK-SAME: 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0 + // CHECK-NEXT: ---- + // + sparse_tensor.print %1 : tensor<6x6xi32, #DCSR> // Release the resources. bufferization.dealloc_tensor %sparse_filter : tensor<3x3xi32, #DCSR> bufferization.dealloc_tensor %0 : tensor<6x6xi32> bufferization.dealloc_tensor %1 : tensor<6x6xi32, #DCSR> - bufferization.dealloc_tensor %sparse_ret : tensor<6x6xi32> return } diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir index 55ecac8dbc18..8a5712d3fa1b 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -82,7 +82,7 @@ module { // // Main driver that reads tensor from file and calls the sparse kernel. // - func.func @entry() { + func.func @main() { %d0 = arith.constant 0.0 : f64 %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_foreach_slices.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_foreach_slices.mlir index 46b83be21dcf..aef3a947e4f0 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_foreach_slices.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_foreach_slices.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -100,7 +100,7 @@ module { return } - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index %c2 = arith.constant 2 : index diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir index c1547033062d..e1f73eb4ac4f 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -39,7 +39,7 @@ module { // // Main driver. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index %f0 = arith.constant 0.0 : f64 @@ -78,12 +78,20 @@ module { } %sv = sparse_tensor.convert %output : tensor to tensor - %n0 = sparse_tensor.number_of_entries %sv : tensor - // Print the number of non-zeros for verification. // - // CHECK: 5 - vector.print %n0 : index + // Verify the outputs. + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 50 ) + // CHECK-NEXT: lvl = ( 50 ) + // CHECK-NEXT: pos[0] : ( 0, 5 + // CHECK-NEXT: crd[0] : ( 1, 9, 17, 27, 30 + // CHECK-NEXT: values : ( 84, 34, 8, 40, 93 + // CHECK-NEXT: ---- + // + sparse_tensor.print %sv : tensor // Release the resources. bufferization.dealloc_tensor %sv : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir index 70a63ae7bc92..3ce45e5fd971 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -160,7 +160,7 @@ module { // // Main driver. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %du = arith.constant -1 : i64 %df = arith.constant -1.0 : f32 @@ -208,63 +208,112 @@ module { // // Verify result. // - // CHECK: 2 - // CHECK-NEXT: 8 - // CHECK-NEXT: 8 - // CHECK-NEXT: 8 - // CHECK-NEXT: 2 - // CHECK-NEXT: 12 - // CHECK-NEXT: 12 - // CHECK-NEXT: 12 - // CHECK-NEXT: ( 20, 80 ) - // CHECK-NEXT: ( 0, 1, 12, 3, 24, 5, 6, 7 ) - // CHECK-NEXT: ( 0, 2, 8, 24, 64, 160, 384, 896 ) - // CHECK-NEXT: ( 1, 3, 6, 11, 20, 37, 70, 135 ) - // CHECK-NEXT: ( 10, 120 ) - // CHECK-NEXT: ( 0, 1, 2, 3, 1, 12, 3, 4, 2, 3, 4, 25 ) - // CHECK-NEXT: ( 0, 0, 0, 0, 0, 2, 2, 3, 0, 2, 12, 24 ) - // CHECK-NEXT: ( 1, 2, 3, 4, 2, 4, 4, 5, 3, 4, 7, 9 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 2 + // CHECK-NEXT: dim = ( 8 ) + // CHECK-NEXT: lvl = ( 8 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 2, 4 + // CHECK-NEXT: values : ( 20, 80 + // CHECK-NEXT: ---- // - %n0 = sparse_tensor.number_of_entries %0 : tensor<8xi64, #SparseVector> - %n1 = sparse_tensor.number_of_entries %1 : tensor<8xi64, #SparseVector> - %n2 = sparse_tensor.number_of_entries %2 : tensor<8xi64, #SparseVector> - %n3 = sparse_tensor.number_of_entries %3 : tensor<8xi64, #SparseVector> - %n4 = sparse_tensor.number_of_entries %4 : tensor<3x4xi64, #SparseMatrix> - %n5 = sparse_tensor.number_of_entries %5 : tensor<3x4xi64, #SparseMatrix> - %n6 = sparse_tensor.number_of_entries %6 : tensor<3x4xi64, #SparseMatrix> - %n7 = sparse_tensor.number_of_entries %7 : tensor<3x4xi64, #SparseMatrix> - %8 = sparse_tensor.values %0 : tensor<8xi64, #SparseVector> to memref - %9 = sparse_tensor.values %1 : tensor<8xi64, #SparseVector> to memref - %10 = sparse_tensor.values %2 : tensor<8xi64, #SparseVector> to memref - %11 = sparse_tensor.values %3 : tensor<8xi64, #SparseVector> to memref - %12 = sparse_tensor.values %4 : tensor<3x4xi64, #SparseMatrix> to memref - %13 = sparse_tensor.values %5 : tensor<3x4xi64, #SparseMatrix> to memref - %14 = sparse_tensor.values %6 : tensor<3x4xi64, #SparseMatrix> to memref - %15 = sparse_tensor.values %7 : tensor<3x4xi64, #SparseMatrix> to memref - %16 = vector.transfer_read %8[%c0], %du: memref, vector<2xi64> - %17 = vector.transfer_read %9[%c0], %du: memref, vector<8xi64> - %18 = vector.transfer_read %10[%c0], %du: memref, vector<8xi64> - %19 = vector.transfer_read %11[%c0], %du: memref, vector<8xi64> - %20 = vector.transfer_read %12[%c0], %du: memref, vector<2xi64> - %21 = vector.transfer_read %13[%c0], %du: memref, vector<12xi64> - %22 = vector.transfer_read %14[%c0], %du: memref, vector<12xi64> - %23 = vector.transfer_read %15[%c0], %du: memref, vector<12xi64> - vector.print %n0 : index - vector.print %n1 : index - vector.print %n2 : index - vector.print %n3 : index - vector.print %n4 : index - vector.print %n5 : index - vector.print %n6 : index - vector.print %n7 : index - vector.print %16 : vector<2xi64> - vector.print %17 : vector<8xi64> - vector.print %18 : vector<8xi64> - vector.print %19 : vector<8xi64> - vector.print %20 : vector<2xi64> - vector.print %21 : vector<12xi64> - vector.print %22 : vector<12xi64> - vector.print %23 : vector<12xi64> + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 8 + // CHECK-NEXT: dim = ( 8 ) + // CHECK-NEXT: lvl = ( 8 ) + // CHECK-NEXT: pos[0] : ( 0, 8 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7 + // CHECK-NEXT: values : ( 0, 1, 12, 3, 24, 5, 6, 7 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 8 + // CHECK-NEXT: dim = ( 8 ) + // CHECK-NEXT: lvl = ( 8 ) + // CHECK-NEXT: pos[0] : ( 0, 8 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7 + // CHECK-NEXT: values : ( 0, 2, 8, 24, 64, 160, 384, 896 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 8 + // CHECK-NEXT: dim = ( 8 ) + // CHECK-NEXT: lvl = ( 8 ) + // CHECK-NEXT: pos[0] : ( 0, 8 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7 + // CHECK-NEXT: values : ( 1, 3, 6, 11, 20, 37, 70, 135 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 2 + // CHECK-NEXT: dim = ( 3, 4 ) + // CHECK-NEXT: lvl = ( 3, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 1, 2 + // CHECK-NEXT: crd[1] : ( 1, 3 + // CHECK-NEXT: values : ( 10, 120 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 3, 4 ) + // CHECK-NEXT: lvl = ( 3, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: values : ( 0, 1, 2, 3, 1, 12, 3, 4, 2, 3, 4, 25 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 3, 4 ) + // CHECK-NEXT: lvl = ( 3, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: values : ( 0, 0, 0, 0, 0, 2, 2, 3, 0, 2, 12, 24 + // CHECK-NEXT: ---- + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 3, 4 ) + // CHECK-NEXT: lvl = ( 3, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 2, 4, 4, 5, 3, 4, 7, 9 + // CHECK-NEXT: ---- + // + sparse_tensor.print %0 : tensor<8xi64, #SparseVector> + sparse_tensor.print %1 : tensor<8xi64, #SparseVector> + sparse_tensor.print %2 : tensor<8xi64, #SparseVector> + sparse_tensor.print %3 : tensor<8xi64, #SparseVector> + sparse_tensor.print %4 : tensor<3x4xi64, #SparseMatrix> + sparse_tensor.print %5 : tensor<3x4xi64, #SparseMatrix> + sparse_tensor.print %6 : tensor<3x4xi64, #SparseMatrix> + sparse_tensor.print %7 : tensor<3x4xi64, #SparseMatrix> + + // + // Call the f32 kernel, verify the result. + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 2, 3 ) + // CHECK-NEXT: lvl = ( 2, 3 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 1 + // CHECK-NEXT: pos[1] : ( 0, 3, 6 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: values : ( 0, 10, 0, 1, 1, 42 + // CHECK-NEXT: ---- + // + %100 = call @add_outer_2d(%sf32) : (tensor<2x3xf32, #SparseMatrix>) + -> tensor<2x3xf32, #SparseMatrix> + sparse_tensor.print %100 : tensor<2x3xf32, #SparseMatrix> // Release resources. bufferization.dealloc_tensor %sv : tensor<8xi64, #SparseVector> @@ -279,17 +328,6 @@ module { bufferization.dealloc_tensor %5 : tensor<3x4xi64, #SparseMatrix> bufferization.dealloc_tensor %6 : tensor<3x4xi64, #SparseMatrix> bufferization.dealloc_tensor %7 : tensor<3x4xi64, #SparseMatrix> - - // - // Call the f32 kernel, verify the result, release the resources. - // - // CHECK-NEXT: ( 0, 10, 0, 1, 1, 42 ) - // - %100 = call @add_outer_2d(%sf32) : (tensor<2x3xf32, #SparseMatrix>) - -> tensor<2x3xf32, #SparseMatrix> - %101 = sparse_tensor.values %100 : tensor<2x3xf32, #SparseMatrix> to memref - %102 = vector.transfer_read %101[%c0], %df: memref, vector<6xf32> - vector.print %102 : vector<6xf32> bufferization.dealloc_tensor %sf32 : tensor<2x3xf32, #SparseMatrix> bufferization.dealloc_tensor %100 : tensor<2x3xf32, #SparseMatrix> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir index 4bb1f3d12871..fc7b82fdecea 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -138,7 +138,7 @@ module { // // Main driver. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %du = arith.constant -1 : i64 diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir index 364a188cf37c..db6612402357 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -48,7 +48,7 @@ module { // // Main driver. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index %c2 = arith.constant 2 : index -- GitLab From 16ae493f56c1857ec0f6f2777e9b8a2e5151b4ef Mon Sep 17 00:00:00 2001 From: Peter Rong Date: Tue, 12 Mar 2024 18:07:44 -0700 Subject: [PATCH 0074/1407] [FuzzMutate] Only use undef when explictly asked to (#84959) Per discussion in https://github.com/SecurityLab-UCD/IRFuzzer/issues/49, generating undef during fuzzing seems to be less fruitful. Let's eliminate undef in favor of poison unless the user explicitly asked for it. Signed-off-by: Peter Rong --- llvm/lib/FuzzMutate/OpDescriptor.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/llvm/lib/FuzzMutate/OpDescriptor.cpp b/llvm/lib/FuzzMutate/OpDescriptor.cpp index 4baf45284de1..6ec70d917918 100644 --- a/llvm/lib/FuzzMutate/OpDescriptor.cpp +++ b/llvm/lib/FuzzMutate/OpDescriptor.cpp @@ -8,10 +8,15 @@ #include "llvm/FuzzMutate/OpDescriptor.h" #include "llvm/IR/Constants.h" +#include "llvm/Support/CommandLine.h" using namespace llvm; using namespace fuzzerop; +static cl::opt UseUndef("use-undef", + cl::desc("Use undef when generating programs."), + cl::init(false)); + void fuzzerop::makeConstantsWithType(Type *T, std::vector &Cs) { if (auto *IntTy = dyn_cast(T)) { uint64_t W = IntTy->getBitWidth(); @@ -42,7 +47,8 @@ void fuzzerop::makeConstantsWithType(Type *T, std::vector &Cs) { Cs.push_back(ConstantVector::getSplat(EC, Elt)); } } else { - Cs.push_back(UndefValue::get(T)); + if (UseUndef) + Cs.push_back(UndefValue::get(T)); Cs.push_back(PoisonValue::get(T)); } } -- GitLab From c6a93fe80b3cf30ff82d06e959c1177798c858ae Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Tue, 12 Mar 2024 18:25:36 -0700 Subject: [PATCH 0075/1407] [libc] Use __builtin_ffsll for RPC lane mask (#85000) src/__support/GPU/utils.h doesn't compile on a 32-bit platforms because __builtin_ffsl uses long which is a 32-bit number. Use __builtin_ffsll which uses long long which is guaranteed to be at least 64-bits. --- libc/src/__support/GPU/utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/src/__support/GPU/utils.h b/libc/src/__support/GPU/utils.h index 93022e8de811..cb04a3562eb1 100644 --- a/libc/src/__support/GPU/utils.h +++ b/libc/src/__support/GPU/utils.h @@ -23,7 +23,7 @@ namespace LIBC_NAMESPACE { namespace gpu { /// Get the first active thread inside the lane. LIBC_INLINE uint64_t get_first_lane_id(uint64_t lane_mask) { - return __builtin_ffsl(lane_mask) - 1; + return __builtin_ffsll(lane_mask) - 1; } /// Conditional that is only true for a single thread in a lane. -- GitLab From 9d6c43b4aed117f53167e72749b31a943941345d Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Wed, 13 Mar 2024 09:26:53 +0800 Subject: [PATCH 0076/1407] [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (#84285) I suddenly found that the clang scan deps may use all concurrent threads to scan the files. It makes sense in the batch mode. But in P1689 per file mode, it simply wastes times and resources. This patch itself should be a NFC patch. It simply moves codes. --- clang/tools/clang-scan-deps/ClangScanDeps.cpp | 195 +++++++++--------- 1 file changed, 100 insertions(+), 95 deletions(-) diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index d042fecc3dbe..eaa76dd43e41 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -867,13 +867,6 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { // Print out the dependency results to STDOUT by default. SharedStream DependencyOS(llvm::outs()); - DependencyScanningService Service(ScanMode, Format, OptimizeArgs, - EagerLoadModules); - llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(NumThreads)); - std::vector> WorkerTools; - for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) - WorkerTools.push_back(std::make_unique(Service)); - std::vector Inputs = AdjustingCompilations->getAllCompileCommands(); @@ -893,102 +886,114 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { if (Format == ScanningOutputFormat::Full) FD.emplace(ModuleName.empty() ? Inputs.size() : 0); - if (Verbose) { - llvm::outs() << "Running clang-scan-deps on " << Inputs.size() - << " files using " << Pool.getMaxConcurrency() << " workers\n"; - } + auto ScanningTask = [&](DependencyScanningService &Service) { + DependencyScanningTool WorkerTool(Service); + + llvm::DenseSet AlreadySeenModules; + while (auto MaybeInputIndex = GetNextInputIndex()) { + size_t LocalIndex = *MaybeInputIndex; + const tooling::CompileCommand *Input = &Inputs[LocalIndex]; + std::string Filename = std::move(Input->Filename); + std::string CWD = std::move(Input->Directory); + + std::optional MaybeModuleName; + if (!ModuleName.empty()) + MaybeModuleName = ModuleName; + + std::string OutputDir(ModuleFilesDir); + if (OutputDir.empty()) + OutputDir = getModuleCachePath(Input->CommandLine); + auto LookupOutput = [&](const ModuleID &MID, ModuleOutputKind MOK) { + return ::lookupModuleOutput(MID, MOK, OutputDir); + }; + + // Run the tool on it. + if (Format == ScanningOutputFormat::Make) { + auto MaybeFile = WorkerTool.getDependencyFile(Input->CommandLine, CWD); + if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS, + Errs)) + HadErrors = true; + } else if (Format == ScanningOutputFormat::P1689) { + // It is useful to generate the make-format dependency output during + // the scanning for P1689. Otherwise the users need to scan again for + // it. We will generate the make-format dependency output if we find + // `-MF` in the command lines. + std::string MakeformatOutputPath; + std::string MakeformatOutput; + + auto MaybeRule = WorkerTool.getP1689ModuleDependencyFile( + *Input, CWD, MakeformatOutput, MakeformatOutputPath); + + if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs)) + HadErrors = true; + + if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() && + !HadErrors) { + static std::mutex Lock; + // With compilation database, we may open different files + // concurrently or we may write the same file concurrently. So we + // use a map here to allow multiple compile commands to write to the + // same file. Also we need a lock here to avoid data race. + static llvm::StringMap OSs; + std::unique_lock LockGuard(Lock); + + auto OSIter = OSs.find(MakeformatOutputPath); + if (OSIter == OSs.end()) { + std::error_code EC; + OSIter = + OSs.try_emplace(MakeformatOutputPath, MakeformatOutputPath, EC) + .first; + if (EC) + llvm::errs() << "Failed to open P1689 make format output file \"" + << MakeformatOutputPath << "\" for " << EC.message() + << "\n"; + } + + SharedStream MakeformatOS(OSIter->second); + llvm::Expected MaybeOutput(MakeformatOutput); + if (handleMakeDependencyToolResult(Filename, MaybeOutput, + MakeformatOS, Errs)) + HadErrors = true; + } + } else if (MaybeModuleName) { + auto MaybeModuleDepsGraph = WorkerTool.getModuleDependencies( + *MaybeModuleName, Input->CommandLine, CWD, AlreadySeenModules, + LookupOutput); + if (handleModuleResult(*MaybeModuleName, MaybeModuleDepsGraph, *FD, + LocalIndex, DependencyOS, Errs)) + HadErrors = true; + } else { + auto MaybeTUDeps = WorkerTool.getTranslationUnitDependencies( + Input->CommandLine, CWD, AlreadySeenModules, LookupOutput); + if (handleTranslationUnitResult(Filename, MaybeTUDeps, *FD, LocalIndex, + DependencyOS, Errs)) + HadErrors = true; + } + } + }; + + DependencyScanningService Service(ScanMode, Format, OptimizeArgs, + EagerLoadModules); llvm::Timer T; T.startTimer(); - for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) { - Pool.async([&, I]() { - llvm::DenseSet AlreadySeenModules; - while (auto MaybeInputIndex = GetNextInputIndex()) { - size_t LocalIndex = *MaybeInputIndex; - const tooling::CompileCommand *Input = &Inputs[LocalIndex]; - std::string Filename = std::move(Input->Filename); - std::string CWD = std::move(Input->Directory); - - std::optional MaybeModuleName; - if (!ModuleName.empty()) - MaybeModuleName = ModuleName; - - std::string OutputDir(ModuleFilesDir); - if (OutputDir.empty()) - OutputDir = getModuleCachePath(Input->CommandLine); - auto LookupOutput = [&](const ModuleID &MID, ModuleOutputKind MOK) { - return ::lookupModuleOutput(MID, MOK, OutputDir); - }; + if (Inputs.size() == 1) { + ScanningTask(Service); + } else { + llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(NumThreads)); - // Run the tool on it. - if (Format == ScanningOutputFormat::Make) { - auto MaybeFile = - WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD); - if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS, - Errs)) - HadErrors = true; - } else if (Format == ScanningOutputFormat::P1689) { - // It is useful to generate the make-format dependency output during - // the scanning for P1689. Otherwise the users need to scan again for - // it. We will generate the make-format dependency output if we find - // `-MF` in the command lines. - std::string MakeformatOutputPath; - std::string MakeformatOutput; - - auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile( - *Input, CWD, MakeformatOutput, MakeformatOutputPath); - - if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs)) - HadErrors = true; + if (Verbose) { + llvm::outs() << "Running clang-scan-deps on " << Inputs.size() + << " files using " << Pool.getMaxConcurrency() + << " workers\n"; + } - if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() && - !HadErrors) { - static std::mutex Lock; - // With compilation database, we may open different files - // concurrently or we may write the same file concurrently. So we - // use a map here to allow multiple compile commands to write to the - // same file. Also we need a lock here to avoid data race. - static llvm::StringMap OSs; - std::unique_lock LockGuard(Lock); - - auto OSIter = OSs.find(MakeformatOutputPath); - if (OSIter == OSs.end()) { - std::error_code EC; - OSIter = OSs.try_emplace(MakeformatOutputPath, - MakeformatOutputPath, EC) - .first; - if (EC) - llvm::errs() - << "Failed to open P1689 make format output file \"" - << MakeformatOutputPath << "\" for " << EC.message() - << "\n"; - } + for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) + Pool.async([ScanningTask, &Service]() { ScanningTask(Service); }); - SharedStream MakeformatOS(OSIter->second); - llvm::Expected MaybeOutput(MakeformatOutput); - if (handleMakeDependencyToolResult(Filename, MaybeOutput, - MakeformatOS, Errs)) - HadErrors = true; - } - } else if (MaybeModuleName) { - auto MaybeModuleDepsGraph = WorkerTools[I]->getModuleDependencies( - *MaybeModuleName, Input->CommandLine, CWD, AlreadySeenModules, - LookupOutput); - if (handleModuleResult(*MaybeModuleName, MaybeModuleDepsGraph, *FD, - LocalIndex, DependencyOS, Errs)) - HadErrors = true; - } else { - auto MaybeTUDeps = WorkerTools[I]->getTranslationUnitDependencies( - Input->CommandLine, CWD, AlreadySeenModules, LookupOutput); - if (handleTranslationUnitResult(Filename, MaybeTUDeps, *FD, - LocalIndex, DependencyOS, Errs)) - HadErrors = true; - } - } - }); + Pool.wait(); } - Pool.wait(); T.stopTimer(); if (PrintTiming) -- GitLab From 88986d65e4ed1b2ddd6693ff3e70b84436af767a Mon Sep 17 00:00:00 2001 From: Yinying Li Date: Tue, 12 Mar 2024 21:39:37 -0400 Subject: [PATCH 0077/1407] [mlir][sparse] Fix sparse_generate test (#85009) std::uniform_int_distribution may behave differently in different systems. --- .../SparseTensor/CPU/sparse_generate.mlir | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir index e1f73eb4ac4f..63a6d3acd737 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir @@ -78,20 +78,13 @@ module { } %sv = sparse_tensor.convert %output : tensor to tensor + %n0 = sparse_tensor.number_of_entries %sv : tensor + // Print the number of non-zeros for verification + // as shuffle may generate different numbers. // - // Verify the outputs. - // - // CHECK: ---- Sparse Tensor ---- - // CHECK-NEXT: nse = 5 - // CHECK-NEXT: dim = ( 50 ) - // CHECK-NEXT: lvl = ( 50 ) - // CHECK-NEXT: pos[0] : ( 0, 5 - // CHECK-NEXT: crd[0] : ( 1, 9, 17, 27, 30 - // CHECK-NEXT: values : ( 84, 34, 8, 40, 93 - // CHECK-NEXT: ---- - // - sparse_tensor.print %sv : tensor + // CHECK: 5 + vector.print %n0 : index // Release the resources. bufferization.dealloc_tensor %sv : tensor -- GitLab From 096d061c1c2f91edef7186bd8b61067020f49898 Mon Sep 17 00:00:00 2001 From: Sterling Augustine Date: Wed, 13 Mar 2024 01:37:53 +0000 Subject: [PATCH 0078/1407] Add missing dependency after 80ab8234ac309418637488b97e0a62d8377b2ecf --- .../llvm-project-overlay/libc/test/src/__support/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel index 6837b9880d5a..5434761a9b13 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel @@ -65,6 +65,7 @@ libc_test( srcs = ["integer_to_string_test.cpp"], deps = [ "//libc:__support_cpp_span", + "//libc:__support_cpp_limits", "//libc:__support_cpp_string_view", "//libc:__support_integer_literals", "//libc:__support_integer_to_string", -- GitLab From 34cf6847752a3aad68eaa889ab6e0073a38fc2db Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Tue, 12 Mar 2024 19:33:16 -0700 Subject: [PATCH 0079/1407] [libc] Move `struct timespec` from POSIX to StdC (#85010) `struct timespec` is actually defined in the C standard, not POSIX. --- libc/spec/posix.td | 4 ---- libc/spec/spec.td | 4 ++++ libc/spec/stdc.td | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/libc/spec/posix.td b/libc/spec/posix.td index d0f5a4584dd4..591919aac95d 100644 --- a/libc/spec/posix.td +++ b/libc/spec/posix.td @@ -42,10 +42,6 @@ def StructDirentPtr : PtrType; def StructDirentPtrPtr : PtrType; def ConstStructDirentPtrPtr : ConstType; -def StructTimeSpec : NamedType<"struct timespec">; -def StructTimeSpecPtr : PtrType; -def ConstStructTimeSpecPtr : ConstType; - def StructSchedParam : NamedType<"struct sched_param">; def StructSchedParamPtr : PtrType; def ConstStructSchedParamPtr : ConstType; diff --git a/libc/spec/spec.td b/libc/spec/spec.td index a44a7ae131b5..580bd9c8c3c1 100644 --- a/libc/spec/spec.td +++ b/libc/spec/spec.td @@ -118,6 +118,10 @@ def SigHandlerT : NamedType<"__sighandler_t">; def TimeTType : NamedType<"time_t">; +def StructTimeSpec : NamedType<"struct timespec">; +def StructTimeSpecPtr : PtrType; +def ConstStructTimeSpecPtr : ConstType; + def BSearchCompareT : NamedType<"__bsearchcompare_t">; def QSortCompareT : NamedType<"__qsortcompare_t">; diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index e012d0dee089..e8f73dde88fc 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -1202,6 +1202,7 @@ def StdC : StandardSpec<"stdc"> { [ // Types ClockT, StructTmType, + StructTimeSpec, TimeTType, ], [], // Enumerations -- GitLab From a62222f5f0bf30a5437255521df62750060a4bf4 Mon Sep 17 00:00:00 2001 From: Sterling Augustine Date: Wed, 13 Mar 2024 02:51:09 +0000 Subject: [PATCH 0080/1407] Update BUILD.bazel for 0ebf511ad011a83022edb171e044c98d9d16b1fa --- utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 5f6c43cd6af7..073353a89c89 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -595,6 +595,7 @@ libc_support_library( name = "__support_math_extras", hdrs = ["src/__support/math_extras.h"], deps = [ + ":__support_cpp_bit", ":__support_cpp_limits", ":__support_cpp_type_traits", ":__support_macros_attributes", -- GitLab From 5d7796e674224be54c48a8db981f4134845bcc7c Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Wed, 13 Mar 2024 11:20:38 +0800 Subject: [PATCH 0081/1407] [NFC] [C++20] [Modules] Refactor ReducedBMIGenerator Changes: - Don't lookup the emitting module from HeaderSearch. We will use the module from the ASTContext directly. - Remove some useless arguments. Let's addback in the future if required. --- clang/include/clang/Serialization/ASTWriter.h | 8 +++- clang/lib/Frontend/FrontendActions.cpp | 8 ++-- clang/lib/Serialization/GeneratePCH.cpp | 42 ++++++++++++------- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h index e5db486a71a4..3ed9803fa374 100644 --- a/clang/include/clang/Serialization/ASTWriter.h +++ b/clang/include/clang/Serialization/ASTWriter.h @@ -868,6 +868,8 @@ protected: return SemaPtr->getDiagnostics(); } + virtual Module *getEmittingModule(ASTContext &Ctx); + public: PCHGenerator(const Preprocessor &PP, InMemoryModuleCache &ModuleCache, StringRef OutputFile, StringRef isysroot, @@ -887,10 +889,12 @@ public: }; class ReducedBMIGenerator : public PCHGenerator { +protected: + virtual Module *getEmittingModule(ASTContext &Ctx) override; + public: ReducedBMIGenerator(const Preprocessor &PP, InMemoryModuleCache &ModuleCache, - StringRef OutputFile, std::shared_ptr Buffer, - bool IncludeTimestamps); + StringRef OutputFile); void HandleTranslationUnit(ASTContext &Ctx) override; }; diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index 50338bfa670f..81fcd8d5ae9b 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -293,11 +293,9 @@ GenerateModuleInterfaceAction::CreateOutputFile(CompilerInstance &CI, std::unique_ptr GenerateReducedModuleInterfaceAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { - auto Buffer = std::make_shared(); - return std::make_unique( - CI.getPreprocessor(), CI.getModuleCache(), - CI.getFrontendOpts().OutputFile, Buffer, - /*IncludeTimestamps=*/+CI.getFrontendOpts().IncludeTimestamps); + return std::make_unique(CI.getPreprocessor(), + CI.getModuleCache(), + CI.getFrontendOpts().OutputFile); } bool GenerateHeaderUnitAction::BeginSourceFileAction(CompilerInstance &CI) { diff --git a/clang/lib/Serialization/GeneratePCH.cpp b/clang/lib/Serialization/GeneratePCH.cpp index 2b511b2d5a90..f54db36d4a01 100644 --- a/clang/lib/Serialization/GeneratePCH.cpp +++ b/clang/lib/Serialization/GeneratePCH.cpp @@ -41,6 +41,21 @@ PCHGenerator::PCHGenerator( PCHGenerator::~PCHGenerator() { } +Module *PCHGenerator::getEmittingModule(ASTContext &) { + Module *M = nullptr; + + if (PP.getLangOpts().isCompilingModule()) { + M = PP.getHeaderSearchInfo().lookupModule(PP.getLangOpts().CurrentModule, + SourceLocation(), + /*AllowSearch*/ false); + if (!M) + assert(PP.getDiagnostics().hasErrorOccurred() && + "emitting module but current module doesn't exist"); + } + + return M; +} + void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) { // Don't create a PCH if there were fatal failures during module loading. if (PP.getModuleLoader().HadFatalFailure) @@ -50,16 +65,7 @@ void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) { if (hasErrors && !AllowASTWithErrors) return; - Module *Module = nullptr; - if (PP.getLangOpts().isCompilingModule()) { - Module = PP.getHeaderSearchInfo().lookupModule( - PP.getLangOpts().CurrentModule, SourceLocation(), - /*AllowSearch*/ false); - if (!Module) { - assert(hasErrors && "emitting module but current module doesn't exist"); - return; - } - } + Module *Module = getEmittingModule(Ctx); // Errors that do not prevent the PCH from being written should not cause the // overall compilation to fail either. @@ -84,16 +90,22 @@ ASTDeserializationListener *PCHGenerator::GetASTDeserializationListener() { ReducedBMIGenerator::ReducedBMIGenerator(const Preprocessor &PP, InMemoryModuleCache &ModuleCache, - StringRef OutputFile, - std::shared_ptr Buffer, - bool IncludeTimestamps) + StringRef OutputFile) : PCHGenerator( - PP, ModuleCache, OutputFile, llvm::StringRef(), Buffer, + PP, ModuleCache, OutputFile, llvm::StringRef(), + std::make_shared(), /*Extensions=*/ArrayRef>(), - /*AllowASTWithErrors*/ false, /*IncludeTimestamps=*/IncludeTimestamps, + /*AllowASTWithErrors*/ false, /*IncludeTimestamps=*/false, /*BuildingImplicitModule=*/false, /*ShouldCacheASTInMemory=*/false, /*GeneratingReducedBMI=*/true) {} +Module *ReducedBMIGenerator::getEmittingModule(ASTContext &Ctx) { + Module *M = Ctx.getCurrentNamedModule(); + assert(M->isNamedModuleUnit() && + "ReducedBMIGenerator should only be used with C++20 Named modules."); + return M; +} + void ReducedBMIGenerator::HandleTranslationUnit(ASTContext &Ctx) { PCHGenerator::HandleTranslationUnit(Ctx); -- GitLab From 15a55486a54183d4fc597ed86c0d49fe9482f2bd Mon Sep 17 00:00:00 2001 From: Michael Flanders Date: Tue, 12 Mar 2024 20:25:05 -0700 Subject: [PATCH 0082/1407] [libc][math] Adds entrypoint and test for `nextafterf128` (#84882) --- libc/config/linux/aarch64/entrypoints.txt | 1 + libc/config/linux/riscv/entrypoints.txt | 1 + libc/config/linux/x86_64/entrypoints.txt | 1 + libc/docs/math/index.rst | 2 ++ libc/spec/stdc.td | 1 + libc/src/math/CMakeLists.txt | 1 + libc/src/math/generic/CMakeLists.txt | 19 +++++++++++++++--- libc/src/math/generic/nextafterf128.cpp | 19 ++++++++++++++++++ libc/src/math/nextafterf128.h | 20 +++++++++++++++++++ libc/test/src/math/CMakeLists.txt | 15 ++++++++++++++ libc/test/src/math/nextafterf128_test.cpp | 13 ++++++++++++ libc/test/src/math/smoke/CMakeLists.txt | 15 ++++++++++++++ .../src/math/smoke/nextafterf128_test.cpp | 13 ++++++++++++ 13 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 libc/src/math/generic/nextafterf128.cpp create mode 100644 libc/src/math/nextafterf128.h create mode 100644 libc/test/src/math/nextafterf128_test.cpp create mode 100644 libc/test/src/math/smoke/nextafterf128_test.cpp diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index 1656973cb27c..abd1f83794ed 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -438,6 +438,7 @@ if(LIBC_TYPES_HAS_FLOAT128) libc.src.math.lrintf128 libc.src.math.lroundf128 libc.src.math.modff128 + libc.src.math.nextafterf128 libc.src.math.rintf128 libc.src.math.roundf128 libc.src.math.sqrtf128 diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt index 07d1acfcfe07..006aa787ea6a 100644 --- a/libc/config/linux/riscv/entrypoints.txt +++ b/libc/config/linux/riscv/entrypoints.txt @@ -446,6 +446,7 @@ if(LIBC_TYPES_HAS_FLOAT128) libc.src.math.lrintf128 libc.src.math.lroundf128 libc.src.math.modff128 + libc.src.math.nextafterf128 libc.src.math.rintf128 libc.src.math.roundf128 libc.src.math.sqrtf128 diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index e0324061a9c7..4fb31c593b9d 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -481,6 +481,7 @@ if(LIBC_TYPES_HAS_FLOAT128) libc.src.math.lrintf128 libc.src.math.lroundf128 libc.src.math.modff128 + libc.src.math.nextafterf128 libc.src.math.rintf128 libc.src.math.roundf128 libc.src.math.sqrtf128 diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst index 6984b785125f..ed54a7d091ec 100644 --- a/libc/docs/math/index.rst +++ b/libc/docs/math/index.rst @@ -271,6 +271,8 @@ Basic Operations +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ | nextafterl | |check| | |check| | |check| | |check| | |check| | | | |check| | |check| | |check| | | | +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| nextafterf128| |check| | |check| | | |check| | | | | | | | | | ++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ | nexttoward | |check| | |check| | |check| | |check| | |check| | | | |check| | |check| | |check| | | | +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ | nexttowardf | |check| | |check| | |check| | |check| | |check| | | | |check| | |check| | |check| | | | diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index e8f73dde88fc..938d3722a0f8 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -530,6 +530,7 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"nextafterf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"nextafter", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"nextafterl", RetValSpec, [ArgSpec, ArgSpec]>, + GuardedFunctionSpec<"nextafterf128", RetValSpec, [ArgSpec, ArgSpec], "LIBC_TYPES_HAS_FLOAT128">, FunctionSpec<"nexttowardf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"nexttoward", RetValSpec, [ArgSpec, ArgSpec]>, diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt index bba02aa78a23..750fd5f0e3a9 100644 --- a/libc/src/math/CMakeLists.txt +++ b/libc/src/math/CMakeLists.txt @@ -198,6 +198,7 @@ add_math_entrypoint_object(nearbyintl) add_math_entrypoint_object(nextafter) add_math_entrypoint_object(nextafterf) add_math_entrypoint_object(nextafterl) +add_math_entrypoint_object(nextafterf128) add_math_entrypoint_object(nexttoward) add_math_entrypoint_object(nexttowardf) diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt index bc4e9b34cfc2..667381d615d1 100644 --- a/libc/src/math/generic/CMakeLists.txt +++ b/libc/src/math/generic/CMakeLists.txt @@ -1789,7 +1789,7 @@ add_entrypoint_object( DEPENDS libc.src.__support.FPUtil.manipulation_functions COMPILE_OPTIONS - -O2 + -O3 ) add_entrypoint_object( @@ -1801,7 +1801,7 @@ add_entrypoint_object( DEPENDS libc.src.__support.FPUtil.manipulation_functions COMPILE_OPTIONS - -O2 + -O3 ) add_entrypoint_object( @@ -1813,7 +1813,20 @@ add_entrypoint_object( DEPENDS libc.src.__support.FPUtil.manipulation_functions COMPILE_OPTIONS - -O2 + -O3 +) + +add_entrypoint_object( + nextafterf128 + SRCS + nextafterf128.cpp + HDRS + ../nextafterf128.h + DEPENDS + libc.src.__support.macros.properties.types + libc.src.__support.FPUtil.manipulation_functions + COMPILE_OPTIONS + -O3 ) add_entrypoint_object( diff --git a/libc/src/math/generic/nextafterf128.cpp b/libc/src/math/generic/nextafterf128.cpp new file mode 100644 index 000000000000..905c89022ba1 --- /dev/null +++ b/libc/src/math/generic/nextafterf128.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of nextafterf128 function --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/nextafterf128.h" +#include "src/__support/FPUtil/ManipulationFunctions.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float128, nextafterf128, (float128 x, float128 y)) { + return fputil::nextafter(x, y); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/nextafterf128.h b/libc/src/math/nextafterf128.h new file mode 100644 index 000000000000..a404d33810ec --- /dev/null +++ b/libc/src/math/nextafterf128.h @@ -0,0 +1,20 @@ +//===-- Implementation header for nextafterf128 ------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_NEXTAFTERF128_H +#define LLVM_LIBC_SRC_MATH_NEXTAFTERF128_H + +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE { + +float128 nextafterf128(float128 x, float128 y); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_NEXTAFTERF128_H diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt index b8a4aafcd97a..7b952901da76 100644 --- a/libc/test/src/math/CMakeLists.txt +++ b/libc/test/src/math/CMakeLists.txt @@ -1273,6 +1273,21 @@ add_fp_unittest( libc.src.__support.FPUtil.fp_bits ) +add_fp_unittest( + nextafterf128_test + SUITE + libc-math-unittests + SRCS + nextafterf128_test.cpp + HDRS + NextAfterTest.h + DEPENDS + libc.include.math + libc.src.math.nextafterf128 + libc.src.__support.FPUtil.basic_operations + libc.src.__support.FPUtil.fp_bits +) + # TODO(lntue): The current implementation of fputil::general::fma is only # correctly rounded for the default rounding mode round-to-nearest tie-to-even. add_fp_unittest( diff --git a/libc/test/src/math/nextafterf128_test.cpp b/libc/test/src/math/nextafterf128_test.cpp new file mode 100644 index 000000000000..a8d000ff4de3 --- /dev/null +++ b/libc/test/src/math/nextafterf128_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for nextafterf128 ---------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "NextAfterTest.h" + +#include "src/math/nextafterf128.h" + +LIST_NEXTAFTER_TESTS(float128, LIBC_NAMESPACE::nextafterf128) diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt index d9be172056a8..293e65abd44f 100644 --- a/libc/test/src/math/smoke/CMakeLists.txt +++ b/libc/test/src/math/smoke/CMakeLists.txt @@ -1551,6 +1551,21 @@ add_fp_unittest( libc.src.__support.FPUtil.fp_bits ) +add_fp_unittest( + nextafterf128_test + SUITE + libc-math-smoke-tests + SRCS + nextafterf128_test.cpp + HDRS + NextAfterTest.h + DEPENDS + libc.include.math + libc.src.math.nextafterf128 + libc.src.__support.FPUtil.basic_operations + libc.src.__support.FPUtil.fp_bits +) + # FIXME: These tests are currently spurious for the GPU. if(NOT LIBC_TARGET_OS_IS_GPU) add_fp_unittest( diff --git a/libc/test/src/math/smoke/nextafterf128_test.cpp b/libc/test/src/math/smoke/nextafterf128_test.cpp new file mode 100644 index 000000000000..a8d000ff4de3 --- /dev/null +++ b/libc/test/src/math/smoke/nextafterf128_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for nextafterf128 ---------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "NextAfterTest.h" + +#include "src/math/nextafterf128.h" + +LIST_NEXTAFTER_TESTS(float128, LIBC_NAMESPACE::nextafterf128) -- GitLab From e4edbae0aa6a9739954ee3b494b18f8c599d9d79 Mon Sep 17 00:00:00 2001 From: Lu Weining Date: Wed, 13 Mar 2024 11:51:47 +0800 Subject: [PATCH 0083/1407] Revert "[llvm][LoongArch] Improve loongarch_lasx_xvpermi_q instrinsic" (#84708) Reverts llvm/llvm-project#82984 See the discussion in https://github.com/llvm/llvm-project/pull/83540. --- .../LoongArch/LoongArchISelLowering.cpp | 25 +--------------- .../CodeGen/LoongArch/lasx/intrinsic-permi.ll | 30 ------------------- 2 files changed, 1 insertion(+), 54 deletions(-) diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index c87f5341d7fe..c13b10a320f8 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -968,28 +968,6 @@ static SDValue checkIntrinsicImmArg(SDValue Op, unsigned ImmOp, return SDValue(); } -static SDValue checkAndModifyXVPERMI_QIntrinsicImmArg(SDValue Op, - SelectionDAG &DAG) { - SDValue Op3 = Op->getOperand(3); - uint64_t Imm = Op3->getAsZExtVal(); - // Check the range of ImmArg. - if (!isUInt<8>(Imm)) { - DAG.getContext()->emitError(Op->getOperationName(0) + - ": argument out of range."); - return DAG.getNode(ISD::UNDEF, SDLoc(Op), Op.getValueType()); - } - - // For instruction xvpermi.q, only [1:0] and [5:4] bits of operands[3] - // are used. The unused bits in operands[3] need to be set to 0 to avoid - // causing undefined behavior on LA464. - if ((Imm & 0x33) != Imm) { - Op3 = DAG.getTargetConstant(Imm & 0x33, SDLoc(Op), Op3.getValueType()); - DAG.UpdateNodeOperands(Op.getNode(), Op->getOperand(0), Op->getOperand(1), - Op->getOperand(2), Op3); - } - return SDValue(); -} - SDValue LoongArchTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const { @@ -1247,14 +1225,13 @@ LoongArchTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op, case Intrinsic::loongarch_lsx_vextrins_d: case Intrinsic::loongarch_lasx_xvshuf4i_d: case Intrinsic::loongarch_lasx_xvpermi_w: + case Intrinsic::loongarch_lasx_xvpermi_q: case Intrinsic::loongarch_lasx_xvbitseli_b: case Intrinsic::loongarch_lasx_xvextrins_b: case Intrinsic::loongarch_lasx_xvextrins_h: case Intrinsic::loongarch_lasx_xvextrins_w: case Intrinsic::loongarch_lasx_xvextrins_d: return checkIntrinsicImmArg<8>(Op, 3, DAG); - case Intrinsic::loongarch_lasx_xvpermi_q: - return checkAndModifyXVPERMI_QIntrinsicImmArg(Op, DAG); case Intrinsic::loongarch_lsx_vrepli_b: case Intrinsic::loongarch_lsx_vrepli_h: case Intrinsic::loongarch_lsx_vrepli_w: diff --git a/llvm/test/CodeGen/LoongArch/lasx/intrinsic-permi.ll b/llvm/test/CodeGen/LoongArch/lasx/intrinsic-permi.ll index 92669d2e5895..0d9f9daabc44 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/intrinsic-permi.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/intrinsic-permi.ll @@ -36,33 +36,3 @@ entry: %res = call <32 x i8> @llvm.loongarch.lasx.xvpermi.q(<32 x i8> %va, <32 x i8> %vb, i32 1) ret <32 x i8> %res } - -define <32 x i8> @lasx_xvpermi_q_204(<32 x i8> %va, <32 x i8> %vb) nounwind { -; CHECK-LABEL: lasx_xvpermi_q_204: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 0 -; CHECK-NEXT: ret -entry: - %res = call <32 x i8> @llvm.loongarch.lasx.xvpermi.q(<32 x i8> %va, <32 x i8> %vb, i32 204) - ret <32 x i8> %res -} - -define <32 x i8> @lasx_xvpermi_q_221(<32 x i8> %va, <32 x i8> %vb) nounwind { -; CHECK-LABEL: lasx_xvpermi_q_221: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 17 -; CHECK-NEXT: ret -entry: - %res = call <32 x i8> @llvm.loongarch.lasx.xvpermi.q(<32 x i8> %va, <32 x i8> %vb, i32 221) - ret <32 x i8> %res -} - -define <32 x i8> @lasx_xvpermi_q_255(<32 x i8> %va, <32 x i8> %vb) nounwind { -; CHECK-LABEL: lasx_xvpermi_q_255: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 51 -; CHECK-NEXT: ret -entry: - %res = call <32 x i8> @llvm.loongarch.lasx.xvpermi.q(<32 x i8> %va, <32 x i8> %vb, i32 255) - ret <32 x i8> %res -} -- GitLab From 2dbaf265255a5fa9643a8092ec2dffa881d2cf93 Mon Sep 17 00:00:00 2001 From: Jeff Niu Date: Wed, 13 Mar 2024 00:12:37 -0400 Subject: [PATCH 0084/1407] [mlir][ods] Fix generation of optional custom parsers (#84821) We need to generate `.has_value` for `OptionalParseResult`, also ensure that `auto result` doesn't conflict with `result` which is the variable name for `OperationState`. --- mlir/test/IR/custom-print-parse.mlir | 5 +++++ mlir/test/IR/invalid-custom-print-parse.mlir | 5 +++++ mlir/test/lib/Dialect/Test/TestDialect.cpp | 17 +++++++++++++++++ mlir/test/lib/Dialect/Test/TestOps.td | 11 +++++++++++ mlir/test/mlir-tblgen/attr-or-type-format.td | 2 +- mlir/test/mlir-tblgen/op-format.td | 8 ++++---- mlir/tools/mlir-tblgen/AttrOrTypeFormatGen.cpp | 2 +- mlir/tools/mlir-tblgen/OpFormatGen.cpp | 8 ++++---- 8 files changed, 48 insertions(+), 10 deletions(-) diff --git a/mlir/test/IR/custom-print-parse.mlir b/mlir/test/IR/custom-print-parse.mlir index b157fd1b1ea3..0eadc2e42956 100644 --- a/mlir/test/IR/custom-print-parse.mlir +++ b/mlir/test/IR/custom-print-parse.mlir @@ -14,4 +14,9 @@ module @dimension_list { test.custom_dimension_list_attr dimension_list = ? // CHECK: test.custom_dimension_list_attr dimension_list = ?x? test.custom_dimension_list_attr dimension_list = ?x? + + // CHECK: test.optional_custom_attr + test.optional_custom_attr bar + // CHECK: test.optional_custom_attr foo false + test.optional_custom_attr foo false } diff --git a/mlir/test/IR/invalid-custom-print-parse.mlir b/mlir/test/IR/invalid-custom-print-parse.mlir index 456b16c91bc0..00da145e35e0 100644 --- a/mlir/test/IR/invalid-custom-print-parse.mlir +++ b/mlir/test/IR/invalid-custom-print-parse.mlir @@ -14,3 +14,8 @@ test.custom_dimension_list_attr dimension_list = -1 // expected-error@+2 {{expected ']'}} // expected-error@+1 {{custom op 'test.custom_dimension_list_attr' Failed parsing dimension list.}} test.custom_dimension_list_attr dimension_list = [2x3] + +// ----- + +// expected-error @below {{expected attribute value}} +test.optional_custom_attr foo diff --git a/mlir/test/lib/Dialect/Test/TestDialect.cpp b/mlir/test/lib/Dialect/Test/TestDialect.cpp index 1ee52fc08d77..380c74a47e50 100644 --- a/mlir/test/lib/Dialect/Test/TestDialect.cpp +++ b/mlir/test/lib/Dialect/Test/TestDialect.cpp @@ -499,6 +499,23 @@ void AffineScopeOp::print(OpAsmPrinter &p) { p.printRegion(getRegion(), /*printEntryBlockArgs=*/false); } +//===----------------------------------------------------------------------===// +// Test OptionalCustomAttrOp +//===----------------------------------------------------------------------===// + +static OptionalParseResult parseOptionalCustomParser(AsmParser &p, + IntegerAttr &result) { + if (succeeded(p.parseOptionalKeyword("foo"))) + return p.parseAttribute(result); + return {}; +} + +static void printOptionalCustomParser(AsmPrinter &p, Operation *, + IntegerAttr result) { + p << "foo "; + p.printAttribute(result); +} + //===----------------------------------------------------------------------===// // Test removing op with inner ops. //===----------------------------------------------------------------------===// diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td index dfd2f21a5ea2..e6c3601d08da 100644 --- a/mlir/test/lib/Dialect/Test/TestOps.td +++ b/mlir/test/lib/Dialect/Test/TestOps.td @@ -2048,6 +2048,17 @@ def CustomDimensionListAttrOp : TEST_Op<"custom_dimension_list_attr"> { }]; } +def OptionalCustomAttrOp : TEST_Op<"optional_custom_attr"> { + let description = [{ + Test using a custom directive as the optional group anchor and the first + element to parse. It is expected to return an `OptionalParseResult`. + }]; + let arguments = (ins OptionalAttr:$attr); + let assemblyFormat = [{ + attr-dict (custom($attr)^) : (`bar`)? + }]; +} + //===----------------------------------------------------------------------===// // Test OpAsmInterface. diff --git a/mlir/test/mlir-tblgen/attr-or-type-format.td b/mlir/test/mlir-tblgen/attr-or-type-format.td index b9041e45f856..2884c4ed6a90 100644 --- a/mlir/test/mlir-tblgen/attr-or-type-format.td +++ b/mlir/test/mlir-tblgen/attr-or-type-format.td @@ -648,7 +648,7 @@ def TypeN : TestType<"TestP"> { // TYPE-LABEL: TestQType::parse // TYPE: if (auto result = [&]() -> ::mlir::OptionalParseResult { // TYPE: auto odsCustomResult = parseAB(odsParser -// TYPE: if (!odsCustomResult) return {}; +// TYPE: if (!odsCustomResult.has_value()) return {}; // TYPE: if (::mlir::failed(*odsCustomResult)) return ::mlir::failure(); // TYPE: return ::mlir::success(); // TYPE: }(); result.has_value() && ::mlir::failed(*result)) { diff --git a/mlir/test/mlir-tblgen/op-format.td b/mlir/test/mlir-tblgen/op-format.td index 3250589605f3..4a19ffb3dfcc 100644 --- a/mlir/test/mlir-tblgen/op-format.td +++ b/mlir/test/mlir-tblgen/op-format.td @@ -93,14 +93,14 @@ def OptionalGroupC : TestFormat_Op<[{ }]>, Arguments<(ins DefaultValuedStrAttr:$a)>; // CHECK-LABEL: OptionalGroupD::parse -// CHECK: if (auto result = [&]() -> ::mlir::OptionalParseResult { +// CHECK: if (auto optResult = [&]() -> ::mlir::OptionalParseResult { // CHECK: auto odsResult = parseCustom(parser, aOperand, bOperand); -// CHECK: if (!odsResult) return {}; +// CHECK: if (!odsResult.has_value()) return {}; // CHECK: if (::mlir::failed(*odsResult)) return ::mlir::failure(); // CHECK: return ::mlir::success(); -// CHECK: }(); result.has_value() && ::mlir::failed(*result)) { +// CHECK: }(); optResult.has_value() && ::mlir::failed(*optResult)) { // CHECK: return ::mlir::failure(); -// CHECK: } else if (result.has_value()) { +// CHECK: } else if (optResult.has_value()) { // CHECK-LABEL: OptionalGroupD::print // CHECK-NEXT: if (((getA()) || (getB()))) { diff --git a/mlir/tools/mlir-tblgen/AttrOrTypeFormatGen.cpp b/mlir/tools/mlir-tblgen/AttrOrTypeFormatGen.cpp index f8e0c83da3c8..6098808c646f 100644 --- a/mlir/tools/mlir-tblgen/AttrOrTypeFormatGen.cpp +++ b/mlir/tools/mlir-tblgen/AttrOrTypeFormatGen.cpp @@ -622,7 +622,7 @@ void DefFormat::genCustomParser(CustomDirective *el, FmtContext &ctx, } os.unindent() << ");\n"; if (isOptional) { - os << "if (!odsCustomResult) return {};\n"; + os << "if (!odsCustomResult.has_value()) return {};\n"; os << "if (::mlir::failed(*odsCustomResult)) return ::mlir::failure();\n"; } else { os << "if (::mlir::failed(odsCustomResult)) return {};\n"; diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp index eb8c0aba1d33..1ffac059f198 100644 --- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp +++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp @@ -1025,7 +1025,7 @@ static void genCustomDirectiveParser(CustomDirective *dir, MethodBody &body, body << ");\n"; if (isOptional) { - body << " if (!odsResult) return {};\n" + body << " if (!odsResult.has_value()) return {};\n" << " if (::mlir::failed(*odsResult)) return ::mlir::failure();\n"; } else { body << " if (odsResult) return ::mlir::failure();\n"; @@ -1285,13 +1285,13 @@ void OperationFormat::genElementParser(FormatElement *element, MethodBody &body, region->name); } } else if (auto *custom = dyn_cast(firstElement)) { - body << " if (auto result = [&]() -> ::mlir::OptionalParseResult {\n"; + body << " if (auto optResult = [&]() -> ::mlir::OptionalParseResult {\n"; genCustomDirectiveParser(custom, body, useProperties, opCppClassName, /*isOptional=*/true); body << " return ::mlir::success();\n" - << " }(); result.has_value() && ::mlir::failed(*result)) {\n" + << " }(); optResult.has_value() && ::mlir::failed(*optResult)) {\n" << " return ::mlir::failure();\n" - << " } else if (result.has_value()) {\n"; + << " } else if (optResult.has_value()) {\n"; } genElementParsers(firstElement, thenElements.drop_front(), -- GitLab From e25bf70d50cbf8bdebeacdaf3313486c1b1d0395 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Tue, 12 Mar 2024 21:26:58 -0700 Subject: [PATCH 0085/1407] [libc] Add an empty definition of mbstate_t (#84993) We expect to eventually provide a complete implementation, but having an empty definition is necessary to unblock the use of libc++ in embedded environments. See #84884 for more details. --- libc/config/baremetal/api.td | 4 ++++ libc/config/baremetal/arm/headers.txt | 1 + libc/config/baremetal/riscv/headers.txt | 1 + libc/include/CMakeLists.txt | 10 ++++++++++ libc/include/llvm-libc-types/CMakeLists.txt | 1 + libc/include/llvm-libc-types/mbstate_t.h | 16 ++++++++++++++++ libc/include/uchar.h.def | 16 ++++++++++++++++ libc/spec/spec.td | 2 ++ libc/spec/stdc.td | 12 ++++++++++++ 9 files changed, 63 insertions(+) create mode 100644 libc/include/llvm-libc-types/mbstate_t.h create mode 100644 libc/include/uchar.h.def diff --git a/libc/config/baremetal/api.td b/libc/config/baremetal/api.td index 33b3a03828e9..f096cdcbc9a3 100644 --- a/libc/config/baremetal/api.td +++ b/libc/config/baremetal/api.td @@ -66,3 +66,7 @@ def StdlibAPI : PublicAPI<"stdlib.h"> { def StringAPI : PublicAPI<"string.h"> { let Types = ["size_t"]; } + +def UCharAPI : PublicAPI<"uchar.h"> { + let Types = ["mbstate_t"]; +} diff --git a/libc/config/baremetal/arm/headers.txt b/libc/config/baremetal/arm/headers.txt index 68d7017fda80..962981f8a208 100644 --- a/libc/config/baremetal/arm/headers.txt +++ b/libc/config/baremetal/arm/headers.txt @@ -13,4 +13,5 @@ set(TARGET_PUBLIC_HEADERS libc.include.string libc.include.strings libc.include.sys_queue + libc.include.uchar ) diff --git a/libc/config/baremetal/riscv/headers.txt b/libc/config/baremetal/riscv/headers.txt index 68d7017fda80..962981f8a208 100644 --- a/libc/config/baremetal/riscv/headers.txt +++ b/libc/config/baremetal/riscv/headers.txt @@ -13,4 +13,5 @@ set(TARGET_PUBLIC_HEADERS libc.include.string libc.include.strings libc.include.sys_queue + libc.include.uchar ) diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt index 34d6839fd789..b2cb10459c53 100644 --- a/libc/include/CMakeLists.txt +++ b/libc/include/CMakeLists.txt @@ -582,6 +582,15 @@ add_gen_header( .llvm-libc-types.tcflag_t ) +add_gen_header( + uchar + DEF_FILE uchar.h.def + GEN_HDR uchar.h + DEPENDS + .llvm_libc_common_h + .llvm-libc-types.mbstate_t +) + add_gen_header( wchar DEF_FILE wchar.h.def @@ -589,6 +598,7 @@ add_gen_header( DEPENDS .llvm_libc_common_h .llvm-libc-macros.wchar_macros + .llvm-libc-types.mbstate_t .llvm-libc-types.size_t .llvm-libc-types.wint_t .llvm-libc-types.wchar_t diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt index e4f23b2a7813..7fef976d7b32 100644 --- a/libc/include/llvm-libc-types/CMakeLists.txt +++ b/libc/include/llvm-libc-types/CMakeLists.txt @@ -39,6 +39,7 @@ add_header(uid_t HDR uid_t.h) add_header(imaxdiv_t HDR imaxdiv_t.h) add_header(ino_t HDR ino_t.h) add_header(jmp_buf HDR jmp_buf.h) +add_header(mbstate_t HDR mbstate_t.h) add_header(mode_t HDR mode_t.h) add_header(mtx_t HDR mtx_t.h DEPENDS .__futex_word .__mutex_type) add_header(nlink_t HDR nlink_t.h) diff --git a/libc/include/llvm-libc-types/mbstate_t.h b/libc/include/llvm-libc-types/mbstate_t.h new file mode 100644 index 000000000000..540d50975a26 --- /dev/null +++ b/libc/include/llvm-libc-types/mbstate_t.h @@ -0,0 +1,16 @@ +//===-- Definition of mbstate_t type --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_MBSTATE_T_H +#define LLVM_LIBC_TYPES_MBSTATE_T_H + +// TODO: Complete this once we implement functions that operate on this type. +typedef struct { +} mbstate_t; + +#endif // LLVM_LIBC_TYPES_MBSTATE_T_H diff --git a/libc/include/uchar.h.def b/libc/include/uchar.h.def new file mode 100644 index 000000000000..7e62d43e9cc4 --- /dev/null +++ b/libc/include/uchar.h.def @@ -0,0 +1,16 @@ +//===-- C standard library header uchar.h ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_UCHAR_H +#define LLVM_LIBC_UCHAR_H + +#include <__llvm-libc-common.h> + +%%public_api() + +#endif // LLVM_LIBC_UCHAR_H diff --git a/libc/spec/spec.td b/libc/spec/spec.td index 580bd9c8c3c1..87bf4435e167 100644 --- a/libc/spec/spec.td +++ b/libc/spec/spec.td @@ -153,6 +153,8 @@ def EntryType : NamedType<"ENTRY">; def EntryTypePtr : PtrType; def EntryTypePtrPtr : PtrType; +def MBStateTType : NamedType<"mbstate_t">; + class Macro { string Name = name; } diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 938d3722a0f8..afe01b1bb685 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -1284,12 +1284,23 @@ def StdC : StandardSpec<"stdc"> { ] >; + HeaderSpec UChar = HeaderSpec< + "uchar.h", + [], // Macros + [ //Types + MBStateTType, + ], + [], // Enumerations + [] + >; + HeaderSpec WChar = HeaderSpec< "wchar.h", [ // Macros Macro<"WEOF">, ], [ //Types + MBStateTType, SizeTType, WIntType, WCharType, @@ -1324,6 +1335,7 @@ def StdC : StandardSpec<"stdc"> { Signal, Threads, Time, + UChar, WChar, ]; } -- GitLab From a0283987d07c2d4ce2cc5a4adaee0512f9553797 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Tue, 12 Mar 2024 21:33:49 -0700 Subject: [PATCH 0086/1407] [libc] Include additional baremetal entrypoints (#85020) These functions are usable on embedded platforms and are sometimes used in various baremetal projects. --- libc/config/baremetal/api.td | 1 + libc/config/baremetal/arm/entrypoints.txt | 45 +++++++++++++++++++++ libc/config/baremetal/riscv/entrypoints.txt | 44 ++++++++++++++++++++ 3 files changed, 90 insertions(+) diff --git a/libc/config/baremetal/api.td b/libc/config/baremetal/api.td index f096cdcbc9a3..80d0e0ba22ca 100644 --- a/libc/config/baremetal/api.td +++ b/libc/config/baremetal/api.td @@ -2,6 +2,7 @@ include "config/public_api.td" include "spec/stdc.td" include "spec/stdc_ext.td" +include "spec/bsd_ext.td" include "spec/llvm_libc_stdfix_ext.td" def AssertMacro : MacroDef<"assert"> { diff --git a/libc/config/baremetal/arm/entrypoints.txt b/libc/config/baremetal/arm/entrypoints.txt index 6e4fdb036264..589ec5237e98 100644 --- a/libc/config/baremetal/arm/entrypoints.txt +++ b/libc/config/baremetal/arm/entrypoints.txt @@ -30,6 +30,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.bcmp libc.src.string.bcopy libc.src.string.bzero + libc.src.string.index libc.src.string.memccpy libc.src.string.memchr libc.src.string.memcmp @@ -39,6 +40,8 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.mempcpy libc.src.string.memrchr libc.src.string.memset + libc.src.string.memset_explicit + libc.src.string.rindex libc.src.string.stpcpy libc.src.string.stpncpy libc.src.string.strcasecmp @@ -47,8 +50,11 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strchr libc.src.string.strchrnul libc.src.string.strcmp + libc.src.string.strcoll libc.src.string.strcpy libc.src.string.strcspn + libc.src.string.strerror + libc.src.string.strerror_r libc.src.string.strlcat libc.src.string.strlcpy libc.src.string.strlen @@ -59,10 +65,12 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strnlen libc.src.string.strpbrk libc.src.string.strrchr + libc.src.string.strsep libc.src.string.strspn libc.src.string.strstr libc.src.string.strtok libc.src.string.strtok_r + libc.src.string.strxfrm # inttypes.h entrypoints libc.src.inttypes.imaxabs @@ -117,6 +125,36 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdbit.stdc_first_trailing_one_ui libc.src.stdbit.stdc_first_trailing_one_ul libc.src.stdbit.stdc_first_trailing_one_ull + libc.src.stdbit.stdc_count_zeros_uc + libc.src.stdbit.stdc_count_zeros_us + libc.src.stdbit.stdc_count_zeros_ui + libc.src.stdbit.stdc_count_zeros_ul + libc.src.stdbit.stdc_count_zeros_ull + libc.src.stdbit.stdc_count_ones_uc + libc.src.stdbit.stdc_count_ones_us + libc.src.stdbit.stdc_count_ones_ui + libc.src.stdbit.stdc_count_ones_ul + libc.src.stdbit.stdc_count_ones_ull + libc.src.stdbit.stdc_has_single_bit_uc + libc.src.stdbit.stdc_has_single_bit_us + libc.src.stdbit.stdc_has_single_bit_ui + libc.src.stdbit.stdc_has_single_bit_ul + libc.src.stdbit.stdc_has_single_bit_ull + libc.src.stdbit.stdc_bit_width_uc + libc.src.stdbit.stdc_bit_width_us + libc.src.stdbit.stdc_bit_width_ui + libc.src.stdbit.stdc_bit_width_ul + libc.src.stdbit.stdc_bit_width_ull + libc.src.stdbit.stdc_bit_floor_uc + libc.src.stdbit.stdc_bit_floor_us + libc.src.stdbit.stdc_bit_floor_ui + libc.src.stdbit.stdc_bit_floor_ul + libc.src.stdbit.stdc_bit_floor_ull + libc.src.stdbit.stdc_bit_ceil_uc + libc.src.stdbit.stdc_bit_ceil_us + libc.src.stdbit.stdc_bit_ceil_ui + libc.src.stdbit.stdc_bit_ceil_ul + libc.src.stdbit.stdc_bit_ceil_ull # stdlib.h entrypoints libc.src.stdlib.abort @@ -132,6 +170,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.llabs libc.src.stdlib.lldiv libc.src.stdlib.qsort + libc.src.stdlib.qsort_r + libc.src.stdlib.rand + libc.src.stdlib.srand libc.src.stdlib.strtod libc.src.stdlib.strtof libc.src.stdlib.strtol @@ -201,6 +242,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.fminl libc.src.math.fmod libc.src.math.fmodf + libc.src.math.fmodl libc.src.math.frexp libc.src.math.frexpf libc.src.math.frexpl @@ -212,6 +254,9 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.ldexp libc.src.math.ldexpf libc.src.math.ldexpl + libc.src.math.llogb + libc.src.math.llogbf + libc.src.math.llogbl libc.src.math.llrint libc.src.math.llrintf libc.src.math.llrintl diff --git a/libc/config/baremetal/riscv/entrypoints.txt b/libc/config/baremetal/riscv/entrypoints.txt index 6e4fdb036264..09de1b416e0e 100644 --- a/libc/config/baremetal/riscv/entrypoints.txt +++ b/libc/config/baremetal/riscv/entrypoints.txt @@ -30,6 +30,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.bcmp libc.src.string.bcopy libc.src.string.bzero + libc.src.string.index libc.src.string.memccpy libc.src.string.memchr libc.src.string.memcmp @@ -39,6 +40,8 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.mempcpy libc.src.string.memrchr libc.src.string.memset + libc.src.string.memset_explicit + libc.src.string.rindex libc.src.string.stpcpy libc.src.string.stpncpy libc.src.string.strcasecmp @@ -47,8 +50,11 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strchr libc.src.string.strchrnul libc.src.string.strcmp + libc.src.string.strcoll libc.src.string.strcpy libc.src.string.strcspn + libc.src.string.strerror + libc.src.string.strerror_r libc.src.string.strlcat libc.src.string.strlcpy libc.src.string.strlen @@ -59,10 +65,12 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strnlen libc.src.string.strpbrk libc.src.string.strrchr + libc.src.string.strsep libc.src.string.strspn libc.src.string.strstr libc.src.string.strtok libc.src.string.strtok_r + libc.src.string.strxfrm # inttypes.h entrypoints libc.src.inttypes.imaxabs @@ -117,6 +125,36 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdbit.stdc_first_trailing_one_ui libc.src.stdbit.stdc_first_trailing_one_ul libc.src.stdbit.stdc_first_trailing_one_ull + libc.src.stdbit.stdc_count_zeros_uc + libc.src.stdbit.stdc_count_zeros_us + libc.src.stdbit.stdc_count_zeros_ui + libc.src.stdbit.stdc_count_zeros_ul + libc.src.stdbit.stdc_count_zeros_ull + libc.src.stdbit.stdc_count_ones_uc + libc.src.stdbit.stdc_count_ones_us + libc.src.stdbit.stdc_count_ones_ui + libc.src.stdbit.stdc_count_ones_ul + libc.src.stdbit.stdc_count_ones_ull + libc.src.stdbit.stdc_has_single_bit_uc + libc.src.stdbit.stdc_has_single_bit_us + libc.src.stdbit.stdc_has_single_bit_ui + libc.src.stdbit.stdc_has_single_bit_ul + libc.src.stdbit.stdc_has_single_bit_ull + libc.src.stdbit.stdc_bit_width_uc + libc.src.stdbit.stdc_bit_width_us + libc.src.stdbit.stdc_bit_width_ui + libc.src.stdbit.stdc_bit_width_ul + libc.src.stdbit.stdc_bit_width_ull + libc.src.stdbit.stdc_bit_floor_uc + libc.src.stdbit.stdc_bit_floor_us + libc.src.stdbit.stdc_bit_floor_ui + libc.src.stdbit.stdc_bit_floor_ul + libc.src.stdbit.stdc_bit_floor_ull + libc.src.stdbit.stdc_bit_ceil_uc + libc.src.stdbit.stdc_bit_ceil_us + libc.src.stdbit.stdc_bit_ceil_ui + libc.src.stdbit.stdc_bit_ceil_ul + libc.src.stdbit.stdc_bit_ceil_ull # stdlib.h entrypoints libc.src.stdlib.abort @@ -132,6 +170,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.llabs libc.src.stdlib.lldiv libc.src.stdlib.qsort + libc.src.stdlib.qsort_r + libc.src.stdlib.rand + libc.src.stdlib.srand libc.src.stdlib.strtod libc.src.stdlib.strtof libc.src.stdlib.strtol @@ -212,6 +253,9 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.ldexp libc.src.math.ldexpf libc.src.math.ldexpl + libc.src.math.llogb + libc.src.math.llogbf + libc.src.math.llogbl libc.src.math.llrint libc.src.math.llrintf libc.src.math.llrintl -- GitLab From deebf6b312227e028dd3258b162306b9cdb21cf7 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Tue, 12 Mar 2024 22:21:15 -0700 Subject: [PATCH 0087/1407] [tsan] Disabled test dead locking on glibc-2.38 https://github.com/google/sanitizers/issues/1733 --- compiler-rt/test/lit.common.cfg.py | 2 +- compiler-rt/test/tsan/getline_nohang.cpp | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py index ae28681915af..bd9b926c1505 100644 --- a/compiler-rt/test/lit.common.cfg.py +++ b/compiler-rt/test/lit.common.cfg.py @@ -632,7 +632,7 @@ if config.host_os == "Linux": ver = LooseVersion(ver_string) any_glibc = False - for required in ["2.19", "2.27", "2.30", "2.33", "2.34", "2.37"]: + for required in ["2.19", "2.27", "2.30", "2.33", "2.34", "2.37", "2.38"]: if ver >= LooseVersion(required): config.available_features.add("glibc-" + required) any_glibc = True diff --git a/compiler-rt/test/tsan/getline_nohang.cpp b/compiler-rt/test/tsan/getline_nohang.cpp index d1bb279a450f..c0762da96abb 100644 --- a/compiler-rt/test/tsan/getline_nohang.cpp +++ b/compiler-rt/test/tsan/getline_nohang.cpp @@ -5,6 +5,10 @@ // Make sure TSan doesn't deadlock on a file stream lock at program shutdown. // See https://github.com/google/sanitizers/issues/454 + +// https://github.com/google/sanitizers/issues/1733 +// UNSUPPORTED: glibc-2.38 + #ifdef __FreeBSD__ #define _WITH_GETLINE // to declare getline() #endif -- GitLab From 53613044ddf1bab186cde8d687f7f41bc0b1f9b1 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Tue, 12 Mar 2024 22:28:32 -0700 Subject: [PATCH 0088/1407] [llvm-objcopy] Use SmallVector to make some structs smaller. NFC --- llvm/include/llvm/ObjCopy/CommonConfig.h | 17 ++++++++--------- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/llvm/include/llvm/ObjCopy/CommonConfig.h b/llvm/include/llvm/ObjCopy/CommonConfig.h index 383395941475..3f894b0cd958 100644 --- a/llvm/include/llvm/ObjCopy/CommonConfig.h +++ b/llvm/include/llvm/ObjCopy/CommonConfig.h @@ -22,7 +22,6 @@ // Necessary for llvm::DebugCompressionType::None #include "llvm/Target/TargetOptions.h" #include -#include namespace llvm { namespace objcopy { @@ -126,8 +125,8 @@ public: // provided for that option. class NameMatcher { DenseSet PosNames; - std::vector PosPatterns; - std::vector NegMatchers; + SmallVector PosPatterns; + SmallVector NegMatchers; public: Error addMatcher(Expected Matcher) { @@ -179,8 +178,8 @@ struct NewSymbolInfo { StringRef SymbolName; StringRef SectionName; uint64_t Value = 0; - std::vector Flags; - std::vector BeforeSyms; + SmallVector Flags; + SmallVector BeforeSyms; }; // Specify section name and section body for newly added or updated section. @@ -218,9 +217,9 @@ struct CommonConfig { DiscardType DiscardMode = DiscardType::None; // Repeated options - std::vector AddSection; - std::vector DumpSection; - std::vector UpdateSection; + SmallVector AddSection; + SmallVector DumpSection; + SmallVector UpdateSection; // Section matchers NameMatcher KeepSection; @@ -244,7 +243,7 @@ struct CommonConfig { StringMap SymbolsToRename; // Symbol info specified by --add-symbol option. - std::vector SymbolsToAdd; + SmallVector SymbolsToAdd; // Boolean options bool DeterministicArchives = true; diff --git a/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp b/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp index 6318578b1100..c5c7cc254d79 100644 --- a/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp +++ b/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp @@ -528,7 +528,7 @@ static Expected parseNewSymbolInfo(StringRef FlagValue) { // ArgValue, loads data from the file, and stores section name and data // into the vector of new sections \p NewSections. static Error loadNewSectionData(StringRef ArgValue, StringRef OptionName, - std::vector &NewSections) { + SmallVector &NewSections) { if (!ArgValue.contains('=')) return createStringError(errc::invalid_argument, "bad format for " + OptionName + ": missing '='"); -- GitLab From 0d98582c8b86644e77f8ddd68fc251e41127b7f4 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Tue, 12 Mar 2024 22:31:06 -0700 Subject: [PATCH 0089/1407] [llvm-objcopy] Remove unneeded #include. NFC --- llvm/include/llvm/ObjCopy/CommonConfig.h | 3 +-- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/ObjCopy/CommonConfig.h b/llvm/include/llvm/ObjCopy/CommonConfig.h index 3f894b0cd958..8f69c9fbeaf5 100644 --- a/llvm/include/llvm/ObjCopy/CommonConfig.h +++ b/llvm/include/llvm/ObjCopy/CommonConfig.h @@ -16,11 +16,10 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/Object/ELFTypes.h" +#include "llvm/Support/Compression.h" #include "llvm/Support/GlobPattern.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Regex.h" -// Necessary for llvm::DebugCompressionType::None -#include "llvm/Target/TargetOptions.h" #include namespace llvm { diff --git a/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp b/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp index c5c7cc254d79..a0c6415bf0e6 100644 --- a/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp +++ b/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp @@ -10,6 +10,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/ObjCopy/CommonConfig.h" #include "llvm/ObjCopy/ConfigManager.h" -- GitLab From 4d62929852849f768d7397f634cfdebc85de96a4 Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Wed, 13 Mar 2024 13:48:32 +0800 Subject: [PATCH 0090/1407] [C++20] [Modules] Disambuguous Clang module and C++20 Named module further This patch tries to make the boundary of clang module and C++20 named module more clear. The changes included: - Rename `TranslationUnitKind::TU_Module` to `TranslationUnitKind::TU_ClangModule`. - Rename `Sema::ActOnModuleInclude` to `Sema::ActOnAnnotModuleInclude`. - Rename `ActOnModuleBegin` to `Sema::ActOnAnnotModuleBegin`. - Rename `Sema::ActOnModuleEnd` to `Sema::ActOnAnnotModuleEnd`. - Removes a warning if we're trying to compile a non-module unit as C++20 module unit. This is not actually useful and makes (the future) implementation unnecessarily complex. This patch meant to be a NFC fix. But it shows that it fixed a bug suprisingly that previously we would surppress the unused-value warning in named modules. Because it shares the same logic with clang modules, which has headers semantics. This shows the change is meaningful. --- .../clang/Basic/DiagnosticSemaKinds.td | 2 - clang/include/clang/Basic/LangOptions.h | 9 +-- .../include/clang/Frontend/FrontendActions.h | 8 ++- clang/include/clang/Sema/Sema.h | 6 +- clang/lib/Parse/Parser.cpp | 30 ++++----- clang/lib/Sema/Sema.cpp | 63 ++++++++----------- clang/lib/Sema/SemaModule.cpp | 10 +-- .../dcl.module/dcl.module.interface/p1.cppm | 4 -- .../Modules/missing-module-declaration.cppm | 13 ---- clang/test/Modules/pr72828.cppm | 2 +- 10 files changed, 58 insertions(+), 89 deletions(-) delete mode 100644 clang/test/Modules/missing-module-declaration.cppm diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index c54105507753..d7ab1635cf12 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -11516,8 +11516,6 @@ def err_module_not_defined : Error< def err_module_redeclaration : Error< "translation unit contains multiple module declarations">; def note_prev_module_declaration : Note<"previous module declaration is here">; -def err_module_declaration_missing : Error< - "missing 'export module' declaration in module interface unit">; def err_module_declaration_missing_after_global_module_introducer : Error< "missing 'module' declaration at end of global module fragment " "introduced here">; diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h index 862952d336ef..08fc706e3cbf 100644 --- a/clang/include/clang/Basic/LangOptions.h +++ b/clang/include/clang/Basic/LangOptions.h @@ -560,11 +560,6 @@ public: return getCompilingModule() != CMK_None; } - /// Are we compiling a standard c++ module interface? - bool isCompilingModuleInterface() const { - return getCompilingModule() == CMK_ModuleInterface; - } - /// Are we compiling a module implementation? bool isCompilingModuleImplementation() const { return !isCompilingModule() && !ModuleName.empty(); @@ -993,8 +988,8 @@ enum TranslationUnitKind { /// not complete. TU_Prefix, - /// The translation unit is a module. - TU_Module, + /// The translation unit is a clang module. + TU_ClangModule, /// The translation unit is a is a complete translation unit that we might /// incrementally extend later. diff --git a/clang/include/clang/Frontend/FrontendActions.h b/clang/include/clang/Frontend/FrontendActions.h index 8441af2ee3e7..a620ddfc4044 100644 --- a/clang/include/clang/Frontend/FrontendActions.h +++ b/clang/include/clang/Frontend/FrontendActions.h @@ -125,7 +125,7 @@ protected: StringRef InFile) override; TranslationUnitKind getTranslationUnitKind() override { - return TU_Module; + return TU_ClangModule; } bool hasASTFileSupport() const override { return false; } @@ -138,7 +138,9 @@ protected: std::unique_ptr CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override; - TranslationUnitKind getTranslationUnitKind() override { return TU_Module; } + TranslationUnitKind getTranslationUnitKind() override { + return TU_ClangModule; + } bool hasASTFileSupport() const override { return false; } }; @@ -159,6 +161,8 @@ protected: std::unique_ptr CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override; + TranslationUnitKind getTranslationUnitKind() override { return TU_Complete; } + std::unique_ptr CreateOutputFile(CompilerInstance &CI, StringRef InFile) override; }; diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index b226851f0303..d6ab2b0c2def 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -8064,13 +8064,13 @@ public: /// The parser has processed a module import translated from a /// #include or similar preprocessing directive. - void ActOnModuleInclude(SourceLocation DirectiveLoc, Module *Mod); + void ActOnAnnotModuleInclude(SourceLocation DirectiveLoc, Module *Mod); void BuildModuleInclude(SourceLocation DirectiveLoc, Module *Mod); /// The parsed has entered a submodule. - void ActOnModuleBegin(SourceLocation DirectiveLoc, Module *Mod); + void ActOnAnnotModuleBegin(SourceLocation DirectiveLoc, Module *Mod); /// The parser has left a submodule. - void ActOnModuleEnd(SourceLocation DirectiveLoc, Module *Mod); + void ActOnAnnotModuleEnd(SourceLocation DirectiveLoc, Module *Mod); /// Create an implicit import of the given module at the given /// source location, for error recovery, if possible. diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp index 1701d153bd0e..cc0e41ed221c 100644 --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -685,7 +685,7 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result, // FIXME: We need a better way to disambiguate C++ clang modules and // standard C++ modules. if (!getLangOpts().CPlusPlusModules || !Mod->isHeaderUnit()) - Actions.ActOnModuleInclude(Loc, Mod); + Actions.ActOnAnnotModuleInclude(Loc, Mod); else { DeclResult Import = Actions.ActOnModuleImport(Loc, SourceLocation(), Loc, Mod); @@ -697,15 +697,17 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result, } case tok::annot_module_begin: - Actions.ActOnModuleBegin(Tok.getLocation(), reinterpret_cast( - Tok.getAnnotationValue())); + Actions.ActOnAnnotModuleBegin( + Tok.getLocation(), + reinterpret_cast(Tok.getAnnotationValue())); ConsumeAnnotationToken(); ImportState = Sema::ModuleImportState::NotACXX20Module; return false; case tok::annot_module_end: - Actions.ActOnModuleEnd(Tok.getLocation(), reinterpret_cast( - Tok.getAnnotationValue())); + Actions.ActOnAnnotModuleEnd( + Tok.getLocation(), + reinterpret_cast(Tok.getAnnotationValue())); ConsumeAnnotationToken(); ImportState = Sema::ModuleImportState::NotACXX20Module; return false; @@ -2708,9 +2710,9 @@ bool Parser::parseMisplacedModuleImport() { // happens. if (MisplacedModuleBeginCount) { --MisplacedModuleBeginCount; - Actions.ActOnModuleEnd(Tok.getLocation(), - reinterpret_cast( - Tok.getAnnotationValue())); + Actions.ActOnAnnotModuleEnd( + Tok.getLocation(), + reinterpret_cast(Tok.getAnnotationValue())); ConsumeAnnotationToken(); continue; } @@ -2720,18 +2722,18 @@ bool Parser::parseMisplacedModuleImport() { return true; case tok::annot_module_begin: // Recover by entering the module (Sema will diagnose). - Actions.ActOnModuleBegin(Tok.getLocation(), - reinterpret_cast( - Tok.getAnnotationValue())); + Actions.ActOnAnnotModuleBegin( + Tok.getLocation(), + reinterpret_cast(Tok.getAnnotationValue())); ConsumeAnnotationToken(); ++MisplacedModuleBeginCount; continue; case tok::annot_module_include: // Module import found where it should not be, for instance, inside a // namespace. Recover by importing the module. - Actions.ActOnModuleInclude(Tok.getLocation(), - reinterpret_cast( - Tok.getAnnotationValue())); + Actions.ActOnAnnotModuleInclude( + Tok.getLocation(), + reinterpret_cast(Tok.getAnnotationValue())); ConsumeAnnotationToken(); // If there is another module import, process it. continue; diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp index 720d5fd5f042..cd0c42d5ffba 100644 --- a/clang/lib/Sema/Sema.cpp +++ b/clang/lib/Sema/Sema.cpp @@ -1207,26 +1207,35 @@ void Sema::ActOnEndOfTranslationUnit() { } // A global-module-fragment is only permitted within a module unit. - bool DiagnosedMissingModuleDeclaration = false; if (!ModuleScopes.empty() && ModuleScopes.back().Module->Kind == Module::ExplicitGlobalModuleFragment) { Diag(ModuleScopes.back().BeginLoc, diag::err_module_declaration_missing_after_global_module_introducer); - DiagnosedMissingModuleDeclaration = true; - } - - if (TUKind == TU_Module) { - // If we are building a module interface unit, we need to have seen the - // module declaration by now. - if (getLangOpts().getCompilingModule() == - LangOptions::CMK_ModuleInterface && - !isCurrentModulePurview() && !DiagnosedMissingModuleDeclaration) { - // FIXME: Make a better guess as to where to put the module declaration. - Diag(getSourceManager().getLocForStartOfFile( - getSourceManager().getMainFileID()), - diag::err_module_declaration_missing); - } + } + + // Now we can decide whether the modules we're building need an initializer. + if (Module *CurrentModule = getCurrentModule(); + CurrentModule && CurrentModule->isInterfaceOrPartition()) { + auto DoesModNeedInit = [this](Module *M) { + if (!getASTContext().getModuleInitializers(M).empty()) + return true; + for (auto [Exported, _] : M->Exports) + if (Exported->isNamedModuleInterfaceHasInit()) + return true; + for (Module *I : M->Imports) + if (I->isNamedModuleInterfaceHasInit()) + return true; + + return false; + }; + CurrentModule->NamedModuleHasInit = + DoesModNeedInit(CurrentModule) || + llvm::any_of(CurrentModule->submodules(), + [&](auto *SubM) { return DoesModNeedInit(SubM); }); + } + + if (TUKind == TU_ClangModule) { // If we are building a module, resolve all of the exported declarations // now. if (Module *CurrentModule = PP.getCurrentModule()) { @@ -1251,28 +1260,6 @@ void Sema::ActOnEndOfTranslationUnit() { } } - // Now we can decide whether the modules we're building need an initializer. - if (Module *CurrentModule = getCurrentModule(); - CurrentModule && CurrentModule->isInterfaceOrPartition()) { - auto DoesModNeedInit = [this](Module *M) { - if (!getASTContext().getModuleInitializers(M).empty()) - return true; - for (auto [Exported, _] : M->Exports) - if (Exported->isNamedModuleInterfaceHasInit()) - return true; - for (Module *I : M->Imports) - if (I->isNamedModuleInterfaceHasInit()) - return true; - - return false; - }; - - CurrentModule->NamedModuleHasInit = - DoesModNeedInit(CurrentModule) || - llvm::any_of(CurrentModule->submodules(), - [&](auto *SubM) { return DoesModNeedInit(SubM); }); - } - // Warnings emitted in ActOnEndOfTranslationUnit() should be emitted for // modules when they are built, not every time they are used. emitAndClearUnusedLocalTypedefWarnings(); @@ -1358,7 +1345,7 @@ void Sema::ActOnEndOfTranslationUnit() { // noise. Don't warn for a use from a module: either we should warn on all // file-scope declarations in modules or not at all, but whether the // declaration is used is immaterial. - if (!Diags.hasErrorOccurred() && TUKind != TU_Module) { + if (!Diags.hasErrorOccurred() && TUKind != TU_ClangModule) { // Output warning for unused file scoped decls. for (UnusedFileScopedDeclsType::iterator I = UnusedFileScopedDecls.begin(ExternalSource.get()), diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp index f08c1cb3a13e..2ddf9d70263a 100644 --- a/clang/lib/Sema/SemaModule.cpp +++ b/clang/lib/Sema/SemaModule.cpp @@ -713,7 +713,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc, return Import; } -void Sema::ActOnModuleInclude(SourceLocation DirectiveLoc, Module *Mod) { +void Sema::ActOnAnnotModuleInclude(SourceLocation DirectiveLoc, Module *Mod) { checkModuleImportContext(*this, Mod, DirectiveLoc, CurContext, true); BuildModuleInclude(DirectiveLoc, Mod); } @@ -723,9 +723,9 @@ void Sema::BuildModuleInclude(SourceLocation DirectiveLoc, Module *Mod) { // in that buffer do not qualify as module imports; they're just an // implementation detail of us building the module. // - // FIXME: Should we even get ActOnModuleInclude calls for those? + // FIXME: Should we even get ActOnAnnotModuleInclude calls for those? bool IsInModuleIncludes = - TUKind == TU_Module && + TUKind == TU_ClangModule && getSourceManager().isWrittenInMainFile(DirectiveLoc); // If we are really importing a module (not just checking layering) due to an @@ -752,7 +752,7 @@ void Sema::BuildModuleInclude(SourceLocation DirectiveLoc, Module *Mod) { } } -void Sema::ActOnModuleBegin(SourceLocation DirectiveLoc, Module *Mod) { +void Sema::ActOnAnnotModuleBegin(SourceLocation DirectiveLoc, Module *Mod) { checkModuleImportContext(*this, Mod, DirectiveLoc, CurContext, true); ModuleScopes.push_back({}); @@ -776,7 +776,7 @@ void Sema::ActOnModuleBegin(SourceLocation DirectiveLoc, Module *Mod) { } } -void Sema::ActOnModuleEnd(SourceLocation EomLoc, Module *Mod) { +void Sema::ActOnAnnotModuleEnd(SourceLocation EomLoc, Module *Mod) { if (getLangOpts().ModulesLocalVisibility) { VisibleModules = std::move(ModuleScopes.back().OuterVisibleModules); // Leaving a module hides namespace names, so our visible namespace cache diff --git a/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.interface/p1.cppm b/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.interface/p1.cppm index 3072b760f9d7..1a01ffac0154 100644 --- a/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.interface/p1.cppm +++ b/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.interface/p1.cppm @@ -15,10 +15,6 @@ module A; // #module-decl // expected-error@-2 {{missing 'export' specifier in module declaration while building module interface}} #define INTERFACE #endif -#else - #ifdef BUILT_AS_INTERFACE - // expected-error@1 {{missing 'export module' declaration in module interface unit}} - #endif #endif #ifndef INTERFACE diff --git a/clang/test/Modules/missing-module-declaration.cppm b/clang/test/Modules/missing-module-declaration.cppm deleted file mode 100644 index d52f6639fe4f..000000000000 --- a/clang/test/Modules/missing-module-declaration.cppm +++ /dev/null @@ -1,13 +0,0 @@ -// RUN: rm -rf %t -// RUN: split-file %s %t -// RUN: cd %t -// -// RUN: %clang_cc1 -std=c++20 %t/B.cppm -I%t -emit-module-interface -o %t/B.pcm -// RUN: %clang_cc1 -std=c++20 %t/A.cppm -I%t -fprebuilt-module-path=%t -emit-module-interface -verify - -//--- A.cppm -import B; // expected-error{{missing 'export module' declaration in module interface unit}} - -//--- B.cppm -module; -export module B; diff --git a/clang/test/Modules/pr72828.cppm b/clang/test/Modules/pr72828.cppm index 574523188507..7432f2831f24 100644 --- a/clang/test/Modules/pr72828.cppm +++ b/clang/test/Modules/pr72828.cppm @@ -17,7 +17,7 @@ struct s { void f() { auto [x] = s(); - [x] {}; + (void) [x] {}; } // Check that we can generate the LLVM IR expectedly. -- GitLab From fe1d02b08ce2ca74de5b18d9f141d503b7985ec5 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Tue, 12 Mar 2024 23:09:36 -0700 Subject: [PATCH 0091/1407] [sanitizer] Reject unsupported -static at link time (#83524) Most sanitizers don't support static linking. One primary reason is the incompatibility with interceptors. `GetTlsSize` is another reason. asan/memprof use `__interception::DoesNotSupportStaticLinking` (`_DYNAMIC` reference) to reject -static at link time. Port this detector to other sanitizers. dfsan actually supports -static for certain cases. Don't touch dfsan. --- compiler-rt/lib/hwasan/hwasan_interceptors.cpp | 1 + compiler-rt/lib/lsan/lsan_interceptors.cpp | 1 + compiler-rt/lib/msan/msan_interceptors.cpp | 2 ++ compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp | 2 ++ compiler-rt/test/sanitizer_common/TestCases/Linux/static-link.c | 2 ++ 5 files changed, 8 insertions(+) create mode 100644 compiler-rt/test/sanitizer_common/TestCases/Linux/static-link.c diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp index 96df4dd0c24d..d519ac3a459b 100644 --- a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp +++ b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp @@ -520,6 +520,7 @@ void InitializeInterceptors() { CHECK_EQ(inited, 0); # if HWASAN_WITH_INTERCEPTORS + __interception::DoesNotSupportStaticLinking(); InitializeCommonInterceptors(); (void)(read_iovec); diff --git a/compiler-rt/lib/lsan/lsan_interceptors.cpp b/compiler-rt/lib/lsan/lsan_interceptors.cpp index 885f7ad5ddba..1fd0010f9ea9 100644 --- a/compiler-rt/lib/lsan/lsan_interceptors.cpp +++ b/compiler-rt/lib/lsan/lsan_interceptors.cpp @@ -543,6 +543,7 @@ namespace __lsan { void InitializeInterceptors() { // Fuchsia doesn't use interceptors that require any setup. #if !SANITIZER_FUCHSIA + __interception::DoesNotSupportStaticLinking(); InitializeSignalInterceptors(); INTERCEPT_FUNCTION(malloc); diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp index 2c9f2c01e14b..6e0b2bf2ef5b 100644 --- a/compiler-rt/lib/msan/msan_interceptors.cpp +++ b/compiler-rt/lib/msan/msan_interceptors.cpp @@ -1762,6 +1762,8 @@ void InitializeInterceptors() { static int inited = 0; CHECK_EQ(inited, 0); + __interception::DoesNotSupportStaticLinking(); + new(interceptor_ctx()) InterceptorContext(); InitializeCommonInterceptors(); diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp index 24309ab66b2e..8ffc703b05ea 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp @@ -2861,6 +2861,8 @@ void InitializeInterceptors() { REAL(memcpy) = internal_memcpy; #endif + __interception::DoesNotSupportStaticLinking(); + new(interceptor_ctx()) InterceptorContext(); // Interpose __tls_get_addr before the common interposers. This is needed diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/static-link.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/static-link.c new file mode 100644 index 000000000000..f45f718cb08d --- /dev/null +++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/static-link.c @@ -0,0 +1,2 @@ +// UNSUPPORTED: hwasan, ubsan +// RUN: not %clangxx -static %s -o /dev/null -- GitLab From cd2f6163137dce45d909aa445cfd57b7188f8ed1 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Wed, 13 Mar 2024 12:42:15 +0530 Subject: [PATCH 0092/1407] AMDGPU: Use list-table for metadata table (#85024) The table syntax for sphinx is really insufferably whitespace dependent. I've been meaning to convert the existing attribute and intrinsic tables to use list-table, which is less painful to merge. --- llvm/docs/AMDGPUUsage.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index 99d7a482710f..fd9ad7fac19a 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -1317,15 +1317,16 @@ LLVM IR Metadata The AMDGPU backend implements the following LLVM IR metadata. -.. table:: AMDGPU LLVM IR Metadata +.. list-table:: AMDGPU LLVM IR Metatdata :name: amdgpu-llvm-ir-metadata-table - ============================================== ========================================================== - LLVM IR Metadata Description - ============================================== ========================================================== - !amdgpu.last.use Sets TH_LOAD_LU temporal hint on load instructions that support it. - Takes priority over nontemporal hint (TH_LOAD_NT). - ============================================== ========================================================== + * - Metadata Name + - Description + - Values + * - !amdgpu.last.use + - Sets TH_LOAD_LU temporal hint on load instructions that support it. + Takes priority over nontemporal hint (TH_LOAD_NT). + - {} LLVM IR Attributes ------------------ -- GitLab From 0a443f13b49b3f392461a0bb60b0146cfc4607c7 Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Wed, 13 Mar 2024 08:32:01 +0100 Subject: [PATCH 0093/1407] [SPIR-V] Add implementation of G_SPLAT_VECTOR opcode and fix invalid types processing (#84766) This PR: * adds support for G_SPLAT_VECTOR generic opcode that may be legally generated instead of G_BUILD_VECTOR by previous passes of the translator (see https://github.com/llvm/llvm-project/pull/80378 for the source of breaking changes); * improves deduction of types for opaque pointers. This PR also fixes the following issues: * if a function has ptr argument(s), two functions that have different SPIR-V type definitions may get identical LLVM function types and break agreements of global register and duplicate checker; * checks for pointer types do not account for TypedPointerType. Update of tests: * A test case is added to cover the issue with function ptr parameters. * The first case, that is support for G_SPLAT_VECTOR generic opcode, is covered by existing test cases. * Multiple additional checks by `spirv-val` is added to cover more possibilities of generation of invalid code. --- llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp | 49 ++++++- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 136 +++++++++++++----- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 32 +++-- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 3 +- .../Target/SPIRV/SPIRVInstructionSelector.cpp | 41 ++++++ llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp | 4 +- llvm/lib/Target/SPIRV/SPIRVUtils.h | 26 ++++ llvm/test/CodeGen/SPIRV/ComparePointers.ll | 1 + llvm/test/CodeGen/SPIRV/capability-kernel.ll | 1 + .../pointers/getelementptr-addressspace.ll | 1 + .../SPIRV/pointers/getelementptr-base-type.ll | 1 + .../kernel-argument-pointer-addressspace.ll | 1 + ...er-type-deduction-no-bitcast-to-generic.ll | 1 + .../pointers/kernel-argument-pointer-type.ll | 1 + .../SPIRV/pointers/load-addressspace.ll | 1 + .../pointers/store-operand-ptr-to-struct.ll | 1 + .../SPIRV/pointers/struct-opaque-pointers.ll | 2 +- .../pointers/two-bitcast-or-param-users.ll | 1 + .../SPIRV/pointers/two-subsequent-bitcasts.ll | 1 + .../SPIRV/pointers/type-deduce-by-call-rev.ll | 28 ++++ .../SPIRV/pointers/type-deduce-by-call.ll | 28 ++++ .../CodeGen/SPIRV/pointers/typeof-ptr-int.ll | 29 ++++ llvm/test/CodeGen/SPIRV/relationals.ll | 1 + llvm/test/CodeGen/SPIRV/simple.ll | 1 + .../AtomicCompareExchangeExplicit_cl20.ll | 1 + .../SPIRV/transcoding/BitReversePref.ll | 1 + .../CodeGen/SPIRV/transcoding/BuildNDRange.ll | 1 + .../SPIRV/transcoding/BuildNDRange_2.ll | 1 + .../CodeGen/SPIRV/transcoding/ConvertPtr.ll | 1 + .../SPIRV/transcoding/DecorationAlignment.ll | 1 + .../transcoding/DecorationMaxByteOffset.ll | 1 + llvm/test/CodeGen/SPIRV/transcoding/DivRem.ll | 1 + .../ExecutionMode_SPIR_to_SPIRV.ll | 1 + .../SPIRV/transcoding/GlobalFunAnnotate.ll | 1 + .../transcoding/OpenCL/atomic_cmpxchg.ll | 1 + .../SPIRV/transcoding/OpenCL/atomic_legacy.ll | 1 + .../OpenCL/atomic_work_item_fence.ll | 1 + .../SPIRV/transcoding/OpenCL/barrier.ll | 1 + .../transcoding/OpenCL/sub_group_mask.ll | 1 + .../transcoding/OpenCL/work_group_barrier.ll | 1 + .../CodeGen/SPIRV/transcoding/atomic_flag.ll | 1 + .../SPIRV/transcoding/atomic_load_store.ll | 1 + .../test/CodeGen/SPIRV/transcoding/bitcast.ll | 1 + .../transcoding/block_w_struct_return.ll | 1 + .../SPIRV/transcoding/builtin_calls.ll | 1 + .../CodeGen/SPIRV/transcoding/builtin_vars.ll | 1 + .../transcoding/builtin_vars_arithmetics.ll | 1 + .../SPIRV/transcoding/builtin_vars_opt.ll | 1 + .../SPIRV/transcoding/check_ro_qualifier.ll | 1 + .../CodeGen/SPIRV/transcoding/cl-types.ll | 1 + .../CodeGen/SPIRV/transcoding/clk_event_t.ll | 1 + .../SPIRV/transcoding/enqueue_kernel.ll | 1 + .../SPIRV/transcoding/explicit-conversions.ll | 1 + .../SPIRV/transcoding/extract_insert_value.ll | 1 + llvm/test/CodeGen/SPIRV/transcoding/fadd.ll | 1 + llvm/test/CodeGen/SPIRV/transcoding/fclamp.ll | 1 + llvm/test/CodeGen/SPIRV/transcoding/fcmp.ll | 1 + llvm/test/CodeGen/SPIRV/transcoding/fdiv.ll | 1 + llvm/test/CodeGen/SPIRV/transcoding/fmod.ll | 1 + llvm/test/CodeGen/SPIRV/transcoding/fmul.ll | 1 + llvm/test/CodeGen/SPIRV/transcoding/fneg.ll | 1 + .../fp_contract_reassoc_fast_mode.ll | 1 + llvm/test/CodeGen/SPIRV/transcoding/frem.ll | 1 + llvm/test/CodeGen/SPIRV/transcoding/fsub.ll | 1 + .../transcoding/get_image_num_mip_levels.ll | 1 + .../CodeGen/SPIRV/transcoding/global_block.ll | 1 + .../CodeGen/SPIRV/transcoding/group_ops.ll | 1 + .../test/CodeGen/SPIRV/transcoding/isequal.ll | 1 + .../SPIRV/transcoding/relationals_double.ll | 1 + .../SPIRV/transcoding/relationals_float.ll | 1 + .../SPIRV/transcoding/relationals_half.ll | 1 + 71 files changed, 382 insertions(+), 56 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-rev.ll create mode 100644 llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call.ll create mode 100644 llvm/test/CodeGen/SPIRV/pointers/typeof-ptr-int.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp index 2d7a00bab38e..f1fbe2ba1bc4 100644 --- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp @@ -85,6 +85,42 @@ static ConstantInt *getConstInt(MDNode *MD, unsigned NumOp) { return nullptr; } +// If the function has pointer arguments, we are forced to re-create this +// function type from the very beginning, changing PointerType by +// TypedPointerType for each pointer argument. Otherwise, the same `Type*` +// potentially corresponds to different SPIR-V function type, effectively +// invalidating logic behind global registry and duplicates tracker. +static FunctionType * +fixFunctionTypeIfPtrArgs(SPIRVGlobalRegistry *GR, const Function &F, + FunctionType *FTy, const SPIRVType *SRetTy, + const SmallVector &SArgTys) { + if (F.getParent()->getNamedMetadata("spv.cloned_funcs")) + return FTy; + + bool hasArgPtrs = false; + for (auto &Arg : F.args()) { + // check if it's an instance of a non-typed PointerType + if (Arg.getType()->isPointerTy()) { + hasArgPtrs = true; + break; + } + } + if (!hasArgPtrs) { + Type *RetTy = FTy->getReturnType(); + // check if it's an instance of a non-typed PointerType + if (!RetTy->isPointerTy()) + return FTy; + } + + // re-create function type, using TypedPointerType instead of PointerType to + // properly trace argument types + const Type *RetTy = GR->getTypeForSPIRVType(SRetTy); + SmallVector ArgTys; + for (auto SArgTy : SArgTys) + ArgTys.push_back(const_cast(GR->getTypeForSPIRVType(SArgTy))); + return FunctionType::get(const_cast(RetTy), ArgTys, false); +} + // This code restores function args/retvalue types for composite cases // because the final types should still be aggregate whereas they're i32 // during the translation to cope with aggregate flattening etc. @@ -162,7 +198,7 @@ static SPIRVType *getArgSPIRVType(const Function &F, unsigned ArgIdx, // If OriginalArgType is non-pointer, use the OriginalArgType (the type cannot // be legally reassigned later). - if (!OriginalArgType->isPointerTy()) + if (!isPointerTy(OriginalArgType)) return GR->getOrCreateSPIRVType(OriginalArgType, MIRBuilder, ArgAccessQual); // In case OriginalArgType is of pointer type, there are three possibilities: @@ -179,8 +215,7 @@ static SPIRVType *getArgSPIRVType(const Function &F, unsigned ArgIdx, SPIRVType *ElementType = GR->getOrCreateSPIRVType(ByValRefType, MIRBuilder); return GR->getOrCreateSPIRVPointerType( ElementType, MIRBuilder, - addressSpaceToStorageClass(Arg->getType()->getPointerAddressSpace(), - ST)); + addressSpaceToStorageClass(getPointerAddressSpace(Arg->getType()), ST)); } for (auto User : Arg->users()) { @@ -240,7 +275,6 @@ bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, static_cast(&MIRBuilder.getMF().getSubtarget()); // Assign types and names to all args, and store their types for later. - FunctionType *FTy = getOriginalFunctionType(F); SmallVector ArgTypeVRegs; if (VRegs.size() > 0) { unsigned i = 0; @@ -255,7 +289,7 @@ bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, if (Arg.hasName()) buildOpName(VRegs[i][0], Arg.getName(), MIRBuilder); - if (Arg.getType()->isPointerTy()) { + if (isPointerTy(Arg.getType())) { auto DerefBytes = static_cast(Arg.getDereferenceableBytes()); if (DerefBytes != 0) buildOpDecorate(VRegs[i][0], MIRBuilder, @@ -322,7 +356,9 @@ bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, MRI->setRegClass(FuncVReg, &SPIRV::IDRegClass); if (F.isDeclaration()) GR->add(&F, &MIRBuilder.getMF(), FuncVReg); + FunctionType *FTy = getOriginalFunctionType(F); SPIRVType *RetTy = GR->getOrCreateSPIRVType(FTy->getReturnType(), MIRBuilder); + FTy = fixFunctionTypeIfPtrArgs(GR, F, FTy, RetTy, ArgTypeVRegs); SPIRVType *FuncTy = GR->getOrCreateOpTypeFunctionWithArgs( FTy, RetTy, ArgTypeVRegs, MIRBuilder); uint32_t FuncControl = getFunctionControl(F); @@ -429,7 +465,6 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, return false; MachineFunction &MF = MIRBuilder.getMF(); GR->setCurrentFunc(MF); - FunctionType *FTy = nullptr; const Function *CF = nullptr; std::string DemangledName; const Type *OrigRetTy = Info.OrigRet.Ty; @@ -444,7 +479,7 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, // TODO: support constexpr casts and indirect calls. if (CF == nullptr) return false; - if ((FTy = getOriginalFunctionType(*CF)) != nullptr) + if (FunctionType *FTy = getOriginalFunctionType(*CF)) OrigRetTy = FTy->getReturnType(); } diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 575e903d05bb..c5b901235402 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -57,8 +57,14 @@ class SPIRVEmitIntrinsics bool TrackConstants = true; DenseMap AggrConsts; DenseSet AggrStores; + + // deduce values type + DenseMap DeducedElTys; + Type *deduceElementType(Value *I); + void preprocessCompositeConstants(IRBuilder<> &B); void preprocessUndefs(IRBuilder<> &B); + CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef Types, Value *Arg, Value *Arg2, ArrayRef Imms, IRBuilder<> &B) { @@ -72,6 +78,7 @@ class SPIRVEmitIntrinsics Args.push_back(Imm); return B.CreateIntrinsic(IntrID, {Types}, Args); } + void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B); void processInstrAfterVisit(Instruction *I, IRBuilder<> &B); void insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B); @@ -156,6 +163,48 @@ static inline void reportFatalOnTokenType(const Instruction *I) { false); } +// Deduce and return a successfully deduced Type of the Instruction, +// or nullptr otherwise. +static Type *deduceElementTypeHelper(Value *I, + std::unordered_set &Visited, + DenseMap &DeducedElTys) { + // maybe already known + auto It = DeducedElTys.find(I); + if (It != DeducedElTys.end()) + return It->second; + + // maybe a cycle + if (Visited.find(I) != Visited.end()) + return nullptr; + Visited.insert(I); + + // fallback value in case when we fail to deduce a type + Type *Ty = nullptr; + // look for known basic patterns of type inference + if (auto *Ref = dyn_cast(I)) + Ty = Ref->getAllocatedType(); + else if (auto *Ref = dyn_cast(I)) + Ty = Ref->getResultElementType(); + else if (auto *Ref = dyn_cast(I)) + Ty = Ref->getValueType(); + else if (auto *Ref = dyn_cast(I)) + Ty = deduceElementTypeHelper(Ref->getPointerOperand(), Visited, + DeducedElTys); + + // remember the found relationship + if (Ty) + DeducedElTys[I] = Ty; + + return Ty; +} + +Type *SPIRVEmitIntrinsics::deduceElementType(Value *I) { + std::unordered_set Visited; + if (Type *Ty = deduceElementTypeHelper(I, Visited, DeducedElTys)) + return Ty; + return IntegerType::getInt8Ty(I->getContext()); +} + void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B) { @@ -280,7 +329,7 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) { // varying element types. In case of IR coming from older versions of LLVM // such bitcasts do not provide sufficient information, should be just skipped // here, and handled in insertPtrCastOrAssignTypeInstr. - if (I.getType()->isPointerTy()) { + if (isPointerTy(I.getType())) { I.replaceAllUsesWith(Source); I.eraseFromParent(); return nullptr; @@ -333,20 +382,10 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( while (BitCastInst *BC = dyn_cast(Pointer)) Pointer = BC->getOperand(0); - // Do not emit spv_ptrcast if Pointer is a GlobalValue of expected type. - GlobalValue *GV = dyn_cast(Pointer); - if (GV && GV->getValueType() == ExpectedElementType) - return; - - // Do not emit spv_ptrcast if Pointer is a result of alloca with expected - // type. - AllocaInst *A = dyn_cast(Pointer); - if (A && A->getAllocatedType() == ExpectedElementType) - return; - - // Do not emit spv_ptrcast if Pointer is a result of GEP of expected type. - GetElementPtrInst *GEPI = dyn_cast(Pointer); - if (GEPI && GEPI->getResultElementType() == ExpectedElementType) + // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType + std::unordered_set Visited; + Type *PointerElemTy = deduceElementTypeHelper(Pointer, Visited, DeducedElTys); + if (PointerElemTy == ExpectedElementType) return; setInsertPointSkippingPhis(B, I); @@ -356,7 +395,7 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( ValueAsMetadata::getConstant(ExpectedElementTypeConst); MDTuple *TyMD = MDNode::get(F->getContext(), CM); MetadataAsValue *VMD = MetadataAsValue::get(F->getContext(), TyMD); - unsigned AddressSpace = Pointer->getType()->getPointerAddressSpace(); + unsigned AddressSpace = getPointerAddressSpace(Pointer->getType()); bool FirstPtrCastOrAssignPtrType = true; // Do not emit new spv_ptrcast if equivalent one already exists or when @@ -401,9 +440,11 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( // spv_assign_ptr_type instead. if (FirstPtrCastOrAssignPtrType && (isa(Pointer) || isa(Pointer))) { - buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {Pointer->getType()}, - ExpectedElementTypeConst, Pointer, - {B.getInt32(AddressSpace)}, B); + CallInst *CI = buildIntrWithMD( + Intrinsic::spv_assign_ptr_type, {Pointer->getType()}, + ExpectedElementTypeConst, Pointer, {B.getInt32(AddressSpace)}, B); + DeducedElTys[CI] = ExpectedElementType; + DeducedElTys[Pointer] = ExpectedElementType; return; } @@ -419,7 +460,7 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I, // Handle basic instructions: StoreInst *SI = dyn_cast(I); if (SI && F->getCallingConv() == CallingConv::SPIR_KERNEL && - SI->getValueOperand()->getType()->isPointerTy() && + isPointerTy(SI->getValueOperand()->getType()) && isa(SI->getValueOperand())) { return replacePointerOperandWithPtrCast( I, SI->getValueOperand(), IntegerType::getInt8Ty(F->getContext()), 0, @@ -440,9 +481,34 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I, if (!CI || CI->isIndirectCall() || CI->getCalledFunction()->isIntrinsic()) return; + // collect information about formal parameter types + Function *CalledF = CI->getCalledFunction(); + SmallVector CalledArgTys; + bool HaveTypes = false; + for (auto &CalledArg : CalledF->args()) { + if (!isPointerTy(CalledArg.getType())) { + CalledArgTys.push_back(nullptr); + continue; + } + auto It = DeducedElTys.find(&CalledArg); + Type *ParamTy = It != DeducedElTys.end() ? It->second : nullptr; + if (!ParamTy) { + for (User *U : CalledArg.users()) { + if (Instruction *Inst = dyn_cast(U)) { + std::unordered_set Visited; + ParamTy = deduceElementTypeHelper(Inst, Visited, DeducedElTys); + if (ParamTy) + break; + } + } + } + HaveTypes |= ParamTy != nullptr; + CalledArgTys.push_back(ParamTy); + } + std::string DemangledName = getOclOrSpirvBuiltinDemangledName(CI->getCalledFunction()->getName()); - if (DemangledName.empty()) + if (DemangledName.empty() && !HaveTypes) return; for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) { @@ -455,8 +521,11 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I, if (!isa(ArgOperand) && !isa(ArgOperand)) continue; - Type *ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType( - DemangledName, OpIdx, I->getContext()); + Type *ExpectedType = + OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr; + if (!ExpectedType && !DemangledName.empty()) + ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType( + DemangledName, OpIdx, I->getContext()); if (!ExpectedType) continue; @@ -639,30 +708,25 @@ void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV, void SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B) { reportFatalOnTokenType(I); - if (!I->getType()->isPointerTy() || !requireAssignType(I) || + if (!isPointerTy(I->getType()) || !requireAssignType(I) || isa(I)) return; setInsertPointSkippingPhis(B, I->getNextNode()); - Constant *EltTyConst; - unsigned AddressSpace = I->getType()->getPointerAddressSpace(); - if (auto *AI = dyn_cast(I)) - EltTyConst = UndefValue::get(AI->getAllocatedType()); - else if (auto *GEP = dyn_cast(I)) - EltTyConst = UndefValue::get(GEP->getResultElementType()); - else - EltTyConst = UndefValue::get(IntegerType::getInt8Ty(I->getContext())); - - buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {I->getType()}, EltTyConst, I, - {B.getInt32(AddressSpace)}, B); + Type *ElemTy = deduceElementType(I); + Constant *EltTyConst = UndefValue::get(ElemTy); + unsigned AddressSpace = getPointerAddressSpace(I->getType()); + CallInst *CI = buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {I->getType()}, + EltTyConst, I, {B.getInt32(AddressSpace)}, B); + DeducedElTys[CI] = ElemTy; } void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B) { reportFatalOnTokenType(I); Type *Ty = I->getType(); - if (!Ty->isVoidTy() && !Ty->isPointerTy() && requireAssignType(I)) { + if (!Ty->isVoidTy() && !isPointerTy(Ty) && requireAssignType(I)) { setInsertPointSkippingPhis(B, I->getNextNode()); Type *TypeToAssign = Ty; if (auto *II = dyn_cast(I)) { diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 8556581996fe..bda9c57e534c 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -750,7 +750,7 @@ SPIRVType *SPIRVGlobalRegistry::createSPIRVType( SPIRVType *SPIRVGlobalRegistry::restOfCreateSPIRVType( const Type *Ty, MachineIRBuilder &MIRBuilder, SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) { - if (TypesInProcessing.count(Ty) && !Ty->isPointerTy()) + if (TypesInProcessing.count(Ty) && !isPointerTy(Ty)) return nullptr; TypesInProcessing.insert(Ty); SPIRVType *SpirvType = createSPIRVType(Ty, MIRBuilder, AccessQual, EmitIR); @@ -762,11 +762,15 @@ SPIRVType *SPIRVGlobalRegistry::restOfCreateSPIRVType( // will be added later. For special types it is already added to DT. if (SpirvType->getOpcode() != SPIRV::OpTypeForwardPointer && !Reg.isValid() && !isSpecialOpaqueType(Ty)) { - if (!Ty->isPointerTy()) + if (!isPointerTy(Ty)) DT.add(Ty, &MIRBuilder.getMF(), getSPIRVTypeID(SpirvType)); + else if (isTypedPointerTy(Ty)) + DT.add(cast(Ty)->getElementType(), + getPointerAddressSpace(Ty), &MIRBuilder.getMF(), + getSPIRVTypeID(SpirvType)); else DT.add(Type::getInt8Ty(MIRBuilder.getMF().getFunction().getContext()), - Ty->getPointerAddressSpace(), &MIRBuilder.getMF(), + getPointerAddressSpace(Ty), &MIRBuilder.getMF(), getSPIRVTypeID(SpirvType)); } @@ -787,12 +791,15 @@ SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVType( const Type *Ty, MachineIRBuilder &MIRBuilder, SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) { Register Reg; - if (!Ty->isPointerTy()) + if (!isPointerTy(Ty)) Reg = DT.find(Ty, &MIRBuilder.getMF()); + else if (isTypedPointerTy(Ty)) + Reg = DT.find(cast(Ty)->getElementType(), + getPointerAddressSpace(Ty), &MIRBuilder.getMF()); else Reg = DT.find(Type::getInt8Ty(MIRBuilder.getMF().getFunction().getContext()), - Ty->getPointerAddressSpace(), &MIRBuilder.getMF()); + getPointerAddressSpace(Ty), &MIRBuilder.getMF()); if (Reg.isValid() && !isSpecialOpaqueType(Ty)) return getSPIRVTypeForVReg(Reg); @@ -836,11 +843,16 @@ bool SPIRVGlobalRegistry::isScalarOrVectorOfType(Register VReg, unsigned SPIRVGlobalRegistry::getScalarOrVectorComponentCount(Register VReg) const { - if (SPIRVType *Type = getSPIRVTypeForVReg(VReg)) - return Type->getOpcode() == SPIRV::OpTypeVector - ? static_cast(Type->getOperand(2).getImm()) - : 1; - return 0; + return getScalarOrVectorComponentCount(getSPIRVTypeForVReg(VReg)); +} + +unsigned +SPIRVGlobalRegistry::getScalarOrVectorComponentCount(SPIRVType *Type) const { + if (!Type) + return 0; + return Type->getOpcode() == SPIRV::OpTypeVector + ? static_cast(Type->getOperand(2).getImm()) + : 1; } unsigned diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h index 9c0061d13fd0..25d82ebf9bc7 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -198,9 +198,10 @@ public: // opcode (e.g. OpTypeBool, or OpTypeVector %x 4, where %x is OpTypeBool). bool isScalarOrVectorOfType(Register VReg, unsigned TypeOpcode) const; - // Return number of elements in a vector if the given VReg is associated with + // Return number of elements in a vector if the argument is associated with // a vector type. Return 1 for a scalar type, and 0 for a missing type. unsigned getScalarOrVectorComponentCount(Register VReg) const; + unsigned getScalarOrVectorComponentCount(SPIRVType *Type) const; // For vectors or scalars of booleans, integers and floats, return the scalar // type's bitwidth. Otherwise calls llvm_unreachable(). diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 74df8de6eb90..fd19b7412c4c 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -125,6 +125,8 @@ private: bool selectConstVector(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; + bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, + MachineInstr &I) const; bool selectCmp(Register ResVReg, const SPIRVType *ResType, unsigned comparisonOpcode, MachineInstr &I) const; @@ -313,6 +315,8 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg, case TargetOpcode::G_BUILD_VECTOR: return selectConstVector(ResVReg, ResType, I); + case TargetOpcode::G_SPLAT_VECTOR: + return selectSplatVector(ResVReg, ResType, I); case TargetOpcode::G_SHUFFLE_VECTOR: { MachineBasicBlock &BB = *I.getParent(); @@ -1185,6 +1189,43 @@ bool SPIRVInstructionSelector::selectConstVector(Register ResVReg, return MIB.constrainAllUses(TII, TRI, RBI); } +bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, + const SPIRVType *ResType, + MachineInstr &I) const { + if (ResType->getOpcode() != SPIRV::OpTypeVector) + report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); + unsigned N = GR.getScalarOrVectorComponentCount(ResType); + unsigned OpIdx = I.getNumExplicitDefs(); + if (!I.getOperand(OpIdx).isReg()) + report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); + + // check if we may construct a constant vector + Register OpReg = I.getOperand(OpIdx).getReg(); + bool IsConst = false; + if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) { + if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && + OpDef->getOperand(1).isReg()) { + if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) + OpDef = RefDef; + } + IsConst = OpDef->getOpcode() == TargetOpcode::G_CONSTANT || + OpDef->getOpcode() == TargetOpcode::G_FCONSTANT; + } + + if (!IsConst && N < 2) + report_fatal_error( + "There must be at least two constituent operands in a vector"); + + auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), + TII.get(IsConst ? SPIRV::OpConstantComposite + : SPIRV::OpCompositeConstruct)) + .addDef(ResVReg) + .addUse(GR.getSPIRVTypeID(ResType)); + for (unsigned i = 0; i < N; ++i) + MIB.addUse(OpReg); + return MIB.constrainAllUses(TII, TRI, RBI); +} + bool SPIRVInstructionSelector::selectCmp(Register ResVReg, const SPIRVType *ResType, unsigned CmpOpc, diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp index f81548742a11..4b871bdd5d07 100644 --- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp @@ -149,7 +149,9 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) { getActionDefinitionsBuilder(G_GLOBAL_VALUE).alwaysLegal(); // TODO: add proper rules for vectors legalization. - getActionDefinitionsBuilder({G_BUILD_VECTOR, G_SHUFFLE_VECTOR}).alwaysLegal(); + getActionDefinitionsBuilder( + {G_BUILD_VECTOR, G_SHUFFLE_VECTOR, G_SPLAT_VECTOR}) + .alwaysLegal(); // Vector Reduction Operations getActionDefinitionsBuilder( diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h index e5f35aaca9a8..d5ed501def99 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.h +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h @@ -15,6 +15,7 @@ #include "MCTargetDesc/SPIRVBaseInfo.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/IR/TypedPointerType.h" #include namespace llvm { @@ -100,5 +101,30 @@ bool isEntryPoint(const Function &F); // Parse basic scalar type name, substring TypeName, and return LLVM type. Type *parseBasicTypeName(StringRef TypeName, LLVMContext &Ctx); + +// True if this is an instance of TypedPointerType. +inline bool isTypedPointerTy(const Type *T) { + return T->getTypeID() == Type::TypedPointerTyID; +} + +// True if this is an instance of PointerType. +inline bool isUntypedPointerTy(const Type *T) { + return T->getTypeID() == Type::PointerTyID; +} + +// True if this is an instance of PointerType or TypedPointerType. +inline bool isPointerTy(const Type *T) { + return isUntypedPointerTy(T) || isTypedPointerTy(T); +} + +// Get the address space of this pointer or pointer vector type for instances of +// PointerType or TypedPointerType. +inline unsigned getPointerAddressSpace(const Type *T) { + Type *SubT = T->getScalarType(); + return SubT->getTypeID() == Type::PointerTyID + ? cast(SubT)->getAddressSpace() + : cast(SubT)->getAddressSpace(); +} + } // namespace llvm #endif // LLVM_LIB_TARGET_SPIRV_SPIRVUTILS_H diff --git a/llvm/test/CodeGen/SPIRV/ComparePointers.ll b/llvm/test/CodeGen/SPIRV/ComparePointers.ll index fd2084dbc260..9be05944789b 100644 --- a/llvm/test/CodeGen/SPIRV/ComparePointers.ll +++ b/llvm/test/CodeGen/SPIRV/ComparePointers.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --mattr=+spirv1.3 %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; kernel void test(int global *in, int global *in2) { ;; if (!in) diff --git a/llvm/test/CodeGen/SPIRV/capability-kernel.ll b/llvm/test/CodeGen/SPIRV/capability-kernel.ll index 03ea58c985ad..fea19511d4fd 100644 --- a/llvm/test/CodeGen/SPIRV/capability-kernel.ll +++ b/llvm/test/CodeGen/SPIRV/capability-kernel.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-DAG: OpCapability Addresses diff --git a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-addressspace.ll b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-addressspace.ll index 062863a0e3ad..7e9c6214c281 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-addressspace.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-addressspace.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK: %[[#INT8:]] = OpTypeInt 8 0 ; CHECK: %[[#PTR1:]] = OpTypePointer CrossWorkgroup %[[#INT8]] diff --git a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-base-type.ll b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-base-type.ll index aaf97f8cc836..fc999ba1a3cd 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-base-type.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-base-type.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK: %[[#FLOAT32:]] = OpTypeFloat 32 ; CHECK: %[[#PTR:]] = OpTypePointer CrossWorkgroup %[[#FLOAT32]] diff --git a/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-addressspace.ll b/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-addressspace.ll index 6d1202328197..a3a730ac67e7 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-addressspace.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-addressspace.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-DAG: %[[#INT:]] = OpTypeInt 32 0 ; CHECK-DAG: %[[#PTR1:]] = OpTypePointer Function %[[#INT]] diff --git a/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type-deduction-no-bitcast-to-generic.ll b/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type-deduction-no-bitcast-to-generic.ll index 9e136ce88746..b74a3449980d 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type-deduction-no-bitcast-to-generic.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type-deduction-no-bitcast-to-generic.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-DAG: %[[#IMAGE:]] = OpTypeImage %2 2D 0 0 0 0 Unknown ReadOnly diff --git a/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type.ll b/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type.ll index 1fcc6d9da9c7..b8f205a68e56 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-DAG: %[[#FLOAT32:]] = OpTypeFloat 32 ; CHECK-DAG: %[[#PTR1:]] = OpTypePointer Function %[[#FLOAT32]] diff --git a/llvm/test/CodeGen/SPIRV/pointers/load-addressspace.ll b/llvm/test/CodeGen/SPIRV/pointers/load-addressspace.ll index 1b4e7a3e733f..1667abc51be9 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/load-addressspace.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/load-addressspace.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK: %[[#INT8:]] = OpTypeInt 8 0 ; CHECK: %[[#PTR1:]] = OpTypePointer CrossWorkgroup %[[#INT8]] diff --git a/llvm/test/CodeGen/SPIRV/pointers/store-operand-ptr-to-struct.ll b/llvm/test/CodeGen/SPIRV/pointers/store-operand-ptr-to-struct.ll index 00b03c08e7bb..3a0d65e1e95f 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/store-operand-ptr-to-struct.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/store-operand-ptr-to-struct.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; TODO: OpFunctionParameter should be a pointer of struct base type. ; XFAIL: * diff --git a/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll b/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll index 86f5f5bf24f5..d426fc4dfd4e 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll @@ -1,5 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s -; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} +; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK: %[[TyInt8:.*]] = OpTypeInt 8 0 ; CHECK: %[[TyInt8Ptr:.*]] = OpTypePointer {{[a-zA-Z]+}} %[[TyInt8]] diff --git a/llvm/test/CodeGen/SPIRV/pointers/two-bitcast-or-param-users.ll b/llvm/test/CodeGen/SPIRV/pointers/two-bitcast-or-param-users.ll index 52180d537408..23c3faaf8815 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/two-bitcast-or-param-users.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/two-bitcast-or-param-users.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-DAG: %[[#INT:]] = OpTypeInt 32 ; CHECK-DAG: %[[#GLOBAL_PTR_INT:]] = OpTypePointer CrossWorkgroup %[[#INT]] diff --git a/llvm/test/CodeGen/SPIRV/pointers/two-subsequent-bitcasts.ll b/llvm/test/CodeGen/SPIRV/pointers/two-subsequent-bitcasts.ll index 473c2a8b7311..83234e3986c8 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/two-subsequent-bitcasts.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/two-subsequent-bitcasts.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-DAG: %[[#float:]] = OpTypeFloat 32 ; CHECK-DAG: %[[#pointer:]] = OpTypePointer CrossWorkgroup %[[#float]] diff --git a/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-rev.ll b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-rev.ll new file mode 100644 index 000000000000..76769ab87430 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-rev.ll @@ -0,0 +1,28 @@ +; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-SPIRV-DAG: OpName %[[FooArg:.*]] "known_type_ptr" +; CHECK-SPIRV-DAG: OpName %[[Foo:.*]] "foo" +; CHECK-SPIRV-DAG: OpName %[[ArgToDeduce:.*]] "unknown_type_ptr" +; CHECK-SPIRV-DAG: OpName %[[Bar:.*]] "bar" +; CHECK-SPIRV-DAG: %[[Long:.*]] = OpTypeInt 32 0 +; CHECK-SPIRV-DAG: %[[Void:.*]] = OpTypeVoid +; CHECK-SPIRV-DAG: %[[LongPtr:.*]] = OpTypePointer CrossWorkgroup %[[Long]] +; CHECK-SPIRV-DAG: %[[Fun:.*]] = OpTypeFunction %[[Void]] %[[LongPtr]] +; CHECK-SPIRV: %[[Bar]] = OpFunction %[[Void]] None %[[Fun]] +; CHECK-SPIRV: %[[ArgToDeduce]] = OpFunctionParameter %[[LongPtr]] +; CHECK-SPIRV: OpFunctionCall %[[Void]] %[[Foo]] %[[ArgToDeduce]] +; CHECK-SPIRV: %[[Foo]] = OpFunction %[[Void]] None %[[Fun]] +; CHECK-SPIRV: %[[FooArg]] = OpFunctionParameter %[[LongPtr]] + +define spir_kernel void @bar(ptr addrspace(1) %unknown_type_ptr) { +entry: + call spir_func void @foo(ptr addrspace(1) %unknown_type_ptr) + ret void +} + +define void @foo(ptr addrspace(1) %known_type_ptr) { +entry: + %elem = getelementptr inbounds i32, ptr addrspace(1) %known_type_ptr, i64 0 + ret void +} diff --git a/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call.ll b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call.ll new file mode 100644 index 000000000000..8cbf360a2e38 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call.ll @@ -0,0 +1,28 @@ +; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-SPIRV-DAG: OpName %[[FooArg:.*]] "known_type_ptr" +; CHECK-SPIRV-DAG: OpName %[[Foo:.*]] "foo" +; CHECK-SPIRV-DAG: OpName %[[ArgToDeduce:.*]] "unknown_type_ptr" +; CHECK-SPIRV-DAG: OpName %[[Bar:.*]] "bar" +; CHECK-SPIRV-DAG: %[[Long:.*]] = OpTypeInt 32 0 +; CHECK-SPIRV-DAG: %[[Void:.*]] = OpTypeVoid +; CHECK-SPIRV-DAG: %[[LongPtr:.*]] = OpTypePointer CrossWorkgroup %[[Long]] +; CHECK-SPIRV-DAG: %[[Fun:.*]] = OpTypeFunction %[[Void]] %[[LongPtr]] +; CHECK-SPIRV: %[[Foo]] = OpFunction %[[Void]] None %[[Fun]] +; CHECK-SPIRV: %[[FooArg]] = OpFunctionParameter %[[LongPtr]] +; CHECK-SPIRV: %[[Bar]] = OpFunction %[[Void]] None %[[Fun]] +; CHECK-SPIRV: %[[ArgToDeduce]] = OpFunctionParameter %[[LongPtr]] +; CHECK-SPIRV: OpFunctionCall %[[Void]] %[[Foo]] %[[ArgToDeduce]] + +define void @foo(ptr addrspace(1) %known_type_ptr) { +entry: + %elem = getelementptr inbounds i32, ptr addrspace(1) %known_type_ptr, i64 0 + ret void +} + +define spir_kernel void @bar(ptr addrspace(1) %unknown_type_ptr) { +entry: + call spir_func void @foo(ptr addrspace(1) %unknown_type_ptr) + ret void +} diff --git a/llvm/test/CodeGen/SPIRV/pointers/typeof-ptr-int.ll b/llvm/test/CodeGen/SPIRV/pointers/typeof-ptr-int.ll new file mode 100644 index 000000000000..f144418cf542 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/typeof-ptr-int.ll @@ -0,0 +1,29 @@ +; This test is to check that two functions have different SPIR-V type +; definitions, even though their LLVM function types are identical. + +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: OpName %[[Fun32:.*]] "tp_arg_i32" +; CHECK-DAG: OpName %[[Fun64:.*]] "tp_arg_i64" +; CHECK-DAG: %[[TyI32:.*]] = OpTypeInt 32 0 +; CHECK-DAG: %[[TyVoid:.*]] = OpTypeVoid +; CHECK-DAG: %[[TyPtr32:.*]] = OpTypePointer Function %[[TyI32]] +; CHECK-DAG: %[[TyFun32:.*]] = OpTypeFunction %[[TyVoid]] %[[TyPtr32]] +; CHECK-DAG: %[[TyI64:.*]] = OpTypeInt 64 0 +; CHECK-DAG: %[[TyPtr64:.*]] = OpTypePointer Function %[[TyI64]] +; CHECK-DAG: %[[TyFun64:.*]] = OpTypeFunction %[[TyVoid]] %[[TyPtr64]] +; CHECK-DAG: %[[Fun32]] = OpFunction %[[TyVoid]] None %[[TyFun32]] +; CHECK-DAG: %[[Fun64]] = OpFunction %[[TyVoid]] None %[[TyFun64]] + +define spir_kernel void @tp_arg_i32(ptr %ptr) { +entry: + store i32 1, ptr %ptr + ret void +} + +define spir_kernel void @tp_arg_i64(ptr %ptr) { +entry: + store i64 1, ptr %ptr + ret void +} diff --git a/llvm/test/CodeGen/SPIRV/relationals.ll b/llvm/test/CodeGen/SPIRV/relationals.ll index 1644dc7c03d9..f4fcf4d9f77b 100644 --- a/llvm/test/CodeGen/SPIRV/relationals.ll +++ b/llvm/test/CodeGen/SPIRV/relationals.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} declare dso_local spir_func <4 x i8> @_Z13__spirv_IsNanIDv4_aDv4_fET_T0_(<4 x float>) declare dso_local spir_func <4 x i8> @_Z13__spirv_IsInfIDv4_aDv4_fET_T0_(<4 x float>) diff --git a/llvm/test/CodeGen/SPIRV/simple.ll b/llvm/test/CodeGen/SPIRV/simple.ll index de9efa838385..63c15968c725 100644 --- a/llvm/test/CodeGen/SPIRV/simple.ll +++ b/llvm/test/CodeGen/SPIRV/simple.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; Support of doubles is required. ; CHECK: OpCapability Float64 diff --git a/llvm/test/CodeGen/SPIRV/transcoding/AtomicCompareExchangeExplicit_cl20.ll b/llvm/test/CodeGen/SPIRV/transcoding/AtomicCompareExchangeExplicit_cl20.ll index fdb26bab60fe..55cfcea999d8 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/AtomicCompareExchangeExplicit_cl20.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/AtomicCompareExchangeExplicit_cl20.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; __kernel void testAtomicCompareExchangeExplicit_cl20( ;; volatile global atomic_int* object, diff --git a/llvm/test/CodeGen/SPIRV/transcoding/BitReversePref.ll b/llvm/test/CodeGen/SPIRV/transcoding/BitReversePref.ll index 55161e670ca1..11b0578a0c9c 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/BitReversePref.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/BitReversePref.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-linux %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK: OpDecorate %[[#FUNC_NAME:]] LinkageAttributes "_Z10BitReversei" ; CHECK-NOT: OpBitReverse diff --git a/llvm/test/CodeGen/SPIRV/transcoding/BuildNDRange.ll b/llvm/test/CodeGen/SPIRV/transcoding/BuildNDRange.ll index 95f3673d1c96..b63c1c60d007 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/BuildNDRange.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/BuildNDRange.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV-DAG: %[[#]] = OpBuildNDRange %[[#]] %[[#GWS:]] %[[#LWS:]] %[[#GWO:]] ; CHECK-SPIRV-DAG: %[[#GWS]] = OpConstant %[[#]] 123 diff --git a/llvm/test/CodeGen/SPIRV/transcoding/BuildNDRange_2.ll b/llvm/test/CodeGen/SPIRV/transcoding/BuildNDRange_2.ll index a2ae808259a3..65c992c9b28e 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/BuildNDRange_2.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/BuildNDRange_2.ll @@ -19,6 +19,7 @@ ;; bash$ $PATH_TO_GEN/bin/clang -cc1 -x cl -cl-std=CL2.0 -triple spir64-unknown-unknown -emit-llvm -include opencl-20.h BuildNDRange_2.cl -o BuildNDRange_2.ll ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; TODO(#60133): Requires updates following opaque pointer migration. ; XFAIL: * diff --git a/llvm/test/CodeGen/SPIRV/transcoding/ConvertPtr.ll b/llvm/test/CodeGen/SPIRV/transcoding/ConvertPtr.ll index 34036951e31e..93aecc5331aa 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/ConvertPtr.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/ConvertPtr.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; kernel void testConvertPtrToU(global int *a, global unsigned long *res) { ;; res[0] = (unsigned long)&a[0]; diff --git a/llvm/test/CodeGen/SPIRV/transcoding/DecorationAlignment.ll b/llvm/test/CodeGen/SPIRV/transcoding/DecorationAlignment.ll index 2e9b4a494c04..d4fc5c3280b7 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/DecorationAlignment.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/DecorationAlignment.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpDecorate %[[#ALIGNMENT:]] Alignment 16 ; CHECK-SPIRV: %[[#ALIGNMENT]] = OpFunctionParameter %[[#]] diff --git a/llvm/test/CodeGen/SPIRV/transcoding/DecorationMaxByteOffset.ll b/llvm/test/CodeGen/SPIRV/transcoding/DecorationMaxByteOffset.ll index 64f25b7f4203..966d83516bb3 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/DecorationMaxByteOffset.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/DecorationMaxByteOffset.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpName %[[#PTR_ID:]] "ptr" ; CHECK-SPIRV: OpName %[[#PTR2_ID:]] "ptr2" diff --git a/llvm/test/CodeGen/SPIRV/transcoding/DivRem.ll b/llvm/test/CodeGen/SPIRV/transcoding/DivRem.ll index 2f423c2518e8..67c338094188 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/DivRem.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/DivRem.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV-DAG: %[[#int:]] = OpTypeInt 32 0 ; CHECK-SPIRV-DAG: %[[#int2:]] = OpTypeVector %[[#int]] 2 diff --git a/llvm/test/CodeGen/SPIRV/transcoding/ExecutionMode_SPIR_to_SPIRV.ll b/llvm/test/CodeGen/SPIRV/transcoding/ExecutionMode_SPIR_to_SPIRV.ll index 6d6dd2481b17..6e8726cf03d4 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/ExecutionMode_SPIR_to_SPIRV.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/ExecutionMode_SPIR_to_SPIRV.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV-DAG: OpEntryPoint Kernel %[[#WORKER:]] "worker" ; CHECK-SPIRV-DAG: OpExecutionMode %[[#WORKER]] LocalSizeHint 128 10 1 diff --git a/llvm/test/CodeGen/SPIRV/transcoding/GlobalFunAnnotate.ll b/llvm/test/CodeGen/SPIRV/transcoding/GlobalFunAnnotate.ll index 2796dcbdca94..33bece5b9c00 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/GlobalFunAnnotate.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/GlobalFunAnnotate.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-linux %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpDecorate %[[#]] UserSemantic "annotation_on_function" diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/atomic_cmpxchg.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/atomic_cmpxchg.ll index 331960cdb341..417b89eb36f0 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/atomic_cmpxchg.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/atomic_cmpxchg.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; This test checks that the backend is capable to correctly translate ;; atomic_cmpxchg OpenCL C 1.2 built-in function [1] into corresponding SPIR-V diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/atomic_legacy.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/atomic_legacy.ll index 95eb6ade11a2..3180b57731d0 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/atomic_legacy.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/atomic_legacy.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; This test checks that the backend is capable to correctly translate ;; legacy atomic OpenCL C 1.2 built-in functions [1] into corresponding SPIR-V diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/atomic_work_item_fence.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/atomic_work_item_fence.ll index 0f3a62a3e401..c94c13044185 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/atomic_work_item_fence.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/atomic_work_item_fence.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; This test checks that the backend is capable to correctly translate ;; atomic_work_item_fence OpenCL C 2.0 built-in function [1] into corresponding diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/barrier.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/barrier.ll index a126d94e0633..cf4a24754e7b 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/barrier.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/barrier.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; This test checks that the backend is capable to correctly translate ;; barrier OpenCL C 1.2 built-in function [1] into corresponding SPIR-V diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/sub_group_mask.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/sub_group_mask.ll index 42b127cf3b69..5d9840d3bd5b 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/sub_group_mask.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/sub_group_mask.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpCapability GroupNonUniformBallot ; CHECK-SPIRV: OpDecorate %[[#]] BuiltIn SubgroupGtMask diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/work_group_barrier.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/work_group_barrier.ll index 0874e6f71e04..0702fd0c9cb9 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/work_group_barrier.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/OpenCL/work_group_barrier.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; This test checks that the backend is capable to correctly translate ;; sub_group_barrier built-in function [1] from cl_khr_subgroups extension into diff --git a/llvm/test/CodeGen/SPIRV/transcoding/atomic_flag.ll b/llvm/test/CodeGen/SPIRV/transcoding/atomic_flag.ll index 3c563d373f1b..20204acb1ef5 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/atomic_flag.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/atomic_flag.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; Types: ; CHECK-DAG: %[[#INT:]] = OpTypeInt 32 diff --git a/llvm/test/CodeGen/SPIRV/transcoding/atomic_load_store.ll b/llvm/test/CodeGen/SPIRV/transcoding/atomic_load_store.ll index d013abcade8b..3e5a3ac35693 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/atomic_load_store.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/atomic_load_store.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; Check 'LLVM ==> SPIR-V' conversion of atomic_load and atomic_store. diff --git a/llvm/test/CodeGen/SPIRV/transcoding/bitcast.ll b/llvm/test/CodeGen/SPIRV/transcoding/bitcast.ll index 8dbf4d2c58b4..2c0fc393b135 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/bitcast.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/bitcast.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; Check the bitcast is translated back to bitcast diff --git a/llvm/test/CodeGen/SPIRV/transcoding/block_w_struct_return.ll b/llvm/test/CodeGen/SPIRV/transcoding/block_w_struct_return.ll index 5ecd7f73a52e..2249cbe4e98a 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/block_w_struct_return.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/block_w_struct_return.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV1_4 +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; TODO(#60133): Requires updates following opaque pointer migration. ; XFAIL: * diff --git a/llvm/test/CodeGen/SPIRV/transcoding/builtin_calls.ll b/llvm/test/CodeGen/SPIRV/transcoding/builtin_calls.ll index 9b1ce7663180..0a02a8bf56ac 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/builtin_calls.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/builtin_calls.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV-DAG: OpDecorate %[[#Id:]] BuiltIn GlobalInvocationId ; CHECK-SPIRV-DAG: OpDecorate %[[#Id:]] BuiltIn GlobalLinearId diff --git a/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars.ll b/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars.ll index 82866712c077..f18f27a6de51 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpDecorate %[[#Id:]] BuiltIn GlobalLinearId ; CHECK-SPIRV: %[[#Id:]] = OpVariable %[[#]] diff --git a/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_arithmetics.ll b/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_arithmetics.ll index 22aa40c0c7a7..d39ca3c39383 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_arithmetics.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_arithmetics.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-linux %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; The IR was generated from the following source: ;; #include diff --git a/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_opt.ll b/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_opt.ll index 5b3474f97bfe..03456aef6b6b 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_opt.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars_opt.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-linux %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; The IR was generated from the following source: ;; #include diff --git a/llvm/test/CodeGen/SPIRV/transcoding/check_ro_qualifier.ll b/llvm/test/CodeGen/SPIRV/transcoding/check_ro_qualifier.ll index 6de610b2240d..824ca1b2d692 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/check_ro_qualifier.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/check_ro_qualifier.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: %[[#IMAGE_TYPE:]] = OpTypeImage ; CHECK-SPIRV: %[[#IMAGE_ARG:]] = OpFunctionParameter %[[#IMAGE_TYPE]] diff --git a/llvm/test/CodeGen/SPIRV/transcoding/cl-types.ll b/llvm/test/CodeGen/SPIRV/transcoding/cl-types.ll index 52b7dac8866f..d7e87c05340d 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/cl-types.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/cl-types.ll @@ -19,6 +19,7 @@ ;; } ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV-DAG: OpCapability Sampled1D ; CHECK-SPIRV-DAG: OpCapability SampledBuffer diff --git a/llvm/test/CodeGen/SPIRV/transcoding/clk_event_t.ll b/llvm/test/CodeGen/SPIRV/transcoding/clk_event_t.ll index 9054454879cc..0cd75bb215ad 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/clk_event_t.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/clk_event_t.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpTypeDeviceEvent ; CHECK-SPIRV: OpFunction diff --git a/llvm/test/CodeGen/SPIRV/transcoding/enqueue_kernel.ll b/llvm/test/CodeGen/SPIRV/transcoding/enqueue_kernel.ll index cf124ec0a278..d23b0687face 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/enqueue_kernel.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/enqueue_kernel.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; TODO(#60133): Requires updates following opaque pointer migration. ; XFAIL: * diff --git a/llvm/test/CodeGen/SPIRV/transcoding/explicit-conversions.ll b/llvm/test/CodeGen/SPIRV/transcoding/explicit-conversions.ll index c186a8135fee..49b84c1e9530 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/explicit-conversions.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/explicit-conversions.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpSatConvertSToU diff --git a/llvm/test/CodeGen/SPIRV/transcoding/extract_insert_value.ll b/llvm/test/CodeGen/SPIRV/transcoding/extract_insert_value.ll index fd29bc8a1ebf..0ed1dc76628c 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/extract_insert_value.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/extract_insert_value.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; TODO(#60133): Requires updates following opaque pointer migration. ; XFAIL: * diff --git a/llvm/test/CodeGen/SPIRV/transcoding/fadd.ll b/llvm/test/CodeGen/SPIRV/transcoding/fadd.ll index 78d9a2326655..af76c0e96f9f 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/fadd.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/fadd.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpName %[[#r1:]] "r1" ; CHECK-SPIRV: OpName %[[#r2:]] "r2" diff --git a/llvm/test/CodeGen/SPIRV/transcoding/fclamp.ll b/llvm/test/CodeGen/SPIRV/transcoding/fclamp.ll index cfdcc728fbe4..550ec1a6f255 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/fclamp.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/fclamp.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: %[[#]] = OpExtInst %[[#]] %[[#]] fclamp ; CHECK-SPIRV-NOT: %[[#]] = OpExtInst %[[#]] %[[#]] clamp diff --git a/llvm/test/CodeGen/SPIRV/transcoding/fcmp.ll b/llvm/test/CodeGen/SPIRV/transcoding/fcmp.ll index 572ccc3ed625..46eaba9d5ceb 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/fcmp.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/fcmp.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpName %[[#r1:]] "r1" ; CHECK-SPIRV: OpName %[[#r2:]] "r2" diff --git a/llvm/test/CodeGen/SPIRV/transcoding/fdiv.ll b/llvm/test/CodeGen/SPIRV/transcoding/fdiv.ll index d0ed5640e706..79b786814c71 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/fdiv.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/fdiv.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpName %[[#r1:]] "r1" ; CHECK-SPIRV: OpName %[[#r2:]] "r2" diff --git a/llvm/test/CodeGen/SPIRV/transcoding/fmod.ll b/llvm/test/CodeGen/SPIRV/transcoding/fmod.ll index f506787bcb9c..683b5c24f5b7 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/fmod.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/fmod.ll @@ -2,6 +2,7 @@ ;; { out = fmod( in1, in2 ); } ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: %[[#]] = OpExtInst %[[#]] %[[#]] fmod %[[#]] %[[#]] diff --git a/llvm/test/CodeGen/SPIRV/transcoding/fmul.ll b/llvm/test/CodeGen/SPIRV/transcoding/fmul.ll index 886077a67b4e..fdab29c9041c 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/fmul.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/fmul.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpName %[[#r1:]] "r1" ; CHECK-SPIRV: OpName %[[#r2:]] "r2" diff --git a/llvm/test/CodeGen/SPIRV/transcoding/fneg.ll b/llvm/test/CodeGen/SPIRV/transcoding/fneg.ll index e17601a2c25a..60bbfe6b7f39 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/fneg.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/fneg.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpName %[[#r1:]] "r1" ; CHECK-SPIRV: OpName %[[#r2:]] "r2" diff --git a/llvm/test/CodeGen/SPIRV/transcoding/fp_contract_reassoc_fast_mode.ll b/llvm/test/CodeGen/SPIRV/transcoding/fp_contract_reassoc_fast_mode.ll index c035c35a339e..974043c11991 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/fp_contract_reassoc_fast_mode.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/fp_contract_reassoc_fast_mode.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV-NOT: OpCapability FPFastMathModeINTEL ; CHECK-SPIRV: OpName %[[#mu:]] "mul" diff --git a/llvm/test/CodeGen/SPIRV/transcoding/frem.ll b/llvm/test/CodeGen/SPIRV/transcoding/frem.ll index ecb8f6f950ca..d36ba7f70e45 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/frem.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/frem.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpName %[[#r1:]] "r1" ; CHECK-SPIRV: OpName %[[#r2:]] "r2" diff --git a/llvm/test/CodeGen/SPIRV/transcoding/fsub.ll b/llvm/test/CodeGen/SPIRV/transcoding/fsub.ll index 99d0d0eb84f9..3677c0040562 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/fsub.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/fsub.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV: OpName %[[#r1:]] "r1" ; CHECK-SPIRV: OpName %[[#r2:]] "r2" diff --git a/llvm/test/CodeGen/SPIRV/transcoding/get_image_num_mip_levels.ll b/llvm/test/CodeGen/SPIRV/transcoding/get_image_num_mip_levels.ll index dc307c70612e..fd241963d1e9 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/get_image_num_mip_levels.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/get_image_num_mip_levels.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; Types: ; CHECK-DAG: %[[#INT:]] = OpTypeInt 32 diff --git a/llvm/test/CodeGen/SPIRV/transcoding/global_block.ll b/llvm/test/CodeGen/SPIRV/transcoding/global_block.ll index 2f44e1943b6a..ff1bec4497ba 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/global_block.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/global_block.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV1_4 +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; TODO(#60133): Requires updates following opaque pointer migration. ; XFAIL: * diff --git a/llvm/test/CodeGen/SPIRV/transcoding/group_ops.ll b/llvm/test/CodeGen/SPIRV/transcoding/group_ops.ll index 6aa9faa6c893..2412f406a9c6 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/group_ops.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/group_ops.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV-DAG: %[[#int:]] = OpTypeInt 32 0 ; CHECK-SPIRV-DAG: %[[#float:]] = OpTypeFloat 32 diff --git a/llvm/test/CodeGen/SPIRV/transcoding/isequal.ll b/llvm/test/CodeGen/SPIRV/transcoding/isequal.ll index 3c818afcdb16..c5f3f9e1e2e7 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/isequal.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/isequal.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-SPIRV-NOT: OpSConvert diff --git a/llvm/test/CodeGen/SPIRV/transcoding/relationals_double.ll b/llvm/test/CodeGen/SPIRV/transcoding/relationals_double.ll index f771854672ce..de7673ad7f17 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/relationals_double.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/relationals_double.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; This test checks following SYCL relational builtins with double and double2 ;; types: diff --git a/llvm/test/CodeGen/SPIRV/transcoding/relationals_float.ll b/llvm/test/CodeGen/SPIRV/transcoding/relationals_float.ll index 1f55cebb0911..69a4a30fd65e 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/relationals_float.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/relationals_float.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; This test checks following SYCL relational builtins with float and float2 ;; types: diff --git a/llvm/test/CodeGen/SPIRV/transcoding/relationals_half.ll b/llvm/test/CodeGen/SPIRV/transcoding/relationals_half.ll index 864fb4f29efd..d6a7fda41afd 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/relationals_half.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/relationals_half.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; This test checks following SYCL relational builtins with half and half2 types: ;; isfinite, isinf, isnan, isnormal, signbit, isequal, isnotequal, isgreater -- GitLab From 5a4e2210bd60fec854822d3fb2a2aaa1e20ad2e1 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Wed, 13 Mar 2024 00:48:46 -0700 Subject: [PATCH 0094/1407] Revert "[llvm-exegesis] Use LLVM Support to get thread ID" This reverts commit 1c3b15e9f5bc671e40bcf5d3475f5425466754ce. This (and/or) a related patch was causing build failures on one of the buildbots. More information is available at https://lab.llvm.org/buildbot/#/builders/178/builds/7015. --- llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp | 2 +- .../tools/llvm-exegesis/lib/SubprocessMemory.cpp | 16 +++++++++++----- llvm/tools/llvm-exegesis/lib/SubprocessMemory.h | 5 ++++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 17ce0355ef4f..4e97d188d172 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -301,7 +301,7 @@ private: if (AddMemDefError) return AddMemDefError; - long ParentTID = get_threadid(); + long ParentTID = SubprocessMemory::getCurrentTID(); pid_t ParentOrChildPID = fork(); if (ParentOrChildPID == -1) { diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp index 28b341c46180..1fd81bd407be 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp @@ -10,7 +10,6 @@ #include "Error.h" #include "llvm/Support/Error.h" #include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/Threading.h" #include #ifdef __linux__ @@ -25,6 +24,13 @@ namespace exegesis { #if defined(__linux__) && !defined(__ANDROID__) +long SubprocessMemory::getCurrentTID() { + // We're using the raw syscall here rather than the gettid() function provided + // by most libcs for compatibility as gettid() was only added to glibc in + // version 2.30. + return syscall(SYS_gettid); +} + Error SubprocessMemory::initializeSubprocessMemory(pid_t ProcessID) { // Add the PID to the shared memory name so that if we're running multiple // processes at the same time, they won't interfere with each other. @@ -32,7 +38,7 @@ Error SubprocessMemory::initializeSubprocessMemory(pid_t ProcessID) { // llvm-lit. Additionally add the TID so that downstream consumers // using multiple threads don't run into conflicts. std::string AuxiliaryMemoryName = - formatv("/{0}auxmem{1}", get_threadid(), ProcessID); + formatv("/{0}auxmem{1}", getCurrentTID(), ProcessID); int AuxiliaryMemoryFD = shm_open(AuxiliaryMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryFD == -1) @@ -53,7 +59,7 @@ Error SubprocessMemory::addMemoryDefinition( SharedMemoryNames.reserve(MemoryDefinitions.size()); for (auto &[Name, MemVal] : MemoryDefinitions) { std::string SharedMemoryName = - formatv("/{0}t{1}memdef{2}", ProcessPID, get_threadid(), MemVal.Index); + formatv("/{0}t{1}memdef{2}", ProcessPID, getCurrentTID(), MemVal.Index); SharedMemoryNames.push_back(SharedMemoryName); int SharedMemoryFD = shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); @@ -87,7 +93,7 @@ Error SubprocessMemory::addMemoryDefinition( Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, uint64_t ParentTID, int CounterFileDescriptor) { + pid_t ParentPID, long ParentTID, int CounterFileDescriptor) { std::string AuxiliaryMemoryName = formatv("/{0}auxmem{1}", ParentTID, ParentPID); int AuxiliaryMemoryFileDescriptor = @@ -139,7 +145,7 @@ Error SubprocessMemory::addMemoryDefinition( Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, uint64_t ParentTID, int CounterFileDescriptor) { + pid_t ParentPID, long ParentTID, int CounterFileDescriptor) { return make_error( "setupAuxiliaryMemoryInSubprocess is only supported on Linux"); } diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h index 807046e38ce6..572d1085d9cf 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h @@ -35,6 +35,9 @@ public: static constexpr const size_t AuxiliaryMemoryOffset = 1; static constexpr const size_t AuxiliaryMemorySize = 4096; + // Gets the thread ID for the calling thread. + static long getCurrentTID(); + Error initializeSubprocessMemory(pid_t ProcessID); // The following function sets up memory definitions. It creates shared @@ -54,7 +57,7 @@ public: // section. static Expected setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, uint64_t ParentTID, int CounterFileDescriptor); + pid_t ParentPID, long ParentTID, int CounterFileDescriptor); ~SubprocessMemory(); -- GitLab From 1fe9c417a0bf143f9bb9f9e1fbf7b20f44196883 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Wed, 13 Mar 2024 00:49:23 -0700 Subject: [PATCH 0095/1407] Revert "Reland "[llvm-exegesis] Add thread IDs to subprocess memory names (#84451)"" This reverts commit 8003f553a01a9a2a7eb09fe07e88f1ba9ee7d3a7. This (and/or a related commit) was causing build failures on one of the buildbots that needs more investigation. More information is available at https://lab.llvm.org/buildbot/#/builders/178/builds/7015. --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 9 +++--- .../llvm-exegesis/lib/SubprocessMemory.cpp | 30 ++++++------------- .../llvm-exegesis/lib/SubprocessMemory.h | 5 +--- .../X86/SubprocessMemoryTest.cpp | 5 +--- 4 files changed, 15 insertions(+), 34 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 4e97d188d172..5c9848f3c688 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -301,7 +301,6 @@ private: if (AddMemDefError) return AddMemDefError; - long ParentTID = SubprocessMemory::getCurrentTID(); pid_t ParentOrChildPID = fork(); if (ParentOrChildPID == -1) { @@ -315,7 +314,7 @@ private: // Unregister handlers, signal handling is now handled through ptrace in // the host process. sys::unregisterHandlers(); - prepareAndRunBenchmark(PipeFiles[0], Key, ParentTID); + prepareAndRunBenchmark(PipeFiles[0], Key); // The child process terminates in the above function, so we should never // get to this point. llvm_unreachable("Child process didn't exit when expected."); @@ -416,8 +415,8 @@ private: setrlimit(RLIMIT_CORE, &rlim); } - [[noreturn]] void prepareAndRunBenchmark(int Pipe, const BenchmarkKey &Key, - long ParentTID) const { + [[noreturn]] void prepareAndRunBenchmark(int Pipe, + const BenchmarkKey &Key) const { // Disable core dumps in the child process as otherwise everytime we // encounter an execution failure like a segmentation fault, we will create // a core dump. We report the information directly rather than require the @@ -474,7 +473,7 @@ private: Expected AuxMemFDOrError = SubprocessMemory::setupAuxiliaryMemoryInSubprocess( - Key.MemoryValues, ParentPID, ParentTID, CounterFileDescriptor); + Key.MemoryValues, ParentPID, CounterFileDescriptor); if (!AuxMemFDOrError) exit(ChildProcessExitCodeE::AuxiliaryMemorySetupFailed); diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp index 1fd81bd407be..a49fa077257d 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp @@ -9,13 +9,11 @@ #include "SubprocessMemory.h" #include "Error.h" #include "llvm/Support/Error.h" -#include "llvm/Support/FormatVariadic.h" #include #ifdef __linux__ #include #include -#include #include #endif @@ -24,21 +22,12 @@ namespace exegesis { #if defined(__linux__) && !defined(__ANDROID__) -long SubprocessMemory::getCurrentTID() { - // We're using the raw syscall here rather than the gettid() function provided - // by most libcs for compatibility as gettid() was only added to glibc in - // version 2.30. - return syscall(SYS_gettid); -} - Error SubprocessMemory::initializeSubprocessMemory(pid_t ProcessID) { // Add the PID to the shared memory name so that if we're running multiple // processes at the same time, they won't interfere with each other. // This comes up particularly often when running the exegesis tests with - // llvm-lit. Additionally add the TID so that downstream consumers - // using multiple threads don't run into conflicts. - std::string AuxiliaryMemoryName = - formatv("/{0}auxmem{1}", getCurrentTID(), ProcessID); + // llvm-lit + std::string AuxiliaryMemoryName = "/auxmem" + std::to_string(ProcessID); int AuxiliaryMemoryFD = shm_open(AuxiliaryMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryFD == -1) @@ -58,8 +47,8 @@ Error SubprocessMemory::addMemoryDefinition( pid_t ProcessPID) { SharedMemoryNames.reserve(MemoryDefinitions.size()); for (auto &[Name, MemVal] : MemoryDefinitions) { - std::string SharedMemoryName = - formatv("/{0}t{1}memdef{2}", ProcessPID, getCurrentTID(), MemVal.Index); + std::string SharedMemoryName = "/" + std::to_string(ProcessPID) + "memdef" + + std::to_string(MemVal.Index); SharedMemoryNames.push_back(SharedMemoryName); int SharedMemoryFD = shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); @@ -93,9 +82,8 @@ Error SubprocessMemory::addMemoryDefinition( Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, long ParentTID, int CounterFileDescriptor) { - std::string AuxiliaryMemoryName = - formatv("/{0}auxmem{1}", ParentTID, ParentPID); + pid_t ParentPID, int CounterFileDescriptor) { + std::string AuxiliaryMemoryName = "/auxmem" + std::to_string(ParentPID); int AuxiliaryMemoryFileDescriptor = shm_open(AuxiliaryMemoryName.c_str(), O_RDWR, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryFileDescriptor == -1) @@ -109,8 +97,8 @@ Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( return make_error("Mapping auxiliary memory failed"); AuxiliaryMemoryMapping[0] = CounterFileDescriptor; for (auto &[Name, MemVal] : MemoryDefinitions) { - std::string MemoryValueName = - formatv("/{0}t{1}memdef{2}", ParentPID, ParentTID, MemVal.Index); + std::string MemoryValueName = "/" + std::to_string(ParentPID) + "memdef" + + std::to_string(MemVal.Index); AuxiliaryMemoryMapping[AuxiliaryMemoryOffset + MemVal.Index] = shm_open(MemoryValueName.c_str(), O_RDWR, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryMapping[AuxiliaryMemoryOffset + MemVal.Index] == -1) @@ -145,7 +133,7 @@ Error SubprocessMemory::addMemoryDefinition( Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, long ParentTID, int CounterFileDescriptor) { + pid_t ParentPID, int CounterFileDescriptor) { return make_error( "setupAuxiliaryMemoryInSubprocess is only supported on Linux"); } diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h index 572d1085d9cf..e20b50cdc811 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h @@ -35,9 +35,6 @@ public: static constexpr const size_t AuxiliaryMemoryOffset = 1; static constexpr const size_t AuxiliaryMemorySize = 4096; - // Gets the thread ID for the calling thread. - static long getCurrentTID(); - Error initializeSubprocessMemory(pid_t ProcessID); // The following function sets up memory definitions. It creates shared @@ -57,7 +54,7 @@ public: // section. static Expected setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, long ParentTID, int CounterFileDescriptor); + pid_t ParentPID, int CounterFileDescriptor); ~SubprocessMemory(); diff --git a/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp index 7c23e7b7e9c5..c07ec188a602 100644 --- a/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp +++ b/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #include #endif // __linux__ @@ -50,9 +49,7 @@ protected: std::string getSharedMemoryName(const unsigned TestNumber, const unsigned DefinitionNumber) { - long CurrentTID = syscall(SYS_gettid); - return "/" + std::to_string(getSharedMemoryNumber(TestNumber)) + "t" + - std::to_string(CurrentTID) + "memdef" + + return "/" + std::to_string(getSharedMemoryNumber(TestNumber)) + "memdef" + std::to_string(DefinitionNumber); } -- GitLab From 46682f445adfa06cb74239b17b588e36fcd4fdaa Mon Sep 17 00:00:00 2001 From: Danial Klimkin Date: Wed, 13 Mar 2024 09:00:09 +0100 Subject: [PATCH 0096/1407] Fix missing include past a38b7a432d3cbb093af9310eba5b4982dc0a0243 (#85041) --- clang/include/clang/InstallAPI/FrontendRecords.h | 1 + 1 file changed, 1 insertion(+) diff --git a/clang/include/clang/InstallAPI/FrontendRecords.h b/clang/include/clang/InstallAPI/FrontendRecords.h index 333015b6a113..1f5bc37798be 100644 --- a/clang/include/clang/InstallAPI/FrontendRecords.h +++ b/clang/include/clang/InstallAPI/FrontendRecords.h @@ -11,6 +11,7 @@ #include "clang/AST/Availability.h" #include "clang/AST/DeclObjC.h" +#include "clang/InstallAPI/HeaderFile.h" #include "clang/InstallAPI/MachO.h" namespace clang { -- GitLab From 1c792d24e0a228ad49cc004a1c26bbd7cd87f030 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Wed, 13 Mar 2024 09:01:00 +0100 Subject: [PATCH 0097/1407] [compiler-rt] Fix interceptors with AArch64 BTI (#84061) On AArch64 with BTI, we have to start functions with the appropriate BTI hint to indicate that the function is a valid call target. To support interceptors with AArch64 BTI, add "BTI c". --- compiler-rt/lib/interception/interception.h | 4 ++-- compiler-rt/lib/sanitizer_common/sanitizer_asm.h | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/compiler-rt/lib/interception/interception.h b/compiler-rt/lib/interception/interception.h index 00bcd979638b..38c152952e32 100644 --- a/compiler-rt/lib/interception/interception.h +++ b/compiler-rt/lib/interception/interception.h @@ -204,11 +204,11 @@ const interpose_substitution substitution_##func_name[] \ ".type " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", " \ ASM_TYPE_FUNCTION_STR "\n" \ SANITIZER_STRINGIFY(TRAMPOLINE(func)) ":\n" \ - SANITIZER_STRINGIFY(CFI_STARTPROC) "\n" \ + C_ASM_STARTPROC "\n" \ C_ASM_TAIL_CALL(SANITIZER_STRINGIFY(TRAMPOLINE(func)), \ "__interceptor_" \ SANITIZER_STRINGIFY(ASM_PREEMPTIBLE_SYM(func))) "\n" \ - SANITIZER_STRINGIFY(CFI_ENDPROC) "\n" \ + C_ASM_ENDPROC "\n" \ ".size " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", " \ ".-" SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \ ); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_asm.h b/compiler-rt/lib/sanitizer_common/sanitizer_asm.h index 3af66a4e4499..30e9d15184e5 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_asm.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_asm.h @@ -42,6 +42,16 @@ # define CFI_RESTORE(reg) #endif +#if defined(__aarch64__) && defined(__ARM_FEATURE_BTI_DEFAULT) +# define ASM_STARTPROC CFI_STARTPROC; hint #34 +# define C_ASM_STARTPROC SANITIZER_STRINGIFY(CFI_STARTPROC) "\nhint #34" +#else +# define ASM_STARTPROC CFI_STARTPROC +# define C_ASM_STARTPROC SANITIZER_STRINGIFY(CFI_STARTPROC) +#endif +#define ASM_ENDPROC CFI_ENDPROC +#define C_ASM_ENDPROC SANITIZER_STRINGIFY(CFI_ENDPROC) + #if defined(__x86_64__) || defined(__i386__) || defined(__sparc__) # define ASM_TAIL_CALL jmp #elif defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \ @@ -114,9 +124,9 @@ .globl __interceptor_trampoline_##name; \ ASM_TYPE_FUNCTION(__interceptor_trampoline_##name); \ __interceptor_trampoline_##name: \ - CFI_STARTPROC; \ + ASM_STARTPROC; \ ASM_TAIL_CALL ASM_PREEMPTIBLE_SYM(__interceptor_##name); \ - CFI_ENDPROC; \ + ASM_ENDPROC; \ ASM_SIZE(__interceptor_trampoline_##name) # define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 1 # endif // Architecture supports interceptor trampoline -- GitLab From 0be9592b0077dc63596ce46379cf7b3bd4a405c8 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Wed, 13 Mar 2024 09:02:05 +0100 Subject: [PATCH 0098/1407] [clang] CTAD: Respect requires-clause of the original function template for the synthesized deduction guide (#84913) We ignored the require-clause of the original template when building the deduction guide for type-alias CTAD, this resulted in accepting code which should be rejected (see the test case). This patch fixes it, part of #84492. --- clang/lib/Sema/SemaTemplate.cpp | 19 ++++++++++++++----- clang/test/SemaCXX/cxx20-ctad-type-alias.cpp | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index d8c9a5c09944..51e8db2dfbaa 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -2906,18 +2906,27 @@ void DeclareImplicitDeductionGuidesForTypeAlias( Context.getCanonicalTemplateArgument( Context.getInjectedTemplateArg(NewParam)); } - // FIXME: implement the associated constraint per C++ + // Substitute new template parameters into requires-clause if present. + Expr *RequiresClause = nullptr; + if (Expr *InnerRC = F->getTemplateParameters()->getRequiresClause()) { + MultiLevelTemplateArgumentList Args; + Args.setKind(TemplateSubstitutionKind::Rewrite); + Args.addOuterTemplateArguments(TemplateArgsForBuildingFPrime); + ExprResult E = SemaRef.SubstExpr(InnerRC, Args); + if (E.isInvalid()) + return; + RequiresClause = E.getAs(); + } + // FIXME: implement the is_deducible constraint per C++ // [over.match.class.deduct]p3.3: - // The associated constraints ([temp.constr.decl]) are the - // conjunction of the associated constraints of g and a - // constraint that is satisfied if and only if the arguments + // ... and a constraint that is satisfied if and only if the arguments // of A are deducible (see below) from the return type. auto *FPrimeTemplateParamList = TemplateParameterList::Create( Context, AliasTemplate->getTemplateParameters()->getTemplateLoc(), AliasTemplate->getTemplateParameters()->getLAngleLoc(), FPrimeTemplateParams, AliasTemplate->getTemplateParameters()->getRAngleLoc(), - /*RequiresClause=*/nullptr); + /*RequiresClause=*/RequiresClause); // To form a deduction guide f' from f, we leverage clang's instantiation // mechanism, we construct a template argument list where the template diff --git a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp index 794496ed4184..3ce26c8fcd98 100644 --- a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp +++ b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp @@ -230,3 +230,20 @@ using AFoo = Foo*; // expected-note {{template is declared here}} AFoo s = {1}; // expected-error {{alias template 'AFoo' requires template arguments; argument deduction only allowed for}} } // namespace test17 + +namespace test18 { +template +concept False = false; // expected-note {{because 'false' evaluated to false}} + +template struct Foo { T t; }; + +template requires False // expected-note {{because 'int' does not satisfy 'False'}} +Foo(T) -> Foo; + +template +using Bar = Foo; // expected-note {{could not match 'Foo' against 'int'}} \ + // expected-note {{candidate template ignored: constraints not satisfied}} \ + // expected-note {{candidate function template not viable}} + +Bar s = {1}; // expected-error {{no viable constructor or deduction guide for deduction of template arguments}} +} // namespace test18 -- GitLab From e42e97a4ada141ca1320a49e7fe03245d6765bce Mon Sep 17 00:00:00 2001 From: Sander de Smalen Date: Wed, 13 Mar 2024 08:21:33 +0000 Subject: [PATCH 0099/1407] [AArch64][SME] Don't mark 'smstart za' as using/defining VG. (#84775) VG is only used/defined when changing the streaming mode, using 'smstart sm' or plainly 'smstart' (same for smstop). --- .../Target/AArch64/AArch64ISelLowering.cpp | 12 +++++++++- llvm/lib/Target/AArch64/SMEInstrFormats.td | 2 -- llvm/test/CodeGen/AArch64/sme-write-vg.ll | 24 +++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/sme-write-vg.ll diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 054311d39e7b..5b7a36d2eba7 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -7751,7 +7751,7 @@ void AArch64TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, // register allocator to pass call args in callee saved regs, without extra // copies to avoid these fake clobbers of actually-preserved GPRs. if (MI.getOpcode() == AArch64::MSRpstatesvcrImm1 || - MI.getOpcode() == AArch64::MSRpstatePseudo) + MI.getOpcode() == AArch64::MSRpstatePseudo) { for (unsigned I = MI.getNumOperands() - 1; I > 0; --I) if (MachineOperand &MO = MI.getOperand(I); MO.isReg() && MO.isImplicit() && MO.isDef() && @@ -7759,6 +7759,16 @@ void AArch64TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, AArch64::GPR64RegClass.contains(MO.getReg()))) MI.removeOperand(I); + // The SVE vector length can change when entering/leaving streaming mode. + if (MI.getOperand(0).getImm() == AArch64SVCR::SVCRSM || + MI.getOperand(0).getImm() == AArch64SVCR::SVCRSMZA) { + MI.addOperand(MachineOperand::CreateReg(AArch64::VG, /*IsDef=*/false, + /*IsImplicit=*/true)); + MI.addOperand(MachineOperand::CreateReg(AArch64::VG, /*IsDef=*/true, + /*IsImplicit=*/true)); + } + } + // Add an implicit use of 'VG' for ADDXri/SUBXri, which are instructions that // have nothing to do with VG, were it not that they are used to materialise a // frame-address. If they contain a frame-index to a scalable vector, this diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td index 33cb5f9734b8..44d9a8ac7cb6 100644 --- a/llvm/lib/Target/AArch64/SMEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td @@ -223,8 +223,6 @@ def MSRpstatesvcrImm1 let Inst{8} = imm; let Inst{7-5} = 0b011; // op2 let hasPostISelHook = 1; - let Uses = [VG]; - let Defs = [VG]; } def : InstAlias<"smstart", (MSRpstatesvcrImm1 0b011, 0b1)>; diff --git a/llvm/test/CodeGen/AArch64/sme-write-vg.ll b/llvm/test/CodeGen/AArch64/sme-write-vg.ll new file mode 100644 index 000000000000..577606d45484 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sme-write-vg.ll @@ -0,0 +1,24 @@ +; RUN: llc -mattr=+sme -stop-after=finalize-isel < %s | FileCheck %s + +target triple = "aarch64" + +; Check that we don't define VG for 'smstart za' and 'smstop za' +define void @smstart_za() "aarch64_new_za" nounwind { + ; CHECK-LABEL: name: smstart_za + ; CHECK-NOT: implicit-def {{[^,]*}}$vg + ret void +} + +; Check that we do define VG for 'smstart sm' and 'smstop sm' +define void @smstart_sm() nounwind { + ; CHECK-LABEL: name: smstart_sm + ; CHECK: MSRpstatesvcrImm1 1, 1, + ; CHECK-SAME: implicit-def {{[^,]*}}$vg + ; CHECK: MSRpstatesvcrImm1 1, 0, + ; CHECK-SAME: implicit-def {{[^,]*}}$vg + call void @require_sm() + ret void +} + +declare void @require_sm() "aarch64_pstate_sm_enabled" +declare void @require_za() "aarch64_inout_za" -- GitLab From 06c06e15f45acd3ea24756978629f1d78724870e Mon Sep 17 00:00:00 2001 From: AtariDreams <83477269+AtariDreams@users.noreply.github.com> Date: Wed, 13 Mar 2024 04:23:04 -0400 Subject: [PATCH 0100/1407] [Float2Int] Resolve FIXME: Pick the smallest legal type that fits (#79158) Pick the type based on the smallest bit-width possible, using DataLayout. --- .../llvm/Transforms/Scalar/Float2Int.h | 2 +- llvm/lib/Transforms/Scalar/Float2Int.cpp | 29 +- llvm/test/Transforms/Float2Int/basic.ll | 251 ++++++++++++++---- 3 files changed, 214 insertions(+), 68 deletions(-) diff --git a/llvm/include/llvm/Transforms/Scalar/Float2Int.h b/llvm/include/llvm/Transforms/Scalar/Float2Int.h index 83be329bed60..337e229efcf3 100644 --- a/llvm/include/llvm/Transforms/Scalar/Float2Int.h +++ b/llvm/include/llvm/Transforms/Scalar/Float2Int.h @@ -44,7 +44,7 @@ private: std::optional calcRange(Instruction *I); void walkBackwards(); void walkForwards(); - bool validateAndTransform(); + bool validateAndTransform(const DataLayout &DL); Value *convert(Instruction *I, Type *ToTy); void cleanup(); diff --git a/llvm/lib/Transforms/Scalar/Float2Int.cpp b/llvm/lib/Transforms/Scalar/Float2Int.cpp index ccca8bcc1a56..6ad4be169b58 100644 --- a/llvm/lib/Transforms/Scalar/Float2Int.cpp +++ b/llvm/lib/Transforms/Scalar/Float2Int.cpp @@ -311,7 +311,7 @@ void Float2IntPass::walkForwards() { } // If there is a valid transform to be done, do it. -bool Float2IntPass::validateAndTransform() { +bool Float2IntPass::validateAndTransform(const DataLayout &DL) { bool MadeChange = false; // Iterate over every disjoint partition of the def-use graph. @@ -376,15 +376,23 @@ bool Float2IntPass::validateAndTransform() { LLVM_DEBUG(dbgs() << "F2I: Value not guaranteed to be representable!\n"); continue; } - if (MinBW > 64) { - LLVM_DEBUG( - dbgs() << "F2I: Value requires more than 64 bits to represent!\n"); - continue; - } - // OK, R is known to be representable. Now pick a type for it. - // FIXME: Pick the smallest legal type that will fit. - Type *Ty = (MinBW > 32) ? Type::getInt64Ty(*Ctx) : Type::getInt32Ty(*Ctx); + // OK, R is known to be representable. + // Pick the smallest legal type that will fit. + Type *Ty = DL.getSmallestLegalIntType(*Ctx, MinBW); + if (!Ty) { + // Every supported target supports 64-bit and 32-bit integers, + // so fallback to a 32 or 64-bit integer if the value fits. + if (MinBW <= 32) { + Ty = Type::getInt32Ty(*Ctx); + } else if (MinBW <= 64) { + Ty = Type::getInt64Ty(*Ctx); + } else { + LLVM_DEBUG(dbgs() << "F2I: Value requires more than bits to represent " + "than the target supports!\n"); + continue; + } + } for (auto MI = ECs.member_begin(It), ME = ECs.member_end(); MI != ME; ++MI) @@ -491,7 +499,8 @@ bool Float2IntPass::runImpl(Function &F, const DominatorTree &DT) { walkBackwards(); walkForwards(); - bool Modified = validateAndTransform(); + const DataLayout &DL = F.getParent()->getDataLayout(); + bool Modified = validateAndTransform(DL); if (Modified) cleanup(); return Modified; diff --git a/llvm/test/Transforms/Float2Int/basic.ll b/llvm/test/Transforms/Float2Int/basic.ll index 2854a83179b7..a454b773f4eb 100644 --- a/llvm/test/Transforms/Float2Int/basic.ll +++ b/llvm/test/Transforms/Float2Int/basic.ll @@ -1,16 +1,29 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt < %s -passes='float2int' -S | FileCheck %s +; RUN: opt < %s -passes=float2int -S | FileCheck %s -check-prefixes=CHECK,NONE +; RUN: opt < %s -passes=float2int -S --data-layout="n64" | FileCheck %s -check-prefixes=CHECK,ONLY64 +; RUN: opt < %s -passes=float2int -S --data-layout="n8:16:32:64"| FileCheck %s -check-prefixes=CHECK,MULTIPLE ; ; Positive tests ; define i16 @simple1(i8 %a) { -; CHECK-LABEL: @simple1( -; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 -; CHECK-NEXT: [[T21:%.*]] = add i32 [[TMP1]], 1 -; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[T21]] to i16 -; CHECK-NEXT: ret i16 [[TMP2]] +; NONE-LABEL: @simple1( +; NONE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 +; NONE-NEXT: [[T21:%.*]] = add i32 [[TMP1]], 1 +; NONE-NEXT: [[TMP2:%.*]] = trunc i32 [[T21]] to i16 +; NONE-NEXT: ret i16 [[TMP2]] +; +; ONLY64-LABEL: @simple1( +; ONLY64-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64 +; ONLY64-NEXT: [[T21:%.*]] = add i64 [[TMP1]], 1 +; ONLY64-NEXT: [[TMP2:%.*]] = trunc i64 [[T21]] to i16 +; ONLY64-NEXT: ret i16 [[TMP2]] +; +; MULTIPLE-LABEL: @simple1( +; MULTIPLE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16 +; MULTIPLE-NEXT: [[T21:%.*]] = add i16 [[TMP1]], 1 +; MULTIPLE-NEXT: ret i16 [[T21]] ; %t1 = uitofp i8 %a to float %t2 = fadd float %t1, 1.0 @@ -19,11 +32,23 @@ define i16 @simple1(i8 %a) { } define i8 @simple2(i8 %a) { -; CHECK-LABEL: @simple2( -; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 -; CHECK-NEXT: [[T21:%.*]] = sub i32 [[TMP1]], 1 -; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[T21]] to i8 -; CHECK-NEXT: ret i8 [[TMP2]] +; NONE-LABEL: @simple2( +; NONE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 +; NONE-NEXT: [[T21:%.*]] = sub i32 [[TMP1]], 1 +; NONE-NEXT: [[TMP2:%.*]] = trunc i32 [[T21]] to i8 +; NONE-NEXT: ret i8 [[TMP2]] +; +; ONLY64-LABEL: @simple2( +; ONLY64-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64 +; ONLY64-NEXT: [[T21:%.*]] = sub i64 [[TMP1]], 1 +; ONLY64-NEXT: [[TMP2:%.*]] = trunc i64 [[T21]] to i8 +; ONLY64-NEXT: ret i8 [[TMP2]] +; +; MULTIPLE-LABEL: @simple2( +; MULTIPLE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16 +; MULTIPLE-NEXT: [[T21:%.*]] = sub i16 [[TMP1]], 1 +; MULTIPLE-NEXT: [[TMP2:%.*]] = trunc i16 [[T21]] to i8 +; MULTIPLE-NEXT: ret i8 [[TMP2]] ; %t1 = uitofp i8 %a to float %t2 = fsub float %t1, 1.0 @@ -32,10 +57,22 @@ define i8 @simple2(i8 %a) { } define i32 @simple3(i8 %a) { -; CHECK-LABEL: @simple3( -; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 -; CHECK-NEXT: [[T21:%.*]] = sub i32 [[TMP1]], 1 -; CHECK-NEXT: ret i32 [[T21]] +; NONE-LABEL: @simple3( +; NONE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 +; NONE-NEXT: [[T21:%.*]] = sub i32 [[TMP1]], 1 +; NONE-NEXT: ret i32 [[T21]] +; +; ONLY64-LABEL: @simple3( +; ONLY64-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64 +; ONLY64-NEXT: [[T21:%.*]] = sub i64 [[TMP1]], 1 +; ONLY64-NEXT: [[TMP2:%.*]] = trunc i64 [[T21]] to i32 +; ONLY64-NEXT: ret i32 [[TMP2]] +; +; MULTIPLE-LABEL: @simple3( +; MULTIPLE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16 +; MULTIPLE-NEXT: [[T21:%.*]] = sub i16 [[TMP1]], 1 +; MULTIPLE-NEXT: [[TMP2:%.*]] = zext i16 [[T21]] to i32 +; MULTIPLE-NEXT: ret i32 [[TMP2]] ; %t1 = uitofp i8 %a to float %t2 = fsub float %t1, 1.0 @@ -44,11 +81,23 @@ define i32 @simple3(i8 %a) { } define i1 @cmp(i8 %a, i8 %b) { -; CHECK-LABEL: @cmp( -; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 -; CHECK-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32 -; CHECK-NEXT: [[T31:%.*]] = icmp slt i32 [[TMP1]], [[TMP2]] -; CHECK-NEXT: ret i1 [[T31]] +; NONE-LABEL: @cmp( +; NONE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 +; NONE-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32 +; NONE-NEXT: [[T31:%.*]] = icmp slt i32 [[TMP1]], [[TMP2]] +; NONE-NEXT: ret i1 [[T31]] +; +; ONLY64-LABEL: @cmp( +; ONLY64-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64 +; ONLY64-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i64 +; ONLY64-NEXT: [[T31:%.*]] = icmp slt i64 [[TMP1]], [[TMP2]] +; ONLY64-NEXT: ret i1 [[T31]] +; +; MULTIPLE-LABEL: @cmp( +; MULTIPLE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16 +; MULTIPLE-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i16 +; MULTIPLE-NEXT: [[T31:%.*]] = icmp slt i16 [[TMP1]], [[TMP2]] +; MULTIPLE-NEXT: ret i1 [[T31]] ; %t1 = uitofp i8 %a to float %t2 = uitofp i8 %b to float @@ -70,12 +119,27 @@ define i32 @simple4(i32 %a) { } define i32 @simple5(i8 %a, i8 %b) { -; CHECK-LABEL: @simple5( -; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 -; CHECK-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32 -; CHECK-NEXT: [[T31:%.*]] = add i32 [[TMP1]], 1 -; CHECK-NEXT: [[T42:%.*]] = mul i32 [[T31]], [[TMP2]] -; CHECK-NEXT: ret i32 [[T42]] +; NONE-LABEL: @simple5( +; NONE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 +; NONE-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32 +; NONE-NEXT: [[T31:%.*]] = add i32 [[TMP1]], 1 +; NONE-NEXT: [[T42:%.*]] = mul i32 [[T31]], [[TMP2]] +; NONE-NEXT: ret i32 [[T42]] +; +; ONLY64-LABEL: @simple5( +; ONLY64-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64 +; ONLY64-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i64 +; ONLY64-NEXT: [[T31:%.*]] = add i64 [[TMP1]], 1 +; ONLY64-NEXT: [[T42:%.*]] = mul i64 [[T31]], [[TMP2]] +; ONLY64-NEXT: [[TMP3:%.*]] = trunc i64 [[T42]] to i32 +; ONLY64-NEXT: ret i32 [[TMP3]] +; +; MULTIPLE-LABEL: @simple5( +; MULTIPLE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 +; MULTIPLE-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32 +; MULTIPLE-NEXT: [[T31:%.*]] = add i32 [[TMP1]], 1 +; MULTIPLE-NEXT: [[T42:%.*]] = mul i32 [[T31]], [[TMP2]] +; MULTIPLE-NEXT: ret i32 [[T42]] ; %t1 = uitofp i8 %a to float %t2 = uitofp i8 %b to float @@ -86,12 +150,27 @@ define i32 @simple5(i8 %a, i8 %b) { } define i32 @simple6(i8 %a, i8 %b) { -; CHECK-LABEL: @simple6( -; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 -; CHECK-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32 -; CHECK-NEXT: [[T31:%.*]] = sub i32 0, [[TMP1]] -; CHECK-NEXT: [[T42:%.*]] = mul i32 [[T31]], [[TMP2]] -; CHECK-NEXT: ret i32 [[T42]] +; NONE-LABEL: @simple6( +; NONE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 +; NONE-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32 +; NONE-NEXT: [[T31:%.*]] = sub i32 0, [[TMP1]] +; NONE-NEXT: [[T42:%.*]] = mul i32 [[T31]], [[TMP2]] +; NONE-NEXT: ret i32 [[T42]] +; +; ONLY64-LABEL: @simple6( +; ONLY64-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64 +; ONLY64-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i64 +; ONLY64-NEXT: [[T31:%.*]] = sub i64 0, [[TMP1]] +; ONLY64-NEXT: [[T42:%.*]] = mul i64 [[T31]], [[TMP2]] +; ONLY64-NEXT: [[TMP3:%.*]] = trunc i64 [[T42]] to i32 +; ONLY64-NEXT: ret i32 [[TMP3]] +; +; MULTIPLE-LABEL: @simple6( +; MULTIPLE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 +; MULTIPLE-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32 +; MULTIPLE-NEXT: [[T31:%.*]] = sub i32 0, [[TMP1]] +; MULTIPLE-NEXT: [[T42:%.*]] = mul i32 [[T31]], [[TMP2]] +; MULTIPLE-NEXT: ret i32 [[T42]] ; %t1 = uitofp i8 %a to float %t2 = uitofp i8 %b to float @@ -105,15 +184,37 @@ define i32 @simple6(i8 %a, i8 %b) { ; cause failure of the other. define i32 @multi1(i8 %a, i8 %b, i8 %c, float %d) { -; CHECK-LABEL: @multi1( -; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 -; CHECK-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32 -; CHECK-NEXT: [[FC:%.*]] = uitofp i8 [[C:%.*]] to float -; CHECK-NEXT: [[X1:%.*]] = add i32 [[TMP1]], [[TMP2]] -; CHECK-NEXT: [[Z:%.*]] = fadd float [[FC]], [[D:%.*]] -; CHECK-NEXT: [[W:%.*]] = fptoui float [[Z]] to i32 -; CHECK-NEXT: [[R:%.*]] = add i32 [[X1]], [[W]] -; CHECK-NEXT: ret i32 [[R]] +; NONE-LABEL: @multi1( +; NONE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 +; NONE-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32 +; NONE-NEXT: [[FC:%.*]] = uitofp i8 [[C:%.*]] to float +; NONE-NEXT: [[X1:%.*]] = add i32 [[TMP1]], [[TMP2]] +; NONE-NEXT: [[Z:%.*]] = fadd float [[FC]], [[D:%.*]] +; NONE-NEXT: [[W:%.*]] = fptoui float [[Z]] to i32 +; NONE-NEXT: [[R:%.*]] = add i32 [[X1]], [[W]] +; NONE-NEXT: ret i32 [[R]] +; +; ONLY64-LABEL: @multi1( +; ONLY64-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64 +; ONLY64-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i64 +; ONLY64-NEXT: [[FC:%.*]] = uitofp i8 [[C:%.*]] to float +; ONLY64-NEXT: [[X1:%.*]] = add i64 [[TMP1]], [[TMP2]] +; ONLY64-NEXT: [[TMP3:%.*]] = trunc i64 [[X1]] to i32 +; ONLY64-NEXT: [[Z:%.*]] = fadd float [[FC]], [[D:%.*]] +; ONLY64-NEXT: [[W:%.*]] = fptoui float [[Z]] to i32 +; ONLY64-NEXT: [[R:%.*]] = add i32 [[TMP3]], [[W]] +; ONLY64-NEXT: ret i32 [[R]] +; +; MULTIPLE-LABEL: @multi1( +; MULTIPLE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16 +; MULTIPLE-NEXT: [[TMP2:%.*]] = zext i8 [[B:%.*]] to i16 +; MULTIPLE-NEXT: [[FC:%.*]] = uitofp i8 [[C:%.*]] to float +; MULTIPLE-NEXT: [[X1:%.*]] = add i16 [[TMP1]], [[TMP2]] +; MULTIPLE-NEXT: [[TMP3:%.*]] = zext i16 [[X1]] to i32 +; MULTIPLE-NEXT: [[Z:%.*]] = fadd float [[FC]], [[D:%.*]] +; MULTIPLE-NEXT: [[W:%.*]] = fptoui float [[Z]] to i32 +; MULTIPLE-NEXT: [[R:%.*]] = add i32 [[TMP3]], [[W]] +; MULTIPLE-NEXT: ret i32 [[R]] ; %fa = uitofp i8 %a to float %fb = uitofp i8 %b to float @@ -127,11 +228,22 @@ define i32 @multi1(i8 %a, i8 %b, i8 %c, float %d) { } define i16 @simple_negzero(i8 %a) { -; CHECK-LABEL: @simple_negzero( -; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 -; CHECK-NEXT: [[T21:%.*]] = add i32 [[TMP1]], 0 -; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[T21]] to i16 -; CHECK-NEXT: ret i16 [[TMP2]] +; NONE-LABEL: @simple_negzero( +; NONE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 +; NONE-NEXT: [[T21:%.*]] = add i32 [[TMP1]], 0 +; NONE-NEXT: [[TMP2:%.*]] = trunc i32 [[T21]] to i16 +; NONE-NEXT: ret i16 [[TMP2]] +; +; ONLY64-LABEL: @simple_negzero( +; ONLY64-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64 +; ONLY64-NEXT: [[T21:%.*]] = add i64 [[TMP1]], 0 +; ONLY64-NEXT: [[TMP2:%.*]] = trunc i64 [[T21]] to i16 +; ONLY64-NEXT: ret i16 [[TMP2]] +; +; MULTIPLE-LABEL: @simple_negzero( +; MULTIPLE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16 +; MULTIPLE-NEXT: [[T21:%.*]] = add i16 [[TMP1]], 0 +; MULTIPLE-NEXT: ret i16 [[T21]] ; %t1 = uitofp i8 %a to float %t2 = fadd fast float %t1, -0.0 @@ -140,12 +252,26 @@ define i16 @simple_negzero(i8 %a) { } define i32 @simple_negative(i8 %call) { -; CHECK-LABEL: @simple_negative( -; CHECK-NEXT: [[TMP1:%.*]] = sext i8 [[CALL:%.*]] to i32 -; CHECK-NEXT: [[MUL1:%.*]] = mul i32 [[TMP1]], -3 -; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[MUL1]] to i8 -; CHECK-NEXT: [[CONV3:%.*]] = sext i8 [[TMP2]] to i32 -; CHECK-NEXT: ret i32 [[CONV3]] +; NONE-LABEL: @simple_negative( +; NONE-NEXT: [[TMP1:%.*]] = sext i8 [[CALL:%.*]] to i32 +; NONE-NEXT: [[MUL1:%.*]] = mul i32 [[TMP1]], -3 +; NONE-NEXT: [[TMP2:%.*]] = trunc i32 [[MUL1]] to i8 +; NONE-NEXT: [[CONV3:%.*]] = sext i8 [[TMP2]] to i32 +; NONE-NEXT: ret i32 [[CONV3]] +; +; ONLY64-LABEL: @simple_negative( +; ONLY64-NEXT: [[TMP1:%.*]] = sext i8 [[CALL:%.*]] to i64 +; ONLY64-NEXT: [[MUL1:%.*]] = mul i64 [[TMP1]], -3 +; ONLY64-NEXT: [[TMP2:%.*]] = trunc i64 [[MUL1]] to i8 +; ONLY64-NEXT: [[CONV3:%.*]] = sext i8 [[TMP2]] to i32 +; ONLY64-NEXT: ret i32 [[CONV3]] +; +; MULTIPLE-LABEL: @simple_negative( +; MULTIPLE-NEXT: [[TMP1:%.*]] = sext i8 [[CALL:%.*]] to i16 +; MULTIPLE-NEXT: [[MUL1:%.*]] = mul i16 [[TMP1]], -3 +; MULTIPLE-NEXT: [[TMP2:%.*]] = trunc i16 [[MUL1]] to i8 +; MULTIPLE-NEXT: [[CONV3:%.*]] = sext i8 [[TMP2]] to i32 +; MULTIPLE-NEXT: ret i32 [[CONV3]] ; %conv1 = sitofp i8 %call to float %mul = fmul float %conv1, -3.000000e+00 @@ -155,11 +281,22 @@ define i32 @simple_negative(i8 %call) { } define i16 @simple_fneg(i8 %a) { -; CHECK-LABEL: @simple_fneg( -; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 -; CHECK-NEXT: [[T21:%.*]] = sub i32 0, [[TMP1]] -; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[T21]] to i16 -; CHECK-NEXT: ret i16 [[TMP2]] +; NONE-LABEL: @simple_fneg( +; NONE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32 +; NONE-NEXT: [[T21:%.*]] = sub i32 0, [[TMP1]] +; NONE-NEXT: [[TMP2:%.*]] = trunc i32 [[T21]] to i16 +; NONE-NEXT: ret i16 [[TMP2]] +; +; ONLY64-LABEL: @simple_fneg( +; ONLY64-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64 +; ONLY64-NEXT: [[T21:%.*]] = sub i64 0, [[TMP1]] +; ONLY64-NEXT: [[TMP2:%.*]] = trunc i64 [[T21]] to i16 +; ONLY64-NEXT: ret i16 [[TMP2]] +; +; MULTIPLE-LABEL: @simple_fneg( +; MULTIPLE-NEXT: [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16 +; MULTIPLE-NEXT: [[T21:%.*]] = sub i16 0, [[TMP1]] +; MULTIPLE-NEXT: ret i16 [[T21]] ; %t1 = uitofp i8 %a to float %t2 = fneg fast float %t1 -- GitLab From 676c495195748e8ab2755b62153a718a53f7dae9 Mon Sep 17 00:00:00 2001 From: lcvon007 <141613945+lcvon007@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:38:48 +0800 Subject: [PATCH 0101/1407] [Attributor][FIX] Register right new created BB. (#84929) CBBB will keep same after the first iteration so registerManifestAddedBasicBlock would always register the same basic block later. Co-authored-by: laichunfeng --- llvm/lib/Transforms/IPO/AttributorAttributes.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index 488a6f0bb153..f98833bd1198 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -12371,7 +12371,7 @@ struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo { SplitBlockAndInsertIfThen(LastCmp, IP, /* Unreachable */ false); BasicBlock *CBBB = CB->getParent(); A.registerManifestAddedBasicBlock(*ThenTI->getParent()); - A.registerManifestAddedBasicBlock(*CBBB); + A.registerManifestAddedBasicBlock(*IP->getParent()); auto *SplitTI = cast(LastCmp->getNextNode()); BasicBlock *ElseBB; if (&*IP == CB) { -- GitLab From 2d62ce4bebe484f7c6855b9ef479e9b398595df9 Mon Sep 17 00:00:00 2001 From: mikaelholmen Date: Wed, 13 Mar 2024 09:58:47 +0100 Subject: [PATCH 0102/1407] [ValueTracking] Remove faulty dereference of "InsertBefore" (#85034) In 2fe81edef6f [NFC][RemoveDIs] Insert instruction using iterators in Transforms/ we changed if (*req_idx != *i) return FindInsertedValue(I->getAggregateOperand(), idx_range, - InsertBefore); + *InsertBefore); } but there is no guarantee that is InsertBefore is non-empty at that point, which we e.g can see in the added testcase. Instead just pass on the optional InsertBefore in the recursive call to FindInsertedValue, as we do at several other places already. --- llvm/lib/Analysis/ValueTracking.cpp | 2 +- .../Analysis/Lint/crash_empty_iterator.ll | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Analysis/Lint/crash_empty_iterator.ll diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 371ad41ee965..8a4a2c4f92a0 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -5709,7 +5709,7 @@ llvm::FindInsertedValue(Value *V, ArrayRef idx_range, // looking for, then. if (*req_idx != *i) return FindInsertedValue(I->getAggregateOperand(), idx_range, - *InsertBefore); + InsertBefore); } // If we end up here, the indices of the insertvalue match with those // requested (though possibly only partially). Now we recursively look at diff --git a/llvm/test/Analysis/Lint/crash_empty_iterator.ll b/llvm/test/Analysis/Lint/crash_empty_iterator.ll new file mode 100644 index 000000000000..2fbecbcef5cf --- /dev/null +++ b/llvm/test/Analysis/Lint/crash_empty_iterator.ll @@ -0,0 +1,22 @@ +; RUN: opt -passes="lint" -S < %s | FileCheck %s + +; After 2fe81edef6f0b +; [NFC][RemoveDIs] Insert instruction using iterators in Transforms/ +; this crashed in FindInsertedValue when dereferencing an empty +; optional iterator. +; Just see that it doesn't crash anymore. + +; CHECK-LABEL: @test1 + +%struct = type { i32, i32 } + +define void @test1() { +entry: + %.fca.1.insert = insertvalue %struct zeroinitializer, i32 0, 1 + %0 = extractvalue %struct %.fca.1.insert, 0 + %1 = tail call %struct @foo(i32 %0) + ret void +} + +declare %struct @foo(i32) + -- GitLab From e371ada409b225ea990b5ac0d5cafea26a6046e1 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Wed, 13 Mar 2024 09:25:43 +0000 Subject: [PATCH 0103/1407] [compiler-rt] reimplements GetMemoryProfile for netbsd. (#84841) The actual solution relies on the premise /proc/self/smaps existence. instead relying on native api like freebsd. fixing fuzzer build too. --- compiler-rt/lib/fuzzer/FuzzerUtilLinux.cpp | 2 +- .../lib/sanitizer_common/sanitizer_procmaps_bsd.cpp | 13 +++++++++++++ .../sanitizer_common/sanitizer_procmaps_common.cpp | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/compiler-rt/lib/fuzzer/FuzzerUtilLinux.cpp b/compiler-rt/lib/fuzzer/FuzzerUtilLinux.cpp index 5729448b0beb..e5409f22f0e3 100644 --- a/compiler-rt/lib/fuzzer/FuzzerUtilLinux.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerUtilLinux.cpp @@ -44,7 +44,7 @@ void SetThreadName(std::thread &thread, const std::string &name) { #if LIBFUZZER_LINUX || LIBFUZZER_FREEBSD (void)pthread_setname_np(thread.native_handle(), name.c_str()); #elif LIBFUZZER_NETBSD - (void)pthread_set_name_np(thread.native_handle(), "%s", name.c_str()); + (void)pthread_setname_np(thread.native_handle(), "%s", const_cast(name.c_str())); #endif } diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cpp index 7c2d8e6f1731..7d7c009f6444 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cpp @@ -42,6 +42,19 @@ void GetMemoryProfile(fill_profile_f cb, uptr *stats) { cb(0, InfoProc->ki_rssize * GetPageSizeCached(), false, stats); UnmapOrDie(InfoProc, Size, true); } +#elif SANITIZER_NETBSD +void GetMemoryProfile(fill_profile_f cb, uptr *stats) { + struct kinfo_proc2 *InfoProc; + uptr Len = sizeof(*InfoProc); + uptr Size = Len; + const int Mib[] = {CTL_KERN, KERN_PROC2, KERN_PROC_PID, getpid(), Size, 1}; + InfoProc = (struct kinfo_proc2 *)MmapOrDie(Size, "GetMemoryProfile()"); + CHECK_EQ( + internal_sysctl(Mib, ARRAY_SIZE(Mib), nullptr, (uptr *)InfoProc, &Len, 0), + 0); + cb(0, InfoProc->p_vm_rssize * GetPageSizeCached(), false, stats); + UnmapOrDie(InfoProc, Size, true); +} #endif void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cpp index a7805ad1b083..7214a2b9ea46 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cpp @@ -145,7 +145,7 @@ void MemoryMappingLayout::DumpListOfModules( } } -#if SANITIZER_LINUX || SANITIZER_ANDROID || SANITIZER_SOLARIS || SANITIZER_NETBSD +#if SANITIZER_LINUX || SANITIZER_ANDROID || SANITIZER_SOLARIS void GetMemoryProfile(fill_profile_f cb, uptr *stats) { char *smaps = nullptr; uptr smaps_cap = 0; -- GitLab From 995d1d114e4e4ff708a03cdb0a975209c6197f9f Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 13 Mar 2024 09:47:16 +0000 Subject: [PATCH 0104/1407] [SjLjEHPrepare] Use inverse_depth_first() instead of _ext variant (NFC). (#84920) inverse_depth_first df_iterator_default_set as default set, so there's no need to explicitly use inverse_depth_first_ext. PR: https://github.com/llvm/llvm-project/pull/84920 --- llvm/lib/CodeGen/SjLjEHPrepare.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llvm/lib/CodeGen/SjLjEHPrepare.cpp b/llvm/lib/CodeGen/SjLjEHPrepare.cpp index 515b5764a094..4bad57d279e9 100644 --- a/llvm/lib/CodeGen/SjLjEHPrepare.cpp +++ b/llvm/lib/CodeGen/SjLjEHPrepare.cpp @@ -150,9 +150,7 @@ static void MarkBlocksLiveIn(BasicBlock *BB, if (!LiveBBs.insert(BB).second) return; // already been here. - df_iterator_default_set Visited; - - for (BasicBlock *B : inverse_depth_first_ext(BB, Visited)) + for (BasicBlock *B : inverse_depth_first(BB)) LiveBBs.insert(B); } -- GitLab From 20b15e645cdbde07ae46aefe46ede5ff4d1e8ba3 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Wed, 13 Mar 2024 11:48:33 +0100 Subject: [PATCH 0105/1407] [Tests] Drop inrange attribute from some tests (NFC) These don't actually test anything related to inrange, so drop the attribute. --- llvm/test/CodeGen/AArch64/fold-global-offsets.ll | 2 +- llvm/test/CodeGen/Hexagon/addrmode-immop.mir | 4 ++-- llvm/test/CodeGen/NVPTX/b52037.ll | 2 +- llvm/test/CodeGen/WinCFGuard/cfguard-mingw.ll | 12 ++++++------ llvm/test/CodeGen/X86/tls-align.ll | 2 +- llvm/test/DebugInfo/X86/tu-to-non-tu.ll | 8 ++++---- .../Transforms/Utils/CallPromotionUtilsTest.cpp | 12 ++++++------ 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/fold-global-offsets.ll b/llvm/test/CodeGen/AArch64/fold-global-offsets.ll index 897d35a78495..8de0f0d121f9 100644 --- a/llvm/test/CodeGen/AArch64/fold-global-offsets.ll +++ b/llvm/test/CodeGen/AArch64/fold-global-offsets.ll @@ -131,7 +131,7 @@ define i32 @f7() { ; GISEL-NEXT: ret entry: - %lshr = lshr i128 bitcast (<2 x i64> to i128), 64 + %lshr = lshr i128 bitcast (<2 x i64> to i128), 64 %trunc = trunc i128 %lshr to i64 %inttoptr = inttoptr i64 %trunc to ptr %gep = getelementptr i32, ptr %inttoptr, i64 5 diff --git a/llvm/test/CodeGen/Hexagon/addrmode-immop.mir b/llvm/test/CodeGen/Hexagon/addrmode-immop.mir index 3069cbe5969d..1412d31f35ac 100644 --- a/llvm/test/CodeGen/Hexagon/addrmode-immop.mir +++ b/llvm/test/CodeGen/Hexagon/addrmode-immop.mir @@ -15,7 +15,7 @@ ; Function Attrs: norecurse define void @f0() #0 { b0: - %v0 = load ptr, ptr getelementptr (i8, ptr getelementptr inbounds ({ [3 x ptr], [3 x ptr] }, ptr @g0, i32 0, inrange i32 0, i32 3), i32 sub (i32 ptrtoint (ptr @f1 to i32), i32 1)), align 4 + %v0 = load ptr, ptr getelementptr (i8, ptr getelementptr inbounds ({ [3 x ptr], [3 x ptr] }, ptr @g0, i32 0, i32 0, i32 3), i32 sub (i32 ptrtoint (ptr @f1 to i32), i32 1)), align 4 %v1 = call i32 %v0(ptr nonnull undef) unreachable } @@ -33,7 +33,7 @@ tracksRegLiveness: true body: | bb.0.b0: $r2 = A2_tfrsi @g0 + 12 - $r2 = L2_loadri_io killed $r2, @f1 - 1 :: (load (s32) from `ptr getelementptr (i8, ptr getelementptr inbounds ({ [3 x ptr], [3 x ptr] }, ptr @g0, i32 0, inrange i32 0, i32 3), i32 sub (i32 ptrtoint (ptr @f1 to i32), i32 1))`) + $r2 = L2_loadri_io killed $r2, @f1 - 1 :: (load (s32) from `ptr getelementptr (i8, ptr getelementptr inbounds ({ [3 x ptr], [3 x ptr] }, ptr @g0, i32 0, i32 0, i32 3), i32 sub (i32 ptrtoint (ptr @f1 to i32), i32 1))`) ADJCALLSTACKDOWN 0, 0, implicit-def $r29, implicit-def dead $r30, implicit $r31, implicit $r30, implicit $r29 PS_callr_nr killed $r2, hexagoncsr, implicit undef $r0, implicit-def $r29, implicit-def dead $r0 ADJCALLSTACKUP 0, 0, implicit-def dead $r29, implicit-def dead $r30, implicit-def dead $r31, implicit $r29 diff --git a/llvm/test/CodeGen/NVPTX/b52037.ll b/llvm/test/CodeGen/NVPTX/b52037.ll index d9322dabfa06..5d1c390909f6 100644 --- a/llvm/test/CodeGen/NVPTX/b52037.ll +++ b/llvm/test/CodeGen/NVPTX/b52037.ll @@ -47,7 +47,7 @@ bb: %tmp5 = load ptr, ptr %tmp4, align 8 %tmp9 = getelementptr inbounds %struct.zot, ptr %tmp, i64 0, i32 2, i32 1 store ptr %tmp5, ptr %tmp9, align 8 - store ptr getelementptr inbounds ({ [3 x ptr] }, ptr @global_1, i64 0, inrange i32 0, i64 3), ptr %tmp, align 16 + store ptr getelementptr inbounds ({ [3 x ptr] }, ptr @global_1, i64 0, i32 0, i64 3), ptr %tmp, align 16 %tmp.i1 = tail call i64 @foo() %tmp44.i16 = getelementptr inbounds i16, ptr %tmp5, i64 undef %tmp45.i17 = load i16, ptr %tmp44.i16, align 2 diff --git a/llvm/test/CodeGen/WinCFGuard/cfguard-mingw.ll b/llvm/test/CodeGen/WinCFGuard/cfguard-mingw.ll index 085cde8c6169..7a5baa09f95e 100644 --- a/llvm/test/CodeGen/WinCFGuard/cfguard-mingw.ll +++ b/llvm/test/CodeGen/WinCFGuard/cfguard-mingw.ll @@ -97,7 +97,7 @@ $_ZTI7Derived = comdat any ; Function Attrs: nounwind uwtable define weak_odr dso_local dllexport void @_ZN4BaseC2Ev(ptr noundef nonnull align 8 dereferenceable(12) %0) unnamed_addr #0 comdat align 2 { - store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV4Base, i64 0, inrange i32 0, i64 2), ptr %0, align 8, !tbaa !5 + store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV4Base, i64 0, i32 0, i64 2), ptr %0, align 8, !tbaa !5 %2 = getelementptr inbounds %class.Base, ptr %0, i64 0, i32 1 store i32 0, ptr %2, align 8, !tbaa !8 ret void @@ -105,7 +105,7 @@ define weak_odr dso_local dllexport void @_ZN4BaseC2Ev(ptr noundef nonnull align ; Function Attrs: nounwind uwtable define weak_odr dso_local dllexport void @_ZN4BaseC1Ev(ptr noundef nonnull align 8 dereferenceable(12) %0) unnamed_addr #0 comdat align 2 { - store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV4Base, i64 0, inrange i32 0, i64 2), ptr %0, align 8, !tbaa !5 + store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV4Base, i64 0, i32 0, i64 2), ptr %0, align 8, !tbaa !5 %2 = getelementptr inbounds %class.Base, ptr %0, i64 0, i32 1 store i32 0, ptr %2, align 8, !tbaa !8 ret void @@ -140,10 +140,10 @@ declare dso_local void @_ZdlPv(ptr noundef) local_unnamed_addr #2 ; Function Attrs: nounwind uwtable define weak_odr dso_local dllexport void @_ZN7DerivedC2Ev(ptr noundef nonnull align 8 dereferenceable(16) %0) unnamed_addr #0 comdat align 2 { - store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV4Base, i64 0, inrange i32 0, i64 2), ptr %0, align 8, !tbaa !5 + store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV4Base, i64 0, i32 0, i64 2), ptr %0, align 8, !tbaa !5 %2 = getelementptr inbounds %class.Base, ptr %0, i64 0, i32 1 store i32 0, ptr %2, align 8, !tbaa !8 - store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV7Derived, i64 0, inrange i32 0, i64 2), ptr %0, align 8, !tbaa !5 + store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV7Derived, i64 0, i32 0, i64 2), ptr %0, align 8, !tbaa !5 %3 = getelementptr inbounds %class.Derived, ptr %0, i64 0, i32 1 store i32 0, ptr %3, align 4, !tbaa !12 ret void @@ -151,10 +151,10 @@ define weak_odr dso_local dllexport void @_ZN7DerivedC2Ev(ptr noundef nonnull al ; Function Attrs: nounwind uwtable define weak_odr dso_local dllexport void @_ZN7DerivedC1Ev(ptr noundef nonnull align 8 dereferenceable(16) %0) unnamed_addr #0 comdat align 2 { - store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV4Base, i64 0, inrange i32 0, i64 2), ptr %0, align 8, !tbaa !5 + store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV4Base, i64 0, i32 0, i64 2), ptr %0, align 8, !tbaa !5 %2 = getelementptr inbounds %class.Base, ptr %0, i64 0, i32 1 store i32 0, ptr %2, align 8, !tbaa !8 - store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV7Derived, i64 0, inrange i32 0, i64 2), ptr %0, align 8, !tbaa !5 + store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV7Derived, i64 0, i32 0, i64 2), ptr %0, align 8, !tbaa !5 %3 = getelementptr inbounds %class.Derived, ptr %0, i64 0, i32 1 store i32 0, ptr %3, align 4, !tbaa !12 ret void diff --git a/llvm/test/CodeGen/X86/tls-align.ll b/llvm/test/CodeGen/X86/tls-align.ll index 3c8ee6b3f8ab..e996c00dbf1d 100644 --- a/llvm/test/CodeGen/X86/tls-align.ll +++ b/llvm/test/CodeGen/X86/tls-align.ll @@ -12,7 +12,7 @@ define internal fastcc void @foo() unnamed_addr { entry: - store <8 x ptr> , ptr @array, align 32 + store <8 x ptr> , ptr @array, align 32 ret void } diff --git a/llvm/test/DebugInfo/X86/tu-to-non-tu.ll b/llvm/test/DebugInfo/X86/tu-to-non-tu.ll index 3ad97adbb5ad..f80bd8b97896 100644 --- a/llvm/test/DebugInfo/X86/tu-to-non-tu.ll +++ b/llvm/test/DebugInfo/X86/tu-to-non-tu.ll @@ -156,14 +156,14 @@ %struct.templ_non_tu.1 = type { ptr } @_ZTV6non_tu = dso_local unnamed_addr constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI6non_tu, ptr @_ZN6non_tu2f1Ev] }, align 8 -@v1 = dso_local global { { ptr } } { { ptr } { ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV6non_tu, i32 0, inrange i32 0, i32 2) } }, align 8, !dbg !0 +@v1 = dso_local global { { ptr } } { { ptr } { ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV6non_tu, i32 0, i32 0, i32 2) } }, align 8, !dbg !0 @v5 = dso_local global %struct.ref_internal zeroinitializer, align 1, !dbg !5 @_ZTV12templ_non_tuIiE = dso_local unnamed_addr constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI12templ_non_tuIiE, ptr @_ZN12templ_non_tuIiE2f1Ev] }, align 8 -@v2 = dso_local global { { ptr } } { { ptr } { ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV12templ_non_tuIiE, i32 0, inrange i32 0, i32 2) } }, align 8, !dbg !13 +@v2 = dso_local global { { ptr } } { { ptr } { ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV12templ_non_tuIiE, i32 0, i32 0, i32 2) } }, align 8, !dbg !13 @_ZTV12templ_non_tuIlE = dso_local unnamed_addr constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI12templ_non_tuIlE, ptr @_ZN12templ_non_tuIlE2f1Ev] }, align 8 -@v3 = dso_local global { { ptr } } { { ptr } { ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV12templ_non_tuIlE, i32 0, inrange i32 0, i32 2) } }, align 8, !dbg !32 +@v3 = dso_local global { { ptr } } { { ptr } { ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV12templ_non_tuIlE, i32 0, i32 0, i32 2) } }, align 8, !dbg !32 @_ZTV12templ_non_tuIbE = dso_local unnamed_addr constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr @_ZTI12templ_non_tuIbE, ptr @_ZN12templ_non_tuIbE2f1Ev] }, align 8 -@v4 = dso_local global { { ptr } } { { ptr } { ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV12templ_non_tuIbE, i32 0, inrange i32 0, i32 2) } }, align 8, !dbg !46 +@v4 = dso_local global { { ptr } } { { ptr } { ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV12templ_non_tuIbE, i32 0, i32 0, i32 2) } }, align 8, !dbg !46 @v6 = dso_local global %class.ref_internal_template zeroinitializer, align 1, !dbg !60 @v7 = dso_local global %class.ref_from_ref_internal_template zeroinitializer, align 1, !dbg !69 @_ZTVN10__cxxabiv117__class_type_infoE = external dso_local global ptr diff --git a/llvm/unittests/Transforms/Utils/CallPromotionUtilsTest.cpp b/llvm/unittests/Transforms/Utils/CallPromotionUtilsTest.cpp index eff8e27d36d6..0e9641c5846f 100644 --- a/llvm/unittests/Transforms/Utils/CallPromotionUtilsTest.cpp +++ b/llvm/unittests/Transforms/Utils/CallPromotionUtilsTest.cpp @@ -37,7 +37,7 @@ define void @f() { entry: %o = alloca %class.Impl %base = getelementptr %class.Impl, %class.Impl* %o, i64 0, i32 0, i32 0 - store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV4Impl, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %base + store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV4Impl, i64 0, i32 0, i64 2) to i32 (...)**), i32 (...)*** %base %f = getelementptr inbounds %class.Impl, %class.Impl* %o, i64 0, i32 1 store i32 3, i32* %f %base.i = getelementptr inbounds %class.Impl, %class.Impl* %o, i64 0, i32 0 @@ -171,7 +171,7 @@ define void @f() { entry: %o = alloca %class.Impl %base = getelementptr %class.Impl, %class.Impl* %o, i64 0, i32 0, i32 0 - store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV4Impl, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %base + store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV4Impl, i64 0, i32 0, i64 2) to i32 (...)**), i32 (...)*** %base %f = getelementptr inbounds %class.Impl, %class.Impl* %o, i64 0, i32 1 store i32 3, i32* %f %base.i = getelementptr inbounds %class.Impl, %class.Impl* %o, i64 0, i32 0 @@ -213,7 +213,7 @@ define void @f() { entry: %o = alloca %class.Impl %base = getelementptr %class.Impl, %class.Impl* %o, i64 0, i32 0, i32 0 - store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV4Impl, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %base + store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV4Impl, i64 0, i32 0, i64 2) to i32 (...)**), i32 (...)*** %base %f = getelementptr inbounds %class.Impl, %class.Impl* %o, i64 0, i32 1 store i32 3, i32* %f %base.i = getelementptr inbounds %class.Impl, %class.Impl* %o, i64 0, i32 0 @@ -256,7 +256,7 @@ entry: %a = alloca %struct.A, align 8 %0 = bitcast %struct.A* %a to i8* %1 = getelementptr %struct.A, %struct.A* %a, i64 0, i32 0 - store i32 (...)** bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %1, align 8 + store i32 (...)** bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1A, i64 0, i32 0, i64 2) to i32 (...)**), i32 (...)*** %1, align 8 %2 = bitcast %struct.A* %a to i8* %3 = bitcast i8* %2 to i8** %vtable.i = load i8*, i8** %3, align 8 @@ -271,7 +271,7 @@ entry: %a = alloca %struct.A, align 8 %0 = bitcast %struct.A* %a to i8* %1 = getelementptr %struct.A, %struct.A* %a, i64 0, i32 0 - store i32 (...)** bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %1, align 8 + store i32 (...)** bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1A, i64 0, i32 0, i64 2) to i32 (...)**), i32 (...)*** %1, align 8 %2 = bitcast %struct.A* %a to i8* %3 = bitcast i8* %2 to i8** %vtable.i = load i8*, i8** %3, align 8 @@ -340,7 +340,7 @@ define %struct1 @f() { entry: %o = alloca %class.Impl %base = getelementptr %class.Impl, %class.Impl* %o, i64 0, i32 0, i32 0 - store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV4Impl, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %base + store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTV4Impl, i64 0, i32 0, i64 2) to i32 (...)**), i32 (...)*** %base %f = getelementptr inbounds %class.Impl, %class.Impl* %o, i64 0, i32 1 store i32 3, i32* %f %base.i = getelementptr inbounds %class.Impl, %class.Impl* %o, i64 0, i32 0 -- GitLab From e4b27359fde552f65fd3434398a6b80104e1a20d Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Wed, 13 Mar 2024 11:53:35 +0100 Subject: [PATCH 0106/1407] [ThinLTO] Drop inrange attribute from tests (NFC) The inrange attribute is not relevant to the optimizations being tested here. Additionally, all the inrange attributes in these files don't actually carry any additional information, as the "range" covers the whole object. --- llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_bar.ll | 2 +- llvm/test/ThinLTO/X86/devirt_after_filtering_unreachable.ll | 2 +- llvm/test/ThinLTO/X86/devirt_external_comdat_same_guid.ll | 2 +- llvm/test/ThinLTO/X86/devirt_local_same_guid.ll | 2 +- llvm/test/ThinLTO/X86/lower_type_test_phi.ll | 4 ++-- llvm/test/ThinLTO/X86/nodevirt-nonpromoted-typeid.ll | 2 +- llvm/test/ThinLTO/X86/type_test_noindircall.ll | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_bar.ll b/llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_bar.ll index 721d6efb7b53..d8c6525ed606 100644 --- a/llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_bar.ll +++ b/llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_bar.ll @@ -23,7 +23,7 @@ define hidden i32 @_Z3barv() local_unnamed_addr #0 { entry: %b = alloca %struct.A, align 8 call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %b) - store ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV1A, i64 0, inrange i32 0, i64 2), ptr %b, align 8, !tbaa !4 + store ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV1A, i64 0, i32 0, i64 2), ptr %b, align 8, !tbaa !4 %call = call i32 @_Z3fooP1A(ptr nonnull %b) %add = add nsw i32 %call, 10 call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %b) #4 diff --git a/llvm/test/ThinLTO/X86/devirt_after_filtering_unreachable.ll b/llvm/test/ThinLTO/X86/devirt_after_filtering_unreachable.ll index 68b83debef7d..39f42da77d81 100644 --- a/llvm/test/ThinLTO/X86/devirt_after_filtering_unreachable.ll +++ b/llvm/test/ThinLTO/X86/devirt_after_filtering_unreachable.ll @@ -71,7 +71,7 @@ target triple = "x86_64-unknown-linux-gnu" define hidden i32 @main() { entry: %call = tail call ptr @_Znwm(i64 8) - store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV7Derived, i64 0, inrange i32 0, i64 2), ptr %call + store ptr getelementptr inbounds ({ [5 x ptr] }, ptr @_ZTV7Derived, i64 0, i32 0, i64 2), ptr %call tail call void @_Z3fooP4Base(ptr nonnull %call) ret i32 0 } diff --git a/llvm/test/ThinLTO/X86/devirt_external_comdat_same_guid.ll b/llvm/test/ThinLTO/X86/devirt_external_comdat_same_guid.ll index 241753256804..1f0737b71925 100644 --- a/llvm/test/ThinLTO/X86/devirt_external_comdat_same_guid.ll +++ b/llvm/test/ThinLTO/X86/devirt_external_comdat_same_guid.ll @@ -51,7 +51,7 @@ define i32 @_ZN1B1nEi(ptr %this, i32 %a) #0 comdat($_ZTV1B) { ; Ensures that vtable of B is live so that we will attempt devirt. define dso_local i32 @use_B(ptr %a) { entry: - store ptr getelementptr inbounds ({ [4 x ptr] }, ptr @_ZTV1B, i64 0, inrange i32 0, i64 2), ptr %a, align 8 + store ptr getelementptr inbounds ({ [4 x ptr] }, ptr @_ZTV1B, i64 0, i32 0, i64 2), ptr %a, align 8 ret i32 0 } diff --git a/llvm/test/ThinLTO/X86/devirt_local_same_guid.ll b/llvm/test/ThinLTO/X86/devirt_local_same_guid.ll index 3efea8de3fbc..2205545e3250 100644 --- a/llvm/test/ThinLTO/X86/devirt_local_same_guid.ll +++ b/llvm/test/ThinLTO/X86/devirt_local_same_guid.ll @@ -37,7 +37,7 @@ define internal i32 @_ZN1B1nEi(ptr %this, i32 %a) #0 { ; Ensures that vtable of B is live so that we will attempt devirt. define dso_local i32 @use_B(ptr %a) { entry: - store ptr getelementptr inbounds ({ [4 x ptr] }, ptr @_ZTV1B, i64 0, inrange i32 0, i64 2), ptr %a, align 8 + store ptr getelementptr inbounds ({ [4 x ptr] }, ptr @_ZTV1B, i64 0, i32 0, i64 2), ptr %a, align 8 ret i32 0 } diff --git a/llvm/test/ThinLTO/X86/lower_type_test_phi.ll b/llvm/test/ThinLTO/X86/lower_type_test_phi.ll index 722ffe3cc228..81d85f64584f 100644 --- a/llvm/test/ThinLTO/X86/lower_type_test_phi.ll +++ b/llvm/test/ThinLTO/X86/lower_type_test_phi.ll @@ -117,7 +117,7 @@ $_ZTV2D2 = comdat any define ptr @_Z2b1v() { entry: %call = tail call ptr @_Znwm(i64 8) - store ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV2D1, i64 0, inrange i32 0, i64 2), ptr %call, align 8 + store ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV2D1, i64 0, i32 0, i64 2), ptr %call, align 8 ret ptr %call } @@ -126,7 +126,7 @@ declare ptr @_Znwm(i64) define ptr @_Z2b2v() { entry: %call = tail call ptr @_Znwm(i64 8) - store ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV2D2, i64 0, inrange i32 0, i64 2), ptr %call, align 8 + store ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV2D2, i64 0, i32 0, i64 2), ptr %call, align 8 ret ptr %call } diff --git a/llvm/test/ThinLTO/X86/nodevirt-nonpromoted-typeid.ll b/llvm/test/ThinLTO/X86/nodevirt-nonpromoted-typeid.ll index c6e61edcd97a..7d71c595fbb1 100644 --- a/llvm/test/ThinLTO/X86/nodevirt-nonpromoted-typeid.ll +++ b/llvm/test/ThinLTO/X86/nodevirt-nonpromoted-typeid.ll @@ -55,7 +55,7 @@ entry: %this.addr = alloca ptr, align 8 store ptr %this, ptr %this.addr, align 8 %this1 = load ptr, ptr %this.addr - store ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV1D, i64 0, inrange i32 0, i64 2), ptr %this1, align 8 + store ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTV1D, i64 0, i32 0, i64 2), ptr %this1, align 8 ret void } diff --git a/llvm/test/ThinLTO/X86/type_test_noindircall.ll b/llvm/test/ThinLTO/X86/type_test_noindircall.ll index 2d0faaa25602..cc85e44c2806 100644 --- a/llvm/test/ThinLTO/X86/type_test_noindircall.ll +++ b/llvm/test/ThinLTO/X86/type_test_noindircall.ll @@ -38,8 +38,8 @@ target triple = "x86_64-grtev4-linux-gnu" define internal void @_ZN12_GLOBAL__N_18RealFileD2Ev(ptr %this) unnamed_addr #0 align 2 { entry: ; CHECK-IR: store - store ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTVN12_GLOBAL__N_18RealFileE, i64 0, inrange i32 0, i64 2), ptr %this, align 8 - %0 = tail call i1 @llvm.type.test(ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTVN12_GLOBAL__N_18RealFileE, i64 0, inrange i32 0, i64 2), metadata !"4$09c6cc733fc6accb91e5d7b87cb48f2d") + store ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTVN12_GLOBAL__N_18RealFileE, i64 0, i32 0, i64 2), ptr %this, align 8 + %0 = tail call i1 @llvm.type.test(ptr getelementptr inbounds ({ [3 x ptr] }, ptr @_ZTVN12_GLOBAL__N_18RealFileE, i64 0, i32 0, i64 2), metadata !"4$09c6cc733fc6accb91e5d7b87cb48f2d") tail call void @llvm.assume(i1 %0) ; CHECK-IR-NEXT: ret void ret void -- GitLab From 5a744776bb6192dae04360609457c9f49dce43a2 Mon Sep 17 00:00:00 2001 From: dyung Date: Wed, 13 Mar 2024 04:10:03 -0700 Subject: [PATCH 0107/1407] Mark test as XFAIL that started failing after 418f0066eb. (#85027) Similar failures were previously seen and XFAILed in https://reviews.llvm.org/D118468. See the phabricator review for a description of the problem, and the linked discourse thread for what the failing output looks like. This change should fix the issue on two buildbots that are running older versions of GDB: - https://lab.llvm.org/buildbot/#/builders/217/builds/37559 - https://lab.llvm.org/buildbot/#/builders/247/builds/15173 --- .../debuginfo-tests/llgdb-tests/forward-declare-class.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cross-project-tests/debuginfo-tests/llgdb-tests/forward-declare-class.cpp b/cross-project-tests/debuginfo-tests/llgdb-tests/forward-declare-class.cpp index 850eed6ad95d..130c439faec8 100644 --- a/cross-project-tests/debuginfo-tests/llgdb-tests/forward-declare-class.cpp +++ b/cross-project-tests/debuginfo-tests/llgdb-tests/forward-declare-class.cpp @@ -1,6 +1,7 @@ // RUN: %clangxx %target_itanium_abi_host_triple -O0 -g %s -c -o %t.o // RUN: %test_debuginfo %s %t.o // Radar 9168773 +// XFAIL: !system-darwin && gdb-clang-incompatibility // DEBUGGER: ptype A // Work around a gdb bug where it believes that a class is a -- GitLab From 560d7c51fdee4cc15766aa9b62c0cd8f0f18a353 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 13 Mar 2024 11:13:28 +0000 Subject: [PATCH 0108/1407] [DAG] Add SDPatternMatch m_And/m_Or/m_Xor matchers for logic ops --- llvm/include/llvm/CodeGen/SDPatternMatch.h | 15 +++++++++++++++ .../CodeGen/SelectionDAGPatternMatchTest.cpp | 11 +++++++++++ 2 files changed, 26 insertions(+) diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h index 412bf42677cc..92b478cec0a0 100644 --- a/llvm/include/llvm/CodeGen/SDPatternMatch.h +++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h @@ -495,6 +495,21 @@ inline BinaryOpc_match m_Mul(const LHS &L, const RHS &R) { return BinaryOpc_match(ISD::MUL, L, R); } +template +inline BinaryOpc_match m_And(const LHS &L, const RHS &R) { + return BinaryOpc_match(ISD::AND, L, R); +} + +template +inline BinaryOpc_match m_Or(const LHS &L, const RHS &R) { + return BinaryOpc_match(ISD::OR, L, R); +} + +template +inline BinaryOpc_match m_Xor(const LHS &L, const RHS &R) { + return BinaryOpc_match(ISD::XOR, L, R); +} + template inline BinaryOpc_match m_UDiv(const LHS &L, const RHS &R) { return BinaryOpc_match(ISD::UDIV, L, R); diff --git a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp index 17fc3ce8af26..2b764c9e1f0f 100644 --- a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp +++ b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp @@ -130,6 +130,9 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) { SDValue Add = DAG->getNode(ISD::ADD, DL, Int32VT, Op0, Op1); SDValue Sub = DAG->getNode(ISD::SUB, DL, Int32VT, Add, Op0); SDValue Mul = DAG->getNode(ISD::MUL, DL, Int32VT, Add, Sub); + SDValue And = DAG->getNode(ISD::AND, DL, Int32VT, Op0, Op1); + SDValue Xor = DAG->getNode(ISD::XOR, DL, Int32VT, Op1, Op0); + SDValue Or = DAG->getNode(ISD::OR, DL, Int32VT, Op0, Op1); SDValue SFAdd = DAG->getNode(ISD::STRICT_FADD, DL, {Float32VT, MVT::Other}, {DAG->getEntryNode(), Op2, Op2}); @@ -144,6 +147,14 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) { EXPECT_TRUE( sd_match(SFAdd, m_ChainedBinOp(ISD::STRICT_FADD, m_SpecificVT(Float32VT), m_SpecificVT(Float32VT)))); + + EXPECT_TRUE(sd_match(And, m_c_BinOp(ISD::AND, m_Value(), m_Value()))); + EXPECT_TRUE(sd_match(And, m_And(m_Value(), m_Value()))); + EXPECT_TRUE(sd_match(Xor, m_c_BinOp(ISD::XOR, m_Value(), m_Value()))); + EXPECT_TRUE(sd_match(Xor, m_Xor(m_Value(), m_Value()))); + EXPECT_TRUE(sd_match(Or, m_c_BinOp(ISD::OR, m_Value(), m_Value()))); + EXPECT_TRUE(sd_match(Or, m_Or(m_Value(), m_Value()))); + SDValue BindVal; EXPECT_TRUE(sd_match(SFAdd, m_ChainedBinOp(ISD::STRICT_FADD, m_Value(BindVal), m_Deferred(BindVal)))); -- GitLab From 99be3875fb161a5786aaad5dab0b92fa052e47d1 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray Date: Wed, 13 Mar 2024 11:19:32 +0000 Subject: [PATCH 0109/1407] [ARM][AArch64] Add missing Arm CPU part-ids to enable -mcpu=native (#84899) Update Host.cpp with some missing Arm CPU part identifiers, to enable `-mcpu=native` on these processors. These are found in the Technical Reference Manuals listed under "part num" or "part no" --- llvm/lib/TargetParser/Host.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp index ee4fd0425ca5..f65ed259eade 100644 --- a/llvm/lib/TargetParser/Host.cpp +++ b/llvm/lib/TargetParser/Host.cpp @@ -196,14 +196,24 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) { .Case("0xb36", "arm1136j-s") .Case("0xb56", "arm1156t2-s") .Case("0xb76", "arm1176jz-s") + .Case("0xc05", "cortex-a5") + .Case("0xc07", "cortex-a7") .Case("0xc08", "cortex-a8") .Case("0xc09", "cortex-a9") .Case("0xc0f", "cortex-a15") + .Case("0xc0e", "cortex-a17") .Case("0xc20", "cortex-m0") .Case("0xc23", "cortex-m3") .Case("0xc24", "cortex-m4") + .Case("0xc27", "cortex-m7") + .Case("0xd20", "cortex-m23") + .Case("0xd21", "cortex-m33") .Case("0xd24", "cortex-m52") .Case("0xd22", "cortex-m55") + .Case("0xd23", "cortex-m85") + .Case("0xc18", "cortex-r8") + .Case("0xd13", "cortex-r52") + .Case("0xd15", "cortex-r82") .Case("0xd02", "cortex-a34") .Case("0xd04", "cortex-a35") .Case("0xd03", "cortex-a53") @@ -211,13 +221,17 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) { .Case("0xd46", "cortex-a510") .Case("0xd80", "cortex-a520") .Case("0xd07", "cortex-a57") + .Case("0xd06", "cortex-a65") + .Case("0xd43", "cortex-a65ae") .Case("0xd08", "cortex-a72") .Case("0xd09", "cortex-a73") .Case("0xd0a", "cortex-a75") .Case("0xd0b", "cortex-a76") + .Case("0xd0e", "cortex-a76ae") .Case("0xd0d", "cortex-a77") .Case("0xd41", "cortex-a78") .Case("0xd42", "cortex-a78ae") + .Case("0xd4b", "cortex-a78c") .Case("0xd47", "cortex-a710") .Case("0xd4d", "cortex-a715") .Case("0xd81", "cortex-a720") @@ -226,6 +240,7 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) { .Case("0xd48", "cortex-x2") .Case("0xd4e", "cortex-x3") .Case("0xd82", "cortex-x4") + .Case("0xd4a", "neoverse-e1") .Case("0xd0c", "neoverse-n1") .Case("0xd49", "neoverse-n2") .Case("0xd40", "neoverse-v1") -- GitLab From a7af53e99bb1fc92f45c14df2acf2da8f849af2f Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 13 Mar 2024 12:00:15 +0000 Subject: [PATCH 0110/1407] [DAG] visitSUB - convert some folds to use SDPatternMatch General cleanup and allows us to handle several commutable matches with a single pattern --- llvm/include/llvm/CodeGen/SDPatternMatch.h | 1 + llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 78 ++++++------------- llvm/test/CodeGen/AArch64/xor.ll | 2 +- 3 files changed, 25 insertions(+), 56 deletions(-) diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h index 92b478cec0a0..a86c7400fa09 100644 --- a/llvm/include/llvm/CodeGen/SDPatternMatch.h +++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h @@ -663,6 +663,7 @@ inline SpecificInt_match m_SpecificInt(uint64_t V) { } inline SpecificInt_match m_Zero() { return m_SpecificInt(0U); } +inline SpecificInt_match m_One() { return m_SpecificInt(1U); } inline SpecificInt_match m_AllOnes() { return m_SpecificInt(~0U); } /// Match true boolean value based on the information provided by diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 735cec8ecc06..87033e824aa5 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -3789,63 +3789,34 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1)); } - // fold ((A+(B+or-C))-B) -> A+or-C - if (N0.getOpcode() == ISD::ADD && - (N0.getOperand(1).getOpcode() == ISD::SUB || - N0.getOperand(1).getOpcode() == ISD::ADD) && - N0.getOperand(1).getOperand(0) == N1) - return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0), - N0.getOperand(1).getOperand(1)); - - // fold ((A+(C+B))-B) -> A+C - if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD && - N0.getOperand(1).getOperand(1) == N1) - return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), - N0.getOperand(1).getOperand(0)); + SDValue A, B, C; + + // fold ((A+(B+C))-B) -> A+C + if (sd_match(N0, m_Add(m_Value(A), m_Add(m_Specific(N1), m_Value(C))))) + return DAG.getNode(ISD::ADD, DL, VT, A, C); + + // fold ((A+(B-C))-B) -> A-C + if (sd_match(N0, m_Add(m_Value(A), m_Sub(m_Specific(N1), m_Value(C))))) + return DAG.getNode(ISD::SUB, DL, VT, A, C); // fold ((A-(B-C))-C) -> A-B - if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB && - N0.getOperand(1).getOperand(1) == N1) - return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), - N0.getOperand(1).getOperand(0)); + if (sd_match(N0, m_Sub(m_Value(A), m_Sub(m_Value(B), m_Specific(N1))))) + return DAG.getNode(ISD::SUB, DL, VT, A, B); // fold (A-(B-C)) -> A+(C-B) - if (N1.getOpcode() == ISD::SUB && N1.hasOneUse()) + if (sd_match(N1, m_OneUse(m_Sub(m_Value(B), m_Value(C))))) return DAG.getNode(ISD::ADD, DL, VT, N0, - DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1), - N1.getOperand(0))); + DAG.getNode(ISD::SUB, DL, VT, C, B)); // A - (A & B) -> A & (~B) - if (N1.getOpcode() == ISD::AND) { - SDValue A = N1.getOperand(0); - SDValue B = N1.getOperand(1); - if (A != N0) - std::swap(A, B); - if (A == N0 && - (N1.hasOneUse() || isConstantOrConstantVector(B, /*NoOpaques=*/true))) { - SDValue InvB = - DAG.getNode(ISD::XOR, DL, VT, B, DAG.getAllOnesConstant(DL, VT)); - return DAG.getNode(ISD::AND, DL, VT, A, InvB); - } - } + if (sd_match(N1, m_And(m_Specific(N0), m_Value(B))) && + (N1.hasOneUse() || isConstantOrConstantVector(B, /*NoOpaques=*/true))) + return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getNOT(DL, B, VT)); - // fold (X - (-Y * Z)) -> (X + (Y * Z)) - if (N1.getOpcode() == ISD::MUL && N1.hasOneUse()) { - if (N1.getOperand(0).getOpcode() == ISD::SUB && - isNullOrNullSplat(N1.getOperand(0).getOperand(0))) { - SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, - N1.getOperand(0).getOperand(1), - N1.getOperand(1)); - return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); - } - if (N1.getOperand(1).getOpcode() == ISD::SUB && - isNullOrNullSplat(N1.getOperand(1).getOperand(0))) { - SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, - N1.getOperand(0), - N1.getOperand(1).getOperand(1)); - return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); - } - } + // fold (A - (-B * C)) -> (A + (B * C)) + if (sd_match(N1, m_OneUse(m_Mul(m_Sub(m_Zero(), m_Value(B)), m_Value(C))))) + return DAG.getNode(ISD::ADD, DL, VT, N0, + DAG.getNode(ISD::MUL, DL, VT, B, C)); // If either operand of a sub is undef, the result is undef if (N0.isUndef()) @@ -3865,12 +3836,9 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { if (SDValue V = foldSubToUSubSat(VT, N)) return V; - // (x - y) - 1 -> add (xor y, -1), x - if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && isOneOrOneSplat(N1)) { - SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), - DAG.getAllOnesConstant(DL, VT)); - return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0)); - } + // (A - B) - 1 -> add (xor B, -1), A + if (sd_match(N, m_Sub(m_OneUse(m_Sub(m_Value(A), m_Value(B))), m_One()))) + return DAG.getNode(ISD::ADD, DL, VT, A, DAG.getNOT(DL, B, VT)); // Look for: // sub y, (xor x, -1) diff --git a/llvm/test/CodeGen/AArch64/xor.ll b/llvm/test/CodeGen/AArch64/xor.ll index d92402cf43b3..7d7f7bfdbf38 100644 --- a/llvm/test/CodeGen/AArch64/xor.ll +++ b/llvm/test/CodeGen/AArch64/xor.ll @@ -51,7 +51,7 @@ define <4 x i32> @vec_add_of_not_decrement(<4 x i32> %x, <4 x i32> %y) { ; CHECK-LABEL: vec_add_of_not_decrement: ; CHECK: // %bb.0: ; CHECK-NEXT: mvn v1.16b, v1.16b -; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret %t0 = sub <4 x i32> %x, %y %r = sub <4 x i32> %t0, -- GitLab From 7cd61f888c479da51215071336b34f6918cad3d8 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Wed, 13 Mar 2024 11:50:58 +0000 Subject: [PATCH 0111/1407] [AMDGPU] Remove unneeded MnemonicAlias. NFC. This is unneeded because MUBUF_Real_Atomic_gfx11_gfx12 on the line above generates it automatically. --- llvm/lib/Target/AMDGPU/BUFInstructions.td | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index c7091028b3b5..4ae514ffcf78 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -2616,7 +2616,6 @@ defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x042, "buffer defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_gfx11<0x050, "buffer_atomic_cmpswap_f32">; defm BUFFER_ATOMIC_COND_SUB_U32 : MUBUF_Real_Atomic_gfx12<0x050>; defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomic_gfx11_gfx12<0x037, "buffer_atomic_sub_clamp_u32", "buffer_atomic_csub_u32">; -def : Mnem_gfx11_gfx12<"buffer_atomic_csub", "buffer_atomic_csub_u32">; defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_gfx11_gfx12<0x040, "buffer_atomic_dec_u32">; defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04D, "buffer_atomic_dec_u64">; defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_gfx11_gfx12<0x03F, "buffer_atomic_inc_u32">; -- GitLab From ceb744eb2fa0895db1526110462745962fdf43c0 Mon Sep 17 00:00:00 2001 From: Harald van Dijk Date: Wed, 13 Mar 2024 12:08:39 +0000 Subject: [PATCH 0112/1407] [AMDGPU] Fix canonicalization of truncated values. (#83054) We were relying on roundings to implicitly canonicalize, which is generally safe, except with roundings that may be optimized away. Fixes #82937. --- llvm/lib/Target/AMDGPU/AMDGPUInstructions.td | 24 +- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 65 +- llvm/lib/Target/AMDGPU/SIISelLowering.h | 4 +- llvm/lib/Target/AMDGPU/SIInstructions.td | 51 +- llvm/test/CodeGen/AMDGPU/bf16.ll | 911 ++++-------------- llvm/test/CodeGen/AMDGPU/clamp.ll | 64 +- .../AMDGPU/fcanonicalize-elimination.ll | 103 +- llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll | 728 ++++++++------ llvm/test/CodeGen/AMDGPU/fcanonicalize.ll | 22 +- llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll | 22 +- llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll | 2 - llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll | 122 +-- llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll | 122 +-- llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll | 191 ++-- 14 files changed, 1021 insertions(+), 1410 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td index 5ed82c0c4b1b..86f77f7b64e8 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -194,7 +194,25 @@ class HasOneUseTernaryOp : PatFrag< }]; } -class is_canonicalized : PatFrag< +class is_canonicalized_1 : PatFrag< + (ops node:$src0), + (op $src0), + [{ + const SITargetLowering &Lowering = + *static_cast(getTargetLowering()); + + return Lowering.isCanonicalized(*CurDAG, N->getOperand(0)); + }]> { + + let GISelPredicateCode = [{ + const SITargetLowering *TLI = static_cast( + MF.getSubtarget().getTargetLowering()); + + return TLI->isCanonicalized(MI.getOperand(1).getReg(), MF); + }]; +} + +class is_canonicalized_2 : PatFrag< (ops node:$src0, node:$src1), (op $src0, $src1), [{ @@ -210,8 +228,8 @@ class is_canonicalized : PatFrag< const SITargetLowering *TLI = static_cast( MF.getSubtarget().getTargetLowering()); - return TLI->isCanonicalized(MI.getOperand(1).getReg(), const_cast(MF)) && - TLI->isCanonicalized(MI.getOperand(2).getReg(), const_cast(MF)); + return TLI->isCanonicalized(MI.getOperand(1).getReg(), MF) && + TLI->isCanonicalized(MI.getOperand(2).getReg(), MF); }]; } diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 9bc1b8eb598f..5ccf21f76015 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -12572,6 +12572,10 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op, case ISD::FREM: case ISD::FP_ROUND: case ISD::FP_EXTEND: + case ISD::FP16_TO_FP: + case ISD::FP_TO_FP16: + case ISD::BF16_TO_FP: + case ISD::FP_TO_BF16: case ISD::FLDEXP: case AMDGPUISD::FMUL_LEGACY: case AMDGPUISD::FMAD_FTZ: @@ -12591,6 +12595,9 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op, case AMDGPUISD::CVT_F32_UBYTE1: case AMDGPUISD::CVT_F32_UBYTE2: case AMDGPUISD::CVT_F32_UBYTE3: + case AMDGPUISD::FP_TO_FP16: + case AMDGPUISD::SIN_HW: + case AMDGPUISD::COS_HW: return true; // It can/will be lowered or combined as a bit operation. @@ -12600,6 +12607,20 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op, case ISD::FCOPYSIGN: return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1); + case ISD::AND: + if (Op.getValueType() == MVT::i32) { + // Be careful as we only know it is a bitcast floating point type. It + // could be f32, v2f16, we have no way of knowing. Luckily the constant + // value that we optimize for, which comes up in fp32 to bf16 conversions, + // is valid to optimize for all types. + if (auto *RHS = dyn_cast(Op.getOperand(1))) { + if (RHS->getZExtValue() == 0xffff0000) { + return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1); + } + } + } + break; + case ISD::FSIN: case ISD::FCOS: case ISD::FSINCOS: @@ -12665,6 +12686,9 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op, return false; case ISD::BITCAST: + // TODO: This is incorrect as it loses track of the operand's type. We may + // end up effectively bitcasting from f32 to v2f16 or vice versa, and the + // same bits that are canonicalized in one type need not be in the other. return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1); case ISD::TRUNCATE: { // Hack round the mess we make when legalizing extract_vector_elt @@ -12694,25 +12718,26 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op, case Intrinsic::amdgcn_trig_preop: case Intrinsic::amdgcn_log: case Intrinsic::amdgcn_exp2: + case Intrinsic::amdgcn_sqrt: return true; default: break; } - [[fallthrough]]; + break; } default: - // FIXME: denormalsEnabledForType is broken for dynamic - return denormalsEnabledForType(DAG, Op.getValueType()) && - DAG.isKnownNeverSNaN(Op); + break; } - llvm_unreachable("invalid operation"); + // FIXME: denormalsEnabledForType is broken for dynamic + return denormalsEnabledForType(DAG, Op.getValueType()) && + DAG.isKnownNeverSNaN(Op); } -bool SITargetLowering::isCanonicalized(Register Reg, MachineFunction &MF, +bool SITargetLowering::isCanonicalized(Register Reg, const MachineFunction &MF, unsigned MaxDepth) const { - MachineRegisterInfo &MRI = MF.getRegInfo(); + const MachineRegisterInfo &MRI = MF.getRegInfo(); MachineInstr *MI = MRI.getVRegDef(Reg); unsigned Opcode = MI->getOpcode(); @@ -12931,27 +12956,7 @@ SDValue SITargetLowering::performFCanonicalizeCombine( } } - unsigned SrcOpc = N0.getOpcode(); - - // If it's free to do so, push canonicalizes further up the source, which may - // find a canonical source. - // - // TODO: More opcodes. Note this is unsafe for the _ieee minnum/maxnum for - // sNaNs. - if (SrcOpc == ISD::FMINNUM || SrcOpc == ISD::FMAXNUM) { - auto *CRHS = dyn_cast(N0.getOperand(1)); - if (CRHS && N0.hasOneUse()) { - SDLoc SL(N); - SDValue Canon0 = DAG.getNode(ISD::FCANONICALIZE, SL, VT, - N0.getOperand(0)); - SDValue Canon1 = getCanonicalConstantFP(DAG, SL, VT, CRHS->getValueAPF()); - DCI.AddToWorklist(Canon0.getNode()); - - return DAG.getNode(N0.getOpcode(), SL, VT, Canon0, Canon1); - } - } - - return isCanonicalized(DAG, N0) ? N0 : SDValue(); + return SDValue(); } static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) { @@ -15939,8 +15944,8 @@ bool SITargetLowering::denormalsEnabledForType(const SelectionDAG &DAG, } } -bool SITargetLowering::denormalsEnabledForType(LLT Ty, - MachineFunction &MF) const { +bool SITargetLowering::denormalsEnabledForType( + LLT Ty, const MachineFunction &MF) const { switch (Ty.getScalarSizeInBits()) { case 32: return !denormalModeIsFlushAllF32(MF); diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index a20442e3737e..89da4428e3ab 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -523,10 +523,10 @@ public: bool isCanonicalized(SelectionDAG &DAG, SDValue Op, unsigned MaxDepth = 5) const; - bool isCanonicalized(Register Reg, MachineFunction &MF, + bool isCanonicalized(Register Reg, const MachineFunction &MF, unsigned MaxDepth = 5) const; bool denormalsEnabledForType(const SelectionDAG &DAG, EVT VT) const; - bool denormalsEnabledForType(LLT Ty, MachineFunction &MF) const; + bool denormalsEnabledForType(LLT Ty, const MachineFunction &MF) const; bool checkForPhysRegDependency(SDNode *Def, SDNode *User, unsigned Op, const TargetRegisterInfo *TRI, diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index 33c93cdf20c4..3ab788406ecb 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -2944,6 +2944,34 @@ def : GCNPat< (V_BFREV_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1))), sub0, (V_BFREV_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0))), sub1)>; +// If fcanonicalize's operand is implicitly canonicalized, we only need a copy. +let AddedComplexity = 1000 in { +def : GCNPat< + (is_canonicalized_1 f16:$src), + (COPY f16:$src) +>; + +def : GCNPat< + (is_canonicalized_1 v2f16:$src), + (COPY v2f16:$src) +>; + +def : GCNPat< + (is_canonicalized_1 f32:$src), + (COPY f32:$src) +>; + +def : GCNPat< + (is_canonicalized_1 v2f32:$src), + (COPY v2f32:$src) +>; + +def : GCNPat< + (is_canonicalized_1 f64:$src), + (COPY f64:$src) +>; +} + // Prefer selecting to max when legal, but using mul is always valid. let AddedComplexity = -5 in { @@ -3277,8 +3305,8 @@ def : GCNPat < let AddedComplexity = 5 in { def : GCNPat < - (v2f16 (is_canonicalized (f16 (VOP3Mods (f16 VGPR_32:$src0), i32:$src0_mods)), - (f16 (VOP3Mods (f16 VGPR_32:$src1), i32:$src1_mods)))), + (v2f16 (is_canonicalized_2 (f16 (VOP3Mods (f16 VGPR_32:$src0), i32:$src0_mods)), + (f16 (VOP3Mods (f16 VGPR_32:$src1), i32:$src1_mods)))), (V_PACK_B32_F16_e64 $src0_mods, VGPR_32:$src0, $src1_mods, VGPR_32:$src1) >; } @@ -3590,6 +3618,17 @@ FPMinMaxPat; +class +FPMinCanonMaxPat : GCNPat < + (min_or_max (is_canonicalized_1 + (max_or_min_oneuse (VOP3Mods vt:$src0, i32:$src0_mods), + (VOP3Mods vt:$src1, i32:$src1_mods))), + (vt (VOP3Mods vt:$src2, i32:$src2_mods))), + (minmaxInst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2, + DSTCLAMP.NONE, DSTOMOD.NONE) +>; + let OtherPredicates = [isGFX11Plus] in { def : IntMinMaxPat; def : IntMinMaxPat; @@ -3599,6 +3638,10 @@ def : FPMinMaxPat; def : FPMinMaxPat; def : FPMinMaxPat; def : FPMinMaxPat; +def : FPMinCanonMaxPat; +def : FPMinCanonMaxPat; +def : FPMinCanonMaxPat; +def : FPMinCanonMaxPat; } let OtherPredicates = [isGFX9Plus] in { @@ -3612,6 +3655,10 @@ def : FPMinMaxPat, fmi def : FPMinMaxPat, fmaximum_oneuse>; def : FPMinMaxPat, fminimum_oneuse>; def : FPMinMaxPat, fmaximum_oneuse>; +def : FPMinCanonMaxPat, fminimum_oneuse>; +def : FPMinCanonMaxPat, fmaximum_oneuse>; +def : FPMinCanonMaxPat, fminimum_oneuse>; +def : FPMinCanonMaxPat, fmaximum_oneuse>; } // Convert a floating-point power of 2 to the integer exponent. diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll index ebb77c13c4af..98658834e897 100644 --- a/llvm/test/CodeGen/AMDGPU/bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/bf16.ll @@ -16968,7 +16968,7 @@ define bfloat @v_fabs_bf16(bfloat %a) { ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e64 v0, 1.0, |v0| +; GCN-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] ; @@ -16977,7 +16977,7 @@ define bfloat @v_fabs_bf16(bfloat %a) { ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e64 v0, 1.0, |v0| +; GFX7-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; @@ -17163,9 +17163,9 @@ define bfloat @v_fneg_fabs_bf16(bfloat %a) { ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e64 v0, 1.0, |v0| +; GCN-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v0, -1.0, v0 +; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] ; @@ -17174,9 +17174,9 @@ define bfloat @v_fneg_fabs_bf16(bfloat %a) { ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e64 v0, 1.0, |v0| +; GFX7-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v0, -1.0, v0 +; GFX7-NEXT: v_xor_b32_e32 v0, 0x80000000, v0 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; @@ -17280,8 +17280,6 @@ define bfloat @v_minnum_bf16(bfloat %a, bfloat %b) { ; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_min_f32_e32 v0, v0, v1 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] @@ -17293,8 +17291,6 @@ define bfloat @v_minnum_bf16(bfloat %a, bfloat %b) { ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_min_f32_e32 v0, v0, v1 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] @@ -17375,10 +17371,6 @@ define <2 x bfloat> @v_minnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) { ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_min_f32_e32 v1, v1, v3 ; GCN-NEXT: v_min_f32_e32 v0, v0, v2 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 @@ -17396,10 +17388,6 @@ define <2 x bfloat> @v_minnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_min_f32_e32 v1, v1, v3 ; GFX7-NEXT: v_min_f32_e32 v0, v0, v2 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 @@ -17522,12 +17510,6 @@ define <3 x bfloat> @v_minnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) { ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_min_f32_e32 v2, v2, v5 ; GCN-NEXT: v_min_f32_e32 v1, v1, v4 ; GCN-NEXT: v_min_f32_e32 v0, v0, v3 @@ -17551,12 +17533,6 @@ define <3 x bfloat> @v_minnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_min_f32_e32 v2, v2, v5 ; GFX7-NEXT: v_min_f32_e32 v1, v1, v4 ; GFX7-NEXT: v_min_f32_e32 v0, v0, v3 @@ -17688,14 +17664,6 @@ define <4 x bfloat> @v_minnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) { ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7 -; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6 -; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_min_f32_e32 v3, v3, v7 ; GCN-NEXT: v_min_f32_e32 v2, v2, v6 ; GCN-NEXT: v_min_f32_e32 v1, v1, v5 @@ -17725,14 +17693,6 @@ define <4 x bfloat> @v_minnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7 -; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 -; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_min_f32_e32 v3, v3, v7 ; GFX7-NEXT: v_min_f32_e32 v2, v2, v6 ; GFX7-NEXT: v_min_f32_e32 v1, v1, v5 @@ -17951,22 +17911,6 @@ define <8 x bfloat> @v_minnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) { ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15 -; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7 -; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14 -; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6 -; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13 -; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12 -; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11 -; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10 -; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_min_f32_e32 v7, v7, v15 ; GCN-NEXT: v_min_f32_e32 v6, v6, v14 ; GCN-NEXT: v_min_f32_e32 v5, v5, v13 @@ -18020,22 +17964,6 @@ define <8 x bfloat> @v_minnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15 -; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7 -; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14 -; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 -; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13 -; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12 -; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11 -; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10 -; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_min_f32_e32 v7, v7, v15 ; GFX7-NEXT: v_min_f32_e32 v6, v6, v14 ; GFX7-NEXT: v_min_f32_e32 v5, v5, v13 @@ -18382,71 +18310,51 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30 ; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 ; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30 -; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14 ; GCN-NEXT: v_min_f32_e32 v14, v14, v30 ; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13 ; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29 ; GCN-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 ; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29 -; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13 ; GCN-NEXT: v_min_f32_e32 v13, v13, v29 ; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12 ; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28 ; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 ; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28 -; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12 ; GCN-NEXT: v_min_f32_e32 v12, v12, v28 ; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11 ; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27 ; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 ; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27 -; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11 ; GCN-NEXT: v_min_f32_e32 v11, v11, v27 ; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26 ; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 ; GCN-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26 -; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GCN-NEXT: v_min_f32_e32 v10, v10, v26 ; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9 ; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25 ; GCN-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 ; GCN-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25 -; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9 ; GCN-NEXT: v_min_f32_e32 v9, v9, v25 ; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24 ; GCN-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 ; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24 -; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GCN-NEXT: v_min_f32_e32 v8, v8, v24 ; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23 ; GCN-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 ; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23 -; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GCN-NEXT: v_min_f32_e32 v7, v7, v23 ; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6 ; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22 ; GCN-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 ; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22 -; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6 ; GCN-NEXT: v_min_f32_e32 v6, v6, v22 ; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 ; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21 ; GCN-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 ; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21 -; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 ; GCN-NEXT: v_min_f32_e32 v5, v5, v21 ; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16 @@ -18461,8 +18369,6 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15 ; GCN-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 ; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20 -; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4 ; GCN-NEXT: v_min_f32_e32 v4, v4, v20 ; GCN-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 @@ -18474,21 +18380,10 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15 -; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19 -; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18 -; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_min_f32_e32 v3, v3, v19 ; GCN-NEXT: v_min_f32_e32 v2, v2, v18 ; GCN-NEXT: v_min_f32_e32 v1, v1, v17 ; GCN-NEXT: v_min_f32_e32 v0, v0, v16 -; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v20 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 @@ -18503,8 +18398,9 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 ; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 ; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v20 ; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16 ; GCN-NEXT: v_min_f32_e32 v15, v15, v16 ; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 @@ -18513,14 +18409,12 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX7-LABEL: v_minnum_v16bf16: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9 -; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 -; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 -; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 -; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9 -; GFX7-NEXT: v_min_f32_e32 v9, v9, v25 -; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32 +; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 +; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22 +; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX7-NEXT: v_min_f32_e32 v6, v6, v22 +; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32 ; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14 ; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30 ; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13 @@ -18531,13 +18425,13 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27 ; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26 -; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15 +; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9 +; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 ; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24 ; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23 -; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 -; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22 +; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 ; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 @@ -18560,13 +18454,13 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 ; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 ; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 +; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 ; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 ; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 @@ -18579,48 +18473,14 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30 -; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14 -; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29 -; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13 -; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28 -; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12 -; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27 -; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11 -; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26 -; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10 -; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15 -; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24 -; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8 -; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23 -; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7 -; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22 -; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 -; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21 -; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20 -; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 -; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 -; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 -; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19 -; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18 -; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_min_f32_e32 v14, v14, v30 ; GFX7-NEXT: v_min_f32_e32 v13, v13, v29 ; GFX7-NEXT: v_min_f32_e32 v12, v12, v28 ; GFX7-NEXT: v_min_f32_e32 v11, v11, v27 ; GFX7-NEXT: v_min_f32_e32 v10, v10, v26 -; GFX7-NEXT: v_min_f32_e32 v15, v15, v25 +; GFX7-NEXT: v_min_f32_e32 v9, v9, v25 ; GFX7-NEXT: v_min_f32_e32 v8, v8, v24 ; GFX7-NEXT: v_min_f32_e32 v7, v7, v23 -; GFX7-NEXT: v_min_f32_e32 v6, v6, v22 ; GFX7-NEXT: v_min_f32_e32 v5, v5, v21 ; GFX7-NEXT: v_min_f32_e32 v4, v4, v20 ; GFX7-NEXT: v_min_f32_e32 v3, v3, v19 @@ -18634,6 +18494,10 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22 +; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX7-NEXT: v_min_f32_e32 v15, v15, v22 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 @@ -19267,287 +19131,223 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) { ; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 ; GCN-NEXT: v_and_b32_e32 v31, 0xffff0000, v31 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 -; GCN-NEXT: v_mul_f32_e32 v31, 1.0, v31 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:124 ; GCN-NEXT: v_min_f32_e32 v31, v31, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 ; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30 ; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 -; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:120 ; GCN-NEXT: v_min_f32_e32 v30, v30, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 ; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29 ; GCN-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 -; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:116 ; GCN-NEXT: v_min_f32_e32 v29, v29, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116 ; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28 ; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 -; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; GCN-NEXT: v_min_f32_e32 v28, v28, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 ; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27 ; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 -; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:108 ; GCN-NEXT: v_min_f32_e32 v27, v27, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108 ; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26 ; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 -; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:104 ; GCN-NEXT: v_min_f32_e32 v26, v26, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 ; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25 ; GCN-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 -; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:100 ; GCN-NEXT: v_min_f32_e32 v25, v25, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100 ; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24 ; GCN-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 -; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:96 ; GCN-NEXT: v_min_f32_e32 v24, v24, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 ; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23 ; GCN-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 -; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:92 ; GCN-NEXT: v_min_f32_e32 v23, v23, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 ; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22 ; GCN-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:88 ; GCN-NEXT: v_min_f32_e32 v22, v22, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21 ; GCN-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 -; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:84 ; GCN-NEXT: v_min_f32_e32 v21, v21, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84 ; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20 ; GCN-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 -; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:80 ; GCN-NEXT: v_min_f32_e32 v20, v20, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 ; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19 ; GCN-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 -; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:76 ; GCN-NEXT: v_min_f32_e32 v19, v19, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76 ; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18 ; GCN-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 -; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:72 ; GCN-NEXT: v_min_f32_e32 v18, v18, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 ; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17 ; GCN-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:68 ; GCN-NEXT: v_min_f32_e32 v17, v17, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 ; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16 ; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 ; GCN-NEXT: v_min_f32_e32 v16, v16, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 ; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15 ; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; GCN-NEXT: v_min_f32_e32 v15, v15, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14 ; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:56 ; GCN-NEXT: v_min_f32_e32 v14, v14, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 ; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13 ; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:52 ; GCN-NEXT: v_min_f32_e32 v13, v13, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52 ; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12 ; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:48 ; GCN-NEXT: v_min_f32_e32 v12, v12, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 ; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11 ; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:44 ; GCN-NEXT: v_min_f32_e32 v11, v11, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44 ; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GCN-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:40 ; GCN-NEXT: v_min_f32_e32 v10, v10, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 ; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9 ; GCN-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:36 ; GCN-NEXT: v_min_f32_e32 v9, v9, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36 ; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:32 ; GCN-NEXT: v_min_f32_e32 v8, v8, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 ; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:28 ; GCN-NEXT: v_min_f32_e32 v7, v7, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28 ; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6 ; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:24 ; GCN-NEXT: v_min_f32_e32 v6, v6, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 ; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 ; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:20 ; GCN-NEXT: v_min_f32_e32 v5, v5, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20 ; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4 ; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:16 ; GCN-NEXT: v_min_f32_e32 v4, v4, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 ; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 ; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12 ; GCN-NEXT: v_min_f32_e32 v3, v3, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12 ; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 ; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8 ; GCN-NEXT: v_min_f32_e32 v2, v2, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 ; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:4 ; GCN-NEXT: v_min_f32_e32 v1, v1, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 ; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GCN-NEXT: v_min_f32_e32 v0, v0, v32 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 @@ -19590,322 +19390,258 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) { ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128 ; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30 ; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 -; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30 ; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29 ; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 -; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29 ; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28 ; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 -; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28 ; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27 ; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 -; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27 ; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26 ; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 -; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26 ; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 ; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 -; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 ; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24 ; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 -; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24 ; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23 ; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 -; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23 ; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22 ; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22 ; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21 ; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 -; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21 ; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20 ; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 -; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20 ; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19 ; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 -; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19 ; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18 ; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 -; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18 ; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17 ; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17 ; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16 ; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16 ; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15 ; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15 ; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14 ; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14 ; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13 ; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13 ; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12 ; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12 ; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11 ; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11 ; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9 ; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9 ; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GFX7-NEXT: s_waitcnt vmcnt(1) -; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31 -; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 -; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 -; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31 -; GFX7-NEXT: v_min_f32_e32 v31, v31, v32 -; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31 +; GFX7-NEXT: s_waitcnt vmcnt(1) +; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 +; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31 +; GFX7-NEXT: v_min_f32_e32 v31, v31, v32 +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 +; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31 +; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 ; GFX7-NEXT: v_min_f32_e32 v30, v30, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 ; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v29, v29, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116 ; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v28, v28, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 ; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v27, v27, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108 ; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v26, v26, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 ; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v25, v25, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100 ; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v24, v24, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 ; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v23, v23, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 ; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v22, v22, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v21, v21, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84 ; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v20, v20, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 ; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v19, v19, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76 ; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v18, v18, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 ; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v17, v17, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 ; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v16, v16, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 ; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v15, v15, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v14, v14, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 ; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v13, v13, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52 ; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v12, v12, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 ; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v11, v11, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44 ; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v10, v10, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 ; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v9, v9, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36 ; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v8, v8, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v7, v7, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v6, v6, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v5, v5, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v4, v4, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v3, v3, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v2, v2, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v1, v1, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_min_f32_e32 v0, v0, v32 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] @@ -21097,8 +20833,6 @@ define bfloat @v_maxnum_bf16(bfloat %a, bfloat %b) { ; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_max_f32_e32 v0, v0, v1 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] @@ -21110,8 +20844,6 @@ define bfloat @v_maxnum_bf16(bfloat %a, bfloat %b) { ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_max_f32_e32 v0, v0, v1 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] @@ -21192,10 +20924,6 @@ define <2 x bfloat> @v_maxnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) { ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_max_f32_e32 v1, v1, v3 ; GCN-NEXT: v_max_f32_e32 v0, v0, v2 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 @@ -21213,10 +20941,6 @@ define <2 x bfloat> @v_maxnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX7-NEXT: v_max_f32_e32 v0, v0, v2 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 @@ -21339,12 +21063,6 @@ define <3 x bfloat> @v_maxnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) { ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_max_f32_e32 v2, v2, v5 ; GCN-NEXT: v_max_f32_e32 v1, v1, v4 ; GCN-NEXT: v_max_f32_e32 v0, v0, v3 @@ -21368,12 +21086,6 @@ define <3 x bfloat> @v_maxnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v5 ; GFX7-NEXT: v_max_f32_e32 v1, v1, v4 ; GFX7-NEXT: v_max_f32_e32 v0, v0, v3 @@ -21505,14 +21217,6 @@ define <4 x bfloat> @v_maxnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) { ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7 -; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6 -; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_max_f32_e32 v3, v3, v7 ; GCN-NEXT: v_max_f32_e32 v2, v2, v6 ; GCN-NEXT: v_max_f32_e32 v1, v1, v5 @@ -21542,14 +21246,6 @@ define <4 x bfloat> @v_maxnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7 -; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 -; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_max_f32_e32 v3, v3, v7 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v6 ; GFX7-NEXT: v_max_f32_e32 v1, v1, v5 @@ -21768,22 +21464,6 @@ define <8 x bfloat> @v_maxnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) { ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15 -; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7 -; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14 -; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6 -; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13 -; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12 -; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11 -; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10 -; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_max_f32_e32 v7, v7, v15 ; GCN-NEXT: v_max_f32_e32 v6, v6, v14 ; GCN-NEXT: v_max_f32_e32 v5, v5, v13 @@ -21837,22 +21517,6 @@ define <8 x bfloat> @v_maxnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15 -; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7 -; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14 -; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 -; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13 -; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12 -; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11 -; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10 -; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v15 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v14 ; GFX7-NEXT: v_max_f32_e32 v5, v5, v13 @@ -22199,71 +21863,51 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30 ; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 ; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30 -; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14 ; GCN-NEXT: v_max_f32_e32 v14, v14, v30 ; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13 ; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29 ; GCN-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 ; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29 -; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13 ; GCN-NEXT: v_max_f32_e32 v13, v13, v29 ; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12 ; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28 ; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 ; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28 -; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12 ; GCN-NEXT: v_max_f32_e32 v12, v12, v28 ; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11 ; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27 ; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 ; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27 -; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11 ; GCN-NEXT: v_max_f32_e32 v11, v11, v27 ; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26 ; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 ; GCN-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26 -; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GCN-NEXT: v_max_f32_e32 v10, v10, v26 ; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9 ; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25 ; GCN-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 ; GCN-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25 -; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9 ; GCN-NEXT: v_max_f32_e32 v9, v9, v25 ; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24 ; GCN-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 ; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24 -; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GCN-NEXT: v_max_f32_e32 v8, v8, v24 ; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23 ; GCN-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 ; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23 -; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GCN-NEXT: v_max_f32_e32 v7, v7, v23 ; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6 ; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22 ; GCN-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 ; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22 -; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6 ; GCN-NEXT: v_max_f32_e32 v6, v6, v22 ; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 ; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21 ; GCN-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 ; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21 -; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 ; GCN-NEXT: v_max_f32_e32 v5, v5, v21 ; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16 @@ -22278,8 +21922,6 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15 ; GCN-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 ; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20 -; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4 ; GCN-NEXT: v_max_f32_e32 v4, v4, v20 ; GCN-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 @@ -22291,21 +21933,10 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15 -; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19 -; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18 -; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_max_f32_e32 v3, v3, v19 ; GCN-NEXT: v_max_f32_e32 v2, v2, v18 ; GCN-NEXT: v_max_f32_e32 v1, v1, v17 ; GCN-NEXT: v_max_f32_e32 v0, v0, v16 -; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v20 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 @@ -22320,8 +21951,9 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 ; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 ; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v20 ; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16 ; GCN-NEXT: v_max_f32_e32 v15, v15, v16 ; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 @@ -22330,14 +21962,12 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX7-LABEL: v_maxnum_v16bf16: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9 -; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 -; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 -; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 -; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9 -; GFX7-NEXT: v_max_f32_e32 v9, v9, v25 -; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32 +; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 +; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22 +; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX7-NEXT: v_max_f32_e32 v6, v6, v22 +; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32 ; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14 ; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30 ; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13 @@ -22348,13 +21978,13 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27 ; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26 -; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15 +; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9 +; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 ; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24 ; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23 -; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 -; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22 +; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 ; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 @@ -22377,13 +22007,13 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 ; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 ; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 +; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 ; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 ; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 @@ -22392,52 +22022,18 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30 -; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14 -; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29 -; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13 -; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28 -; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12 -; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27 -; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11 -; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26 -; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10 -; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15 -; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24 -; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8 -; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23 -; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7 -; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22 -; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 -; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21 -; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20 -; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 -; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 -; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 -; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19 -; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18 -; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 +; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 +; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX7-NEXT: v_max_f32_e32 v14, v14, v30 ; GFX7-NEXT: v_max_f32_e32 v13, v13, v29 ; GFX7-NEXT: v_max_f32_e32 v12, v12, v28 ; GFX7-NEXT: v_max_f32_e32 v11, v11, v27 ; GFX7-NEXT: v_max_f32_e32 v10, v10, v26 -; GFX7-NEXT: v_max_f32_e32 v15, v15, v25 +; GFX7-NEXT: v_max_f32_e32 v9, v9, v25 ; GFX7-NEXT: v_max_f32_e32 v8, v8, v24 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v23 -; GFX7-NEXT: v_max_f32_e32 v6, v6, v22 ; GFX7-NEXT: v_max_f32_e32 v5, v5, v21 ; GFX7-NEXT: v_max_f32_e32 v4, v4, v20 ; GFX7-NEXT: v_max_f32_e32 v3, v3, v19 @@ -22451,6 +22047,10 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22 +; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX7-NEXT: v_max_f32_e32 v15, v15, v22 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 @@ -23084,287 +22684,223 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) { ; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 ; GCN-NEXT: v_and_b32_e32 v31, 0xffff0000, v31 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 -; GCN-NEXT: v_mul_f32_e32 v31, 1.0, v31 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:124 ; GCN-NEXT: v_max_f32_e32 v31, v31, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 ; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30 ; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 -; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:120 ; GCN-NEXT: v_max_f32_e32 v30, v30, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 ; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29 ; GCN-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 -; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:116 ; GCN-NEXT: v_max_f32_e32 v29, v29, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116 ; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28 ; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 -; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; GCN-NEXT: v_max_f32_e32 v28, v28, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 ; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27 ; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 -; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:108 ; GCN-NEXT: v_max_f32_e32 v27, v27, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108 ; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26 ; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 -; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:104 ; GCN-NEXT: v_max_f32_e32 v26, v26, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 ; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25 ; GCN-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 -; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:100 ; GCN-NEXT: v_max_f32_e32 v25, v25, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100 ; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24 ; GCN-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 -; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:96 ; GCN-NEXT: v_max_f32_e32 v24, v24, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 ; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23 ; GCN-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 -; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:92 ; GCN-NEXT: v_max_f32_e32 v23, v23, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 ; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22 ; GCN-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:88 ; GCN-NEXT: v_max_f32_e32 v22, v22, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21 ; GCN-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 -; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:84 ; GCN-NEXT: v_max_f32_e32 v21, v21, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84 ; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20 ; GCN-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 -; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:80 ; GCN-NEXT: v_max_f32_e32 v20, v20, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 ; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19 ; GCN-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 -; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:76 ; GCN-NEXT: v_max_f32_e32 v19, v19, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76 ; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18 ; GCN-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 -; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:72 ; GCN-NEXT: v_max_f32_e32 v18, v18, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 ; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17 ; GCN-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:68 ; GCN-NEXT: v_max_f32_e32 v17, v17, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 ; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16 ; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 ; GCN-NEXT: v_max_f32_e32 v16, v16, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 ; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15 ; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; GCN-NEXT: v_max_f32_e32 v15, v15, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14 ; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:56 ; GCN-NEXT: v_max_f32_e32 v14, v14, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 ; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13 ; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:52 ; GCN-NEXT: v_max_f32_e32 v13, v13, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52 ; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12 ; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:48 ; GCN-NEXT: v_max_f32_e32 v12, v12, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 ; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11 ; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:44 ; GCN-NEXT: v_max_f32_e32 v11, v11, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44 ; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GCN-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:40 ; GCN-NEXT: v_max_f32_e32 v10, v10, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 ; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9 ; GCN-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:36 ; GCN-NEXT: v_max_f32_e32 v9, v9, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36 ; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:32 ; GCN-NEXT: v_max_f32_e32 v8, v8, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 ; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:28 ; GCN-NEXT: v_max_f32_e32 v7, v7, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28 ; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6 ; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:24 ; GCN-NEXT: v_max_f32_e32 v6, v6, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 ; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 ; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:20 ; GCN-NEXT: v_max_f32_e32 v5, v5, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20 ; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4 ; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:16 ; GCN-NEXT: v_max_f32_e32 v4, v4, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 ; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 ; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12 ; GCN-NEXT: v_max_f32_e32 v3, v3, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12 ; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 ; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8 ; GCN-NEXT: v_max_f32_e32 v2, v2, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 ; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:4 ; GCN-NEXT: v_max_f32_e32 v1, v1, v32 -; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 ; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33 ; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GCN-NEXT: v_max_f32_e32 v0, v0, v32 ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 @@ -23407,322 +22943,258 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) { ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128 ; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30 ; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 -; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30 ; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29 ; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 -; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29 ; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28 ; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 -; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28 ; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27 ; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 -; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27 ; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26 ; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 -; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26 ; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 ; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 -; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25 ; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24 ; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 -; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24 ; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23 ; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 -; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23 ; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22 ; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22 ; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21 ; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 -; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21 ; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20 ; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 -; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20 ; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19 ; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 -; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19 ; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18 ; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 -; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18 ; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17 ; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17 ; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16 ; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16 ; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15 ; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15 ; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14 ; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14 ; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13 ; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13 ; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12 ; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12 ; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11 ; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11 ; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10 ; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9 ; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9 ; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8 ; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; GFX7-NEXT: s_waitcnt vmcnt(1) -; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31 -; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 -; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 -; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31 -; GFX7-NEXT: v_max_f32_e32 v31, v31, v32 -; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2 ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31 +; GFX7-NEXT: s_waitcnt vmcnt(1) +; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 +; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31 +; GFX7-NEXT: v_max_f32_e32 v31, v31, v32 +; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 +; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31 +; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 +; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 ; GFX7-NEXT: v_max_f32_e32 v30, v30, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 ; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v29, v29, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116 ; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v28, v28, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 ; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v27, v27, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108 ; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v26, v26, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 ; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v25, v25, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100 ; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v24, v24, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 ; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v23, v23, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 ; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v22, v22, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v21, v21, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84 ; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v20, v20, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 ; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v19, v19, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76 ; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v18, v18, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 ; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v17, v17, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 ; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v16, v16, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 ; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v15, v15, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v14, v14, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 ; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v13, v13, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52 ; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v12, v12, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 ; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v11, v11, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44 ; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v10, v10, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 ; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v9, v9, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36 ; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v8, v8, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v5, v5, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v4, v4, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v3, v3, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v1, v1, v32 ; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32 -; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32 ; GFX7-NEXT: v_max_f32_e32 v0, v0, v32 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] @@ -25176,7 +24648,6 @@ define { bfloat, i16 } @v_frexp_bf16_i16(bfloat %a) { ; GCN-NEXT: v_frexp_exp_i32_f32_e32 v2, v0 ; GCN-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s4 ; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc ; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] @@ -26818,11 +26289,17 @@ define bfloat @v_canonicalize_bf16(bfloat %a) { ; GCN-LABEL: v_canonicalize_bf16: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0 +; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GFX7-LABEL: v_canonicalize_bf16: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0 +; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] ; ; GFX8-LABEL: v_canonicalize_bf16: diff --git a/llvm/test/CodeGen/AMDGPU/clamp.ll b/llvm/test/CodeGen/AMDGPU/clamp.ll index dfadd8d205b0..947284506a29 100644 --- a/llvm/test/CodeGen/AMDGPU/clamp.ll +++ b/llvm/test/CodeGen/AMDGPU/clamp.ll @@ -2996,18 +2996,16 @@ define amdgpu_kernel void @v_clamp_v2f16_undef_elt(ptr addrspace(1) %out, ptr ad ; GFX6-NEXT: v_mov_b32_e32 v4, 0x7fc00000 ; GFX6-NEXT: s_mov_b64 s[2:3], s[6:7] ; GFX6-NEXT: s_waitcnt vmcnt(0) -; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2 -; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v2 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2 -; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX6-NEXT: v_max_f32_e32 v3, 0x7fc00000, v3 -; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GFX6-NEXT: v_med3_f32 v2, v2, 0, v4 -; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2 -; GFX6-NEXT: v_min_f32_e32 v3, 1.0, v3 +; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3 +; GFX6-NEXT: v_max_f32_e32 v2, 0x7fc00000, v2 +; GFX6-NEXT: v_med3_f32 v3, v3, 0, v4 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3 -; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX6-NEXT: v_or_b32_e32 v2, v3, v2 +; GFX6-NEXT: v_min_f32_e32 v2, 1.0, v2 +; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2 +; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX6-NEXT: v_or_b32_e32 v2, v2, v3 ; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 ; GFX6-NEXT: s_endpgm ; @@ -3095,16 +3093,15 @@ define amdgpu_kernel void @v_clamp_v2f16_not_zero(ptr addrspace(1) %out, ptr add ; GFX6-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 ; GFX6-NEXT: s_mov_b64 s[2:3], s[6:7] ; GFX6-NEXT: s_waitcnt vmcnt(0) -; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2 -; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX6-NEXT: v_cvt_f32_f16_e64 v2, v2 clamp -; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX6-NEXT: v_max_f32_e32 v3, 2.0, v3 -; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2 -; GFX6-NEXT: v_min_f32_e32 v3, 1.0, v3 +; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v2 +; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2 +; GFX6-NEXT: v_cvt_f32_f16_e64 v3, v3 clamp +; GFX6-NEXT: v_max_f32_e32 v2, 2.0, v2 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3 -; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX6-NEXT: v_or_b32_e32 v2, v3, v2 +; GFX6-NEXT: v_min_f32_e32 v2, 1.0, v2 +; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2 +; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX6-NEXT: v_or_b32_e32 v2, v2, v3 ; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 ; GFX6-NEXT: s_endpgm ; @@ -3198,9 +3195,8 @@ define amdgpu_kernel void @v_clamp_v2f16_not_one(ptr addrspace(1) %out, ptr addr ; GFX6-NEXT: s_mov_b64 s[2:3], s[6:7] ; GFX6-NEXT: s_waitcnt vmcnt(0) ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v2 -; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2 ; GFX6-NEXT: v_cvt_f32_f16_e64 v3, v3 clamp -; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2 +; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3 ; GFX6-NEXT: v_med3_f32 v2, v2, 0, 0 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2 @@ -3760,19 +3756,17 @@ define amdgpu_kernel void @v_clamp_v2f16_undef_limit_elts0(ptr addrspace(1) %out ; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3] ; GFX6-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 -; GFX6-NEXT: s_mov_b32 s2, 0x7fc00000 ; GFX6-NEXT: v_mov_b32_e32 v4, 0x7fc00000 +; GFX6-NEXT: s_mov_b64 s[2:3], s[6:7] ; GFX6-NEXT: s_waitcnt vmcnt(0) ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v2 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2 -; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GFX6-NEXT: v_med3_f32 v3, v3, s2, 1.0 +; GFX6-NEXT: v_max_f32_e32 v3, 0x7fc00000, v3 +; GFX6-NEXT: v_min_f32_e32 v3, 1.0, v3 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3 ; GFX6-NEXT: v_med3_f32 v2, v2, 0, v4 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2 -; GFX6-NEXT: s_mov_b64 s[2:3], s[6:7] ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX6-NEXT: v_or_b32_e32 v2, v2, v3 ; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 @@ -3863,18 +3857,16 @@ define amdgpu_kernel void @v_clamp_v2f16_undef_limit_elts1(ptr addrspace(1) %out ; GFX6-NEXT: v_mov_b32_e32 v4, 0x7fc00000 ; GFX6-NEXT: s_mov_b64 s[2:3], s[6:7] ; GFX6-NEXT: s_waitcnt vmcnt(0) -; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2 -; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v2 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2 -; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; GFX6-NEXT: v_max_f32_e32 v3, 0x7fc00000, v3 -; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; GFX6-NEXT: v_med3_f32 v2, v2, 0, v4 -; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2 -; GFX6-NEXT: v_min_f32_e32 v3, 1.0, v3 +; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3 +; GFX6-NEXT: v_max_f32_e32 v2, 0x7fc00000, v2 +; GFX6-NEXT: v_med3_f32 v3, v3, 0, v4 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3 -; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX6-NEXT: v_or_b32_e32 v2, v3, v2 +; GFX6-NEXT: v_min_f32_e32 v2, 1.0, v2 +; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2 +; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX6-NEXT: v_or_b32_e32 v2, v2, v3 ; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 ; GFX6-NEXT: s_endpgm ; diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll index 4ed1b8a520b8..e1981972f58d 100644 --- a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll +++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll @@ -471,25 +471,15 @@ define amdgpu_kernel void @test_fold_canonicalize_minnum_value_from_load_f32_iee ret void } -; GCN-LABEL: test_fold_canonicalize_minnum_value_from_load_f32_nnan_ieee_mode: -; VI-FLUSH: v_mul_f32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}} -; GCN-DENORM-NOT: v_max -; GCN-DENORM-NOT: v_mul - -; GCN: v_min_f32_e32 v{{[0-9]+}}, 0, v{{[0-9]+}} -; GCN-DENORM-NOT: v_max -; GCN-DENORM-NOT: v_mul - -; GFX9: {{flat|global}}_store_dword -define amdgpu_kernel void @test_fold_canonicalize_minnum_value_from_load_f32_nnan_ieee_mode(ptr addrspace(1) %arg) #1 { - %id = tail call i32 @llvm.amdgcn.workitem.id.x() - %gep = getelementptr inbounds float, ptr addrspace(1) %arg, i32 %id - %load = load float, ptr addrspace(1) %gep, align 4 - %v = tail call float @llvm.minnum.f32(float %load, float 0.0) - %canonicalized = tail call float @llvm.canonicalize.f32(float %v) - store float %canonicalized, ptr addrspace(1) %gep, align 4 - ret void -} +; define amdgpu_kernel void @test_fold_canonicalize_minnum_value_from_load_f32_nnan_ieee_mode(ptr addrspace(1) %arg) #1 { +; %id = tail call i32 @llvm.amdgcn.workitem.id.x() +; %gep = getelementptr inbounds float, ptr addrspace(1) %arg, i32 %id +; %load = load float, ptr addrspace(1) %gep, align 4 +; %v = tail call float @llvm.minnum.f32(float %load, float 0.0) +; %canonicalized = tail call float @llvm.canonicalize.f32(float %v) +; store float %canonicalized, ptr addrspace(1) %gep, align 4 +; ret void +; } ; GCN-LABEL: test_fold_canonicalize_minnum_value_f32: ; GCN: v_min_f32_e32 [[V:v[0-9]+]], 0, v{{[0-9]+}} @@ -523,32 +513,15 @@ define amdgpu_kernel void @test_fold_canonicalize_sNaN_value_f32(ptr addrspace(1 ret void } -; GCN-LABEL: test_fold_canonicalize_denorm_value_f32: -; GCN: {{flat|global}}_load_dword [[VAL:v[0-9]+]] - -; GFX9-DENORM: v_max_f32_e32 [[QUIET:v[0-9]+]], [[VAL]], [[VAL]] -; GFX9-DENORM: v_min_f32_e32 [[RESULT:v[0-9]+]], 0x7fffff, [[QUIET]] - -; GFX9-FLUSH: v_max_f32_e32 [[QUIET:v[0-9]+]], [[VAL]], [[VAL]] -; GFX9-FLUSH: v_min_f32_e32 [[RESULT:v[0-9]+]], 0, [[QUIET]] - -; VI-FLUSH: v_mul_f32_e32 [[QUIET_V0:v[0-9]+]], 1.0, [[VAL]] -; VI-FLUSH: v_min_f32_e32 [[RESULT:v[0-9]+]], 0, [[QUIET_V0]] - -; VI-DENORM: v_min_f32_e32 [[RESULT:v[0-9]+]], 0x7fffff, [[VAL]] - -; GCN-NOT: v_mul -; GCN-NOT: v_max -; GCN: {{flat|global}}_store_dword v{{.+}}, [[RESULT]] -define amdgpu_kernel void @test_fold_canonicalize_denorm_value_f32(ptr addrspace(1) %arg) { - %id = tail call i32 @llvm.amdgcn.workitem.id.x() - %gep = getelementptr inbounds float, ptr addrspace(1) %arg, i32 %id - %load = load float, ptr addrspace(1) %gep, align 4 - %v = tail call float @llvm.minnum.f32(float %load, float bitcast (i32 8388607 to float)) - %canonicalized = tail call float @llvm.canonicalize.f32(float %v) - store float %canonicalized, ptr addrspace(1) %gep, align 4 - ret void -} +; define amdgpu_kernel void @test_fold_canonicalize_denorm_value_f32(ptr addrspace(1) %arg) { +; %id = tail call i32 @llvm.amdgcn.workitem.id.x() +; %gep = getelementptr inbounds float, ptr addrspace(1) %arg, i32 %id +; %load = load float, ptr addrspace(1) %gep, align 4 +; %v = tail call float @llvm.minnum.f32(float %load, float bitcast (i32 8388607 to float)) +; %canonicalized = tail call float @llvm.canonicalize.f32(float %v) +; store float %canonicalized, ptr addrspace(1) %gep, align 4 +; ret void +; } ; GCN-LABEL: test_fold_canonicalize_maxnum_value_from_load_f32_ieee_mode: ; GCN: {{flat|global}}_load_dword [[VAL:v[0-9]+]] @@ -674,10 +647,9 @@ define amdgpu_kernel void @test_fold_canonicalize_load_nnan_value_f64(ptr addrsp } ; GCN-LABEL: {{^}}test_fold_canonicalize_load_nnan_value_f16 -; GCN: {{flat|global}}_load_ushort [[V:v[0-9]+]], -; GCN-NOT: v_mul -; GCN-NOT: v_max -; GCN: {{flat|global}}_store_short v{{.+}}, [[V]] +; GCN: {{flat|global}}_load_ushort [[V1:v[0-9]+]], +; GCN: v_max_f16_e32 [[V2:v[0-9]+]], [[V1]], [[V1]] +; GCN: {{flat|global}}_store_short v{{.+}}, [[V2]] define amdgpu_kernel void @test_fold_canonicalize_load_nnan_value_f16(ptr addrspace(1) %arg, ptr addrspace(1) %out) #1 { %id = tail call i32 @llvm.amdgcn.workitem.id.x() %gep = getelementptr inbounds half, ptr addrspace(1) %arg, i32 %id @@ -807,18 +779,13 @@ define half @v_test_canonicalize_extract_element_v2f16(<2 x half> %vec) { ret half %canonicalized } -; GCN-LABEL: {{^}}v_test_canonicalize_insertelement_v2f16: -; GFX9: v_mul_f16_e32 -; GFX9: v_pk_mul_f16 -; GFX9-NOT: v_max -; GFX9-NOT: v_pk_max -define <2 x half> @v_test_canonicalize_insertelement_v2f16(<2 x half> %vec, half %val, i32 %idx) { - %vec.op = fmul <2 x half> %vec, - %ins.op = fmul half %val, 8.0 - %ins = insertelement <2 x half> %vec.op, half %ins.op, i32 %idx - %canonicalized = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %ins) - ret <2 x half> %canonicalized -} +; define <2 x half> @v_test_canonicalize_insertelement_v2f16(<2 x half> %vec, half %val, i32 %idx) { +; %vec.op = fmul <2 x half> %vec, +; %ins.op = fmul half %val, 8.0 +; %ins = insertelement <2 x half> %vec.op, half %ins.op, i32 %idx +; %canonicalized = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %ins) +; ret <2 x half> %canonicalized +; } ; GCN-LABEL: {{^}}v_test_canonicalize_insertelement_noncanon_vec_v2f16: ; GFX9: v_mul_f16 @@ -842,15 +809,11 @@ define <2 x half> @v_test_canonicalize_insertelement_noncanon_insval_v2f16(<2 x ret <2 x half> %canonicalized } -; GCN-LABEL: {{^}}v_test_canonicalize_cvt_pkrtz: -; GCN: s_waitcnt -; GCN-NEXT: v_cvt_pkrtz_f16_f32 v0, v0, v1 -; GCN-NEXT: s_setpc_b64 -define <2 x half> @v_test_canonicalize_cvt_pkrtz(float %a, float %b) { - %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %a, float %b) - %canonicalized = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %cvt) - ret <2 x half> %canonicalized -} +; define <2 x half> @v_test_canonicalize_cvt_pkrtz(float %a, float %b) { +; %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %a, float %b) +; %canonicalized = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %cvt) +; ret <2 x half> %canonicalized +; } ; GCN-LABEL: {{^}}v_test_canonicalize_cubeid: ; GCN: s_waitcnt diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll index 274621307f54..581b7b4cff9e 100644 --- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll @@ -94,7 +94,6 @@ define amdgpu_kernel void @v_test_canonicalize_var_f16(ptr addrspace(1) %out) #1 ; CI-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; CI-NEXT: s_endpgm @@ -147,7 +146,6 @@ define amdgpu_kernel void @s_test_canonicalize_var_f16(ptr addrspace(1) %out, i1 ; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: v_cvt_f32_f16_e32 v0, s2 ; CI-NEXT: s_mov_b32 s2, -1 -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; CI-NEXT: s_endpgm @@ -170,6 +168,35 @@ define amdgpu_kernel void @s_test_canonicalize_var_f16(ptr addrspace(1) %out, i1 ret void } +define half @s_test_canonicalize_arg(half %x) #1 { +; VI-LABEL: s_test_canonicalize_arg: +; VI: ; %bb.0: +; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VI-NEXT: v_max_f16_e32 v0, v0, v0 +; VI-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: s_test_canonicalize_arg: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_max_f16_e32 v0, v0, v0 +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; CI-LABEL: s_test_canonicalize_arg: +; CI: ; %bb.0: +; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 +; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 +; CI-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: s_test_canonicalize_arg: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_max_f16_e32 v0, v0, v0 +; GFX11-NEXT: s_setpc_b64 s[30:31] + %canonicalized = call half @llvm.canonicalize.f16(half %x) + ret half %canonicalized +} + define <2 x half> @v_test_canonicalize_build_vector_v2f16(half %lo, half %hi) #1 { ; VI-LABEL: v_test_canonicalize_build_vector_v2f16: ; VI: ; %bb.0: @@ -242,7 +269,6 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f16(ptr addrspace(1) %ou ; CI-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_cvt_f32_f16_e64 v0, |v0| -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; CI-NEXT: s_endpgm @@ -299,7 +325,6 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f16(ptr addrspace(1 ; CI-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_cvt_f32_f16_e64 v0, -|v0| -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; CI-NEXT: s_endpgm @@ -357,7 +382,6 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_f16(ptr addrspace(1) %ou ; CI-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_cvt_f32_f16_e64 v0, -v0 -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; CI-NEXT: s_endpgm @@ -414,7 +438,6 @@ define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_var_f16(ptr add ; CI-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_cvt_f32_f16_e64 v0, -v0 -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; CI-NEXT: s_endpgm @@ -471,7 +494,6 @@ define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_fabs_var_f16(pt ; CI-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_cvt_f32_f16_e64 v0, -|v0| -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; CI-NEXT: s_endpgm @@ -1246,9 +1268,7 @@ define amdgpu_kernel void @v_test_canonicalize_var_v2f16(ptr addrspace(1) %out) ; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v0 ; CI-NEXT: v_cvt_f32_f16_e32 v1, v1 ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; CI-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; CI-NEXT: v_or_b32_e32 v0, v0, v1 @@ -1323,9 +1343,7 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_v2f16(ptr addrspace(1) % ; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v0 ; CI-NEXT: v_cvt_f32_f16_e64 v1, |v1| ; CI-NEXT: v_cvt_f32_f16_e64 v0, |v0| -; CI-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; CI-NEXT: v_or_b32_e32 v0, v0, v1 @@ -1404,9 +1422,7 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_v2f16(ptr addrspace ; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v0 ; CI-NEXT: v_cvt_f32_f16_e32 v1, v1 ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; CI-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; CI-NEXT: v_or_b32_e32 v0, v0, v1 @@ -1485,9 +1501,7 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_v2f16(ptr addrspace(1) % ; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v0 ; CI-NEXT: v_cvt_f32_f16_e32 v1, v1 ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; CI-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; CI-NEXT: v_or_b32_e32 v0, v0, v1 @@ -1551,9 +1565,7 @@ define amdgpu_kernel void @s_test_canonicalize_var_v2f16(ptr addrspace(1) %out, ; CI-NEXT: v_cvt_f32_f16_e32 v1, s2 ; CI-NEXT: s_mov_b32 s3, 0xf000 ; CI-NEXT: s_mov_b32 s2, -1 -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 -; CI-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ; CI-NEXT: v_or_b32_e32 v0, v1, v0 @@ -2424,7 +2436,6 @@ define <2 x half> @v_test_canonicalize_reg_undef_v2f16(half %val) #1 { ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: v_mov_b32_e32 v1, 0x7fc00000 ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_test_canonicalize_reg_undef_v2f16: @@ -2456,8 +2467,7 @@ define <2 x half> @v_test_canonicalize_undef_reg_v2f16(half %val) #1 { ; CI: ; %bb.0: ; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 -; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; CI-NEXT: v_mul_f32_e32 v1, 1.0, v0 +; CI-NEXT: v_cvt_f32_f16_e32 v1, v0 ; CI-NEXT: v_mov_b32_e32 v0, 0x7fc00000 ; CI-NEXT: s_setpc_b64 s[30:31] ; @@ -2738,7 +2748,6 @@ define <4 x half> @v_test_canonicalize_reg_undef_undef_undef_v4f16(half %val) #1 ; CI-NEXT: v_mov_b32_e32 v2, 0x7fc00000 ; CI-NEXT: v_mov_b32_e32 v3, 0x7fc00000 ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; CI-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_test_canonicalize_reg_undef_undef_undef_v4f16: @@ -2782,8 +2791,6 @@ define <4 x half> @v_test_canonicalize_reg_reg_undef_undef_v4f16(half %val0, hal ; CI-NEXT: v_mov_b32_e32 v3, 0x7fc00000 ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; CI-NEXT: v_cvt_f32_f16_e32 v1, v1 -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; CI-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; CI-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_test_canonicalize_reg_reg_undef_undef_v4f16: @@ -2826,13 +2833,10 @@ define <4 x half> @v_test_canonicalize_reg_undef_reg_reg_v4f16(half %val0, half ; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 -; CI-NEXT: v_cvt_f16_f32_e32 v2, v2 +; CI-NEXT: v_cvt_f16_f32_e32 v3, v2 ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; CI-NEXT: v_cvt_f32_f16_e32 v1, v1 -; CI-NEXT: v_cvt_f32_f16_e32 v3, v2 -; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; CI-NEXT: v_mul_f32_e32 v2, 1.0, v1 -; CI-NEXT: v_mul_f32_e32 v3, 1.0, v3 +; CI-NEXT: v_cvt_f32_f16_e32 v2, v1 +; CI-NEXT: v_cvt_f32_f16_e32 v3, v3 ; CI-NEXT: v_mov_b32_e32 v1, 0x7fc00000 ; CI-NEXT: s_setpc_b64 s[30:31] ; @@ -2878,18 +2882,18 @@ define <6 x half> @v_test_canonicalize_var_v6f16(<6 x half> %val) #1 { ; CI-LABEL: v_test_canonicalize_var_v6f16: ; CI: ; %bb.0: ; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CI-NEXT: v_cvt_f16_f32_e32 v5, v5 +; CI-NEXT: v_cvt_f16_f32_e32 v4, v4 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; CI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; CI-NEXT: v_cvt_f16_f32_e32 v3, v3 -; CI-NEXT: v_cvt_f16_f32_e32 v4, v4 -; CI-NEXT: v_cvt_f16_f32_e32 v5, v5 +; CI-NEXT: v_cvt_f32_f16_e32 v5, v5 +; CI-NEXT: v_cvt_f32_f16_e32 v4, v4 ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; CI-NEXT: v_cvt_f32_f16_e32 v1, v1 ; CI-NEXT: v_cvt_f32_f16_e32 v2, v2 ; CI-NEXT: v_cvt_f32_f16_e32 v3, v3 -; CI-NEXT: v_cvt_f32_f16_e32 v4, v4 -; CI-NEXT: v_cvt_f32_f16_e32 v5, v5 ; CI-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_test_canonicalize_var_v6f16: @@ -2933,22 +2937,22 @@ define <8 x half> @v_test_canonicalize_var_v8f16(<8 x half> %val) #1 { ; CI-LABEL: v_test_canonicalize_var_v8f16: ; CI: ; %bb.0: ; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CI-NEXT: v_cvt_f16_f32_e32 v7, v7 +; CI-NEXT: v_cvt_f16_f32_e32 v6, v6 +; CI-NEXT: v_cvt_f16_f32_e32 v5, v5 +; CI-NEXT: v_cvt_f16_f32_e32 v4, v4 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; CI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; CI-NEXT: v_cvt_f16_f32_e32 v3, v3 -; CI-NEXT: v_cvt_f16_f32_e32 v4, v4 -; CI-NEXT: v_cvt_f16_f32_e32 v5, v5 -; CI-NEXT: v_cvt_f16_f32_e32 v6, v6 -; CI-NEXT: v_cvt_f16_f32_e32 v7, v7 +; CI-NEXT: v_cvt_f32_f16_e32 v7, v7 +; CI-NEXT: v_cvt_f32_f16_e32 v6, v6 +; CI-NEXT: v_cvt_f32_f16_e32 v5, v5 +; CI-NEXT: v_cvt_f32_f16_e32 v4, v4 ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; CI-NEXT: v_cvt_f32_f16_e32 v1, v1 ; CI-NEXT: v_cvt_f32_f16_e32 v2, v2 ; CI-NEXT: v_cvt_f32_f16_e32 v3, v3 -; CI-NEXT: v_cvt_f32_f16_e32 v4, v4 -; CI-NEXT: v_cvt_f32_f16_e32 v5, v5 -; CI-NEXT: v_cvt_f32_f16_e32 v6, v6 -; CI-NEXT: v_cvt_f32_f16_e32 v7, v7 ; CI-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_test_canonicalize_var_v8f16: @@ -3001,30 +3005,30 @@ define <12 x half> @v_test_canonicalize_var_v12f16(<12 x half> %val) #1 { ; CI-LABEL: v_test_canonicalize_var_v12f16: ; CI: ; %bb.0: ; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CI-NEXT: v_cvt_f16_f32_e32 v11, v11 +; CI-NEXT: v_cvt_f16_f32_e32 v10, v10 +; CI-NEXT: v_cvt_f16_f32_e32 v9, v9 +; CI-NEXT: v_cvt_f16_f32_e32 v8, v8 +; CI-NEXT: v_cvt_f16_f32_e32 v7, v7 +; CI-NEXT: v_cvt_f16_f32_e32 v6, v6 +; CI-NEXT: v_cvt_f16_f32_e32 v5, v5 +; CI-NEXT: v_cvt_f16_f32_e32 v4, v4 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; CI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; CI-NEXT: v_cvt_f16_f32_e32 v3, v3 -; CI-NEXT: v_cvt_f16_f32_e32 v4, v4 -; CI-NEXT: v_cvt_f16_f32_e32 v5, v5 -; CI-NEXT: v_cvt_f16_f32_e32 v6, v6 -; CI-NEXT: v_cvt_f16_f32_e32 v7, v7 -; CI-NEXT: v_cvt_f16_f32_e32 v8, v8 -; CI-NEXT: v_cvt_f16_f32_e32 v9, v9 -; CI-NEXT: v_cvt_f16_f32_e32 v10, v10 -; CI-NEXT: v_cvt_f16_f32_e32 v11, v11 +; CI-NEXT: v_cvt_f32_f16_e32 v11, v11 +; CI-NEXT: v_cvt_f32_f16_e32 v10, v10 +; CI-NEXT: v_cvt_f32_f16_e32 v9, v9 +; CI-NEXT: v_cvt_f32_f16_e32 v8, v8 +; CI-NEXT: v_cvt_f32_f16_e32 v7, v7 +; CI-NEXT: v_cvt_f32_f16_e32 v6, v6 +; CI-NEXT: v_cvt_f32_f16_e32 v5, v5 +; CI-NEXT: v_cvt_f32_f16_e32 v4, v4 ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; CI-NEXT: v_cvt_f32_f16_e32 v1, v1 ; CI-NEXT: v_cvt_f32_f16_e32 v2, v2 ; CI-NEXT: v_cvt_f32_f16_e32 v3, v3 -; CI-NEXT: v_cvt_f32_f16_e32 v4, v4 -; CI-NEXT: v_cvt_f32_f16_e32 v5, v5 -; CI-NEXT: v_cvt_f32_f16_e32 v6, v6 -; CI-NEXT: v_cvt_f32_f16_e32 v7, v7 -; CI-NEXT: v_cvt_f32_f16_e32 v8, v8 -; CI-NEXT: v_cvt_f32_f16_e32 v9, v9 -; CI-NEXT: v_cvt_f32_f16_e32 v10, v10 -; CI-NEXT: v_cvt_f32_f16_e32 v11, v11 ; CI-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_test_canonicalize_var_v12f16: @@ -3087,38 +3091,38 @@ define <16 x half> @v_test_canonicalize_var_v16f16(<16 x half> %val) #1 { ; CI-LABEL: v_test_canonicalize_var_v16f16: ; CI: ; %bb.0: ; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CI-NEXT: v_cvt_f16_f32_e32 v15, v15 +; CI-NEXT: v_cvt_f16_f32_e32 v14, v14 +; CI-NEXT: v_cvt_f16_f32_e32 v13, v13 +; CI-NEXT: v_cvt_f16_f32_e32 v12, v12 +; CI-NEXT: v_cvt_f16_f32_e32 v11, v11 +; CI-NEXT: v_cvt_f16_f32_e32 v10, v10 +; CI-NEXT: v_cvt_f16_f32_e32 v9, v9 +; CI-NEXT: v_cvt_f16_f32_e32 v8, v8 +; CI-NEXT: v_cvt_f16_f32_e32 v7, v7 +; CI-NEXT: v_cvt_f16_f32_e32 v6, v6 +; CI-NEXT: v_cvt_f16_f32_e32 v5, v5 +; CI-NEXT: v_cvt_f16_f32_e32 v4, v4 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; CI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; CI-NEXT: v_cvt_f16_f32_e32 v3, v3 -; CI-NEXT: v_cvt_f16_f32_e32 v4, v4 -; CI-NEXT: v_cvt_f16_f32_e32 v5, v5 -; CI-NEXT: v_cvt_f16_f32_e32 v6, v6 -; CI-NEXT: v_cvt_f16_f32_e32 v7, v7 -; CI-NEXT: v_cvt_f16_f32_e32 v8, v8 -; CI-NEXT: v_cvt_f16_f32_e32 v9, v9 -; CI-NEXT: v_cvt_f16_f32_e32 v10, v10 -; CI-NEXT: v_cvt_f16_f32_e32 v11, v11 -; CI-NEXT: v_cvt_f16_f32_e32 v12, v12 -; CI-NEXT: v_cvt_f16_f32_e32 v13, v13 -; CI-NEXT: v_cvt_f16_f32_e32 v14, v14 -; CI-NEXT: v_cvt_f16_f32_e32 v15, v15 +; CI-NEXT: v_cvt_f32_f16_e32 v15, v15 +; CI-NEXT: v_cvt_f32_f16_e32 v14, v14 +; CI-NEXT: v_cvt_f32_f16_e32 v13, v13 +; CI-NEXT: v_cvt_f32_f16_e32 v12, v12 +; CI-NEXT: v_cvt_f32_f16_e32 v11, v11 +; CI-NEXT: v_cvt_f32_f16_e32 v10, v10 +; CI-NEXT: v_cvt_f32_f16_e32 v9, v9 +; CI-NEXT: v_cvt_f32_f16_e32 v8, v8 +; CI-NEXT: v_cvt_f32_f16_e32 v7, v7 +; CI-NEXT: v_cvt_f32_f16_e32 v6, v6 +; CI-NEXT: v_cvt_f32_f16_e32 v5, v5 +; CI-NEXT: v_cvt_f32_f16_e32 v4, v4 ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; CI-NEXT: v_cvt_f32_f16_e32 v1, v1 ; CI-NEXT: v_cvt_f32_f16_e32 v2, v2 ; CI-NEXT: v_cvt_f32_f16_e32 v3, v3 -; CI-NEXT: v_cvt_f32_f16_e32 v4, v4 -; CI-NEXT: v_cvt_f32_f16_e32 v5, v5 -; CI-NEXT: v_cvt_f32_f16_e32 v6, v6 -; CI-NEXT: v_cvt_f32_f16_e32 v7, v7 -; CI-NEXT: v_cvt_f32_f16_e32 v8, v8 -; CI-NEXT: v_cvt_f32_f16_e32 v9, v9 -; CI-NEXT: v_cvt_f32_f16_e32 v10, v10 -; CI-NEXT: v_cvt_f32_f16_e32 v11, v11 -; CI-NEXT: v_cvt_f32_f16_e32 v12, v12 -; CI-NEXT: v_cvt_f32_f16_e32 v13, v13 -; CI-NEXT: v_cvt_f32_f16_e32 v14, v14 -; CI-NEXT: v_cvt_f32_f16_e32 v15, v15 ; CI-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_test_canonicalize_var_v16f16: @@ -3216,68 +3220,68 @@ define <32 x half> @v_test_canonicalize_var_v32f16(<32 x half> %val) #1 { ; CI: ; %bb.0: ; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; CI-NEXT: buffer_load_dword v31, off, s[0:3], s32 +; CI-NEXT: v_cvt_f16_f32_e32 v30, v30 +; CI-NEXT: v_cvt_f16_f32_e32 v29, v29 +; CI-NEXT: v_cvt_f16_f32_e32 v28, v28 +; CI-NEXT: v_cvt_f16_f32_e32 v27, v27 +; CI-NEXT: v_cvt_f16_f32_e32 v26, v26 +; CI-NEXT: v_cvt_f16_f32_e32 v25, v25 +; CI-NEXT: v_cvt_f16_f32_e32 v24, v24 +; CI-NEXT: v_cvt_f16_f32_e32 v23, v23 +; CI-NEXT: v_cvt_f16_f32_e32 v22, v22 +; CI-NEXT: v_cvt_f16_f32_e32 v21, v21 +; CI-NEXT: v_cvt_f16_f32_e32 v20, v20 +; CI-NEXT: v_cvt_f16_f32_e32 v19, v19 +; CI-NEXT: v_cvt_f16_f32_e32 v18, v18 +; CI-NEXT: v_cvt_f16_f32_e32 v17, v17 +; CI-NEXT: v_cvt_f16_f32_e32 v16, v16 +; CI-NEXT: v_cvt_f16_f32_e32 v15, v15 +; CI-NEXT: v_cvt_f16_f32_e32 v14, v14 +; CI-NEXT: v_cvt_f16_f32_e32 v13, v13 +; CI-NEXT: v_cvt_f16_f32_e32 v12, v12 +; CI-NEXT: v_cvt_f16_f32_e32 v11, v11 +; CI-NEXT: v_cvt_f16_f32_e32 v10, v10 +; CI-NEXT: v_cvt_f16_f32_e32 v9, v9 +; CI-NEXT: v_cvt_f16_f32_e32 v8, v8 +; CI-NEXT: v_cvt_f16_f32_e32 v7, v7 +; CI-NEXT: v_cvt_f16_f32_e32 v6, v6 +; CI-NEXT: v_cvt_f16_f32_e32 v5, v5 +; CI-NEXT: v_cvt_f16_f32_e32 v4, v4 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; CI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; CI-NEXT: v_cvt_f16_f32_e32 v3, v3 -; CI-NEXT: v_cvt_f16_f32_e32 v4, v4 -; CI-NEXT: v_cvt_f16_f32_e32 v5, v5 -; CI-NEXT: v_cvt_f16_f32_e32 v6, v6 -; CI-NEXT: v_cvt_f16_f32_e32 v7, v7 -; CI-NEXT: v_cvt_f16_f32_e32 v8, v8 -; CI-NEXT: v_cvt_f16_f32_e32 v9, v9 -; CI-NEXT: v_cvt_f16_f32_e32 v10, v10 -; CI-NEXT: v_cvt_f16_f32_e32 v11, v11 -; CI-NEXT: v_cvt_f16_f32_e32 v12, v12 -; CI-NEXT: v_cvt_f16_f32_e32 v13, v13 -; CI-NEXT: v_cvt_f16_f32_e32 v14, v14 -; CI-NEXT: v_cvt_f16_f32_e32 v15, v15 -; CI-NEXT: v_cvt_f16_f32_e32 v16, v16 -; CI-NEXT: v_cvt_f16_f32_e32 v17, v17 -; CI-NEXT: v_cvt_f16_f32_e32 v18, v18 -; CI-NEXT: v_cvt_f16_f32_e32 v19, v19 -; CI-NEXT: v_cvt_f16_f32_e32 v20, v20 -; CI-NEXT: v_cvt_f16_f32_e32 v21, v21 -; CI-NEXT: v_cvt_f16_f32_e32 v22, v22 -; CI-NEXT: v_cvt_f16_f32_e32 v23, v23 -; CI-NEXT: v_cvt_f16_f32_e32 v24, v24 -; CI-NEXT: v_cvt_f16_f32_e32 v25, v25 -; CI-NEXT: v_cvt_f16_f32_e32 v26, v26 -; CI-NEXT: v_cvt_f16_f32_e32 v27, v27 -; CI-NEXT: v_cvt_f16_f32_e32 v28, v28 -; CI-NEXT: v_cvt_f16_f32_e32 v29, v29 -; CI-NEXT: v_cvt_f16_f32_e32 v30, v30 +; CI-NEXT: v_cvt_f32_f16_e32 v30, v30 +; CI-NEXT: v_cvt_f32_f16_e32 v29, v29 +; CI-NEXT: v_cvt_f32_f16_e32 v28, v28 +; CI-NEXT: v_cvt_f32_f16_e32 v27, v27 +; CI-NEXT: v_cvt_f32_f16_e32 v26, v26 +; CI-NEXT: v_cvt_f32_f16_e32 v25, v25 +; CI-NEXT: v_cvt_f32_f16_e32 v24, v24 +; CI-NEXT: v_cvt_f32_f16_e32 v23, v23 +; CI-NEXT: v_cvt_f32_f16_e32 v22, v22 +; CI-NEXT: v_cvt_f32_f16_e32 v21, v21 +; CI-NEXT: v_cvt_f32_f16_e32 v20, v20 +; CI-NEXT: v_cvt_f32_f16_e32 v19, v19 +; CI-NEXT: v_cvt_f32_f16_e32 v18, v18 +; CI-NEXT: v_cvt_f32_f16_e32 v17, v17 +; CI-NEXT: v_cvt_f32_f16_e32 v16, v16 +; CI-NEXT: v_cvt_f32_f16_e32 v15, v15 +; CI-NEXT: v_cvt_f32_f16_e32 v14, v14 +; CI-NEXT: v_cvt_f32_f16_e32 v13, v13 +; CI-NEXT: v_cvt_f32_f16_e32 v12, v12 +; CI-NEXT: v_cvt_f32_f16_e32 v11, v11 +; CI-NEXT: v_cvt_f32_f16_e32 v10, v10 +; CI-NEXT: v_cvt_f32_f16_e32 v9, v9 +; CI-NEXT: v_cvt_f32_f16_e32 v8, v8 +; CI-NEXT: v_cvt_f32_f16_e32 v7, v7 +; CI-NEXT: v_cvt_f32_f16_e32 v6, v6 +; CI-NEXT: v_cvt_f32_f16_e32 v5, v5 +; CI-NEXT: v_cvt_f32_f16_e32 v4, v4 ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; CI-NEXT: v_cvt_f32_f16_e32 v1, v1 ; CI-NEXT: v_cvt_f32_f16_e32 v2, v2 ; CI-NEXT: v_cvt_f32_f16_e32 v3, v3 -; CI-NEXT: v_cvt_f32_f16_e32 v4, v4 -; CI-NEXT: v_cvt_f32_f16_e32 v5, v5 -; CI-NEXT: v_cvt_f32_f16_e32 v6, v6 -; CI-NEXT: v_cvt_f32_f16_e32 v7, v7 -; CI-NEXT: v_cvt_f32_f16_e32 v8, v8 -; CI-NEXT: v_cvt_f32_f16_e32 v9, v9 -; CI-NEXT: v_cvt_f32_f16_e32 v10, v10 -; CI-NEXT: v_cvt_f32_f16_e32 v11, v11 -; CI-NEXT: v_cvt_f32_f16_e32 v12, v12 -; CI-NEXT: v_cvt_f32_f16_e32 v13, v13 -; CI-NEXT: v_cvt_f32_f16_e32 v14, v14 -; CI-NEXT: v_cvt_f32_f16_e32 v15, v15 -; CI-NEXT: v_cvt_f32_f16_e32 v16, v16 -; CI-NEXT: v_cvt_f32_f16_e32 v17, v17 -; CI-NEXT: v_cvt_f32_f16_e32 v18, v18 -; CI-NEXT: v_cvt_f32_f16_e32 v19, v19 -; CI-NEXT: v_cvt_f32_f16_e32 v20, v20 -; CI-NEXT: v_cvt_f32_f16_e32 v21, v21 -; CI-NEXT: v_cvt_f32_f16_e32 v22, v22 -; CI-NEXT: v_cvt_f32_f16_e32 v23, v23 -; CI-NEXT: v_cvt_f32_f16_e32 v24, v24 -; CI-NEXT: v_cvt_f32_f16_e32 v25, v25 -; CI-NEXT: v_cvt_f32_f16_e32 v26, v26 -; CI-NEXT: v_cvt_f32_f16_e32 v27, v27 -; CI-NEXT: v_cvt_f32_f16_e32 v28, v28 -; CI-NEXT: v_cvt_f32_f16_e32 v29, v29 -; CI-NEXT: v_cvt_f32_f16_e32 v30, v30 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_cvt_f16_f32_e32 v31, v31 ; CI-NEXT: v_cvt_f32_f16_e32 v31, v31 @@ -3456,228 +3460,354 @@ define <64 x half> @v_test_canonicalize_var_v64f16(<64 x half> %val) #1 { ; CI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; CI-NEXT: v_cvt_f16_f32_e32 v3, v3 +; CI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:104 +; CI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108 +; CI-NEXT: v_cvt_f32_f16_e32 v2, v2 +; CI-NEXT: v_cvt_f32_f16_e32 v1, v1 +; CI-NEXT: v_cvt_f32_f16_e32 v3, v3 +; CI-NEXT: v_cvt_f16_f32_e32 v2, v2 +; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; CI-NEXT: v_cvt_f16_f32_e32 v3, v3 ; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; CI-NEXT: v_or_b32_e32 v1, v1, v2 ; CI-NEXT: v_cvt_f16_f32_e32 v2, v4 ; CI-NEXT: v_cvt_f16_f32_e32 v4, v5 ; CI-NEXT: v_cvt_f16_f32_e32 v5, v7 ; CI-NEXT: v_cvt_f16_f32_e32 v7, v9 +; CI-NEXT: v_cvt_f32_f16_e32 v2, v2 +; CI-NEXT: v_cvt_f32_f16_e32 v4, v4 +; CI-NEXT: v_cvt_f32_f16_e32 v5, v5 +; CI-NEXT: v_cvt_f32_f16_e32 v7, v7 +; CI-NEXT: v_cvt_f16_f32_e32 v2, v2 +; CI-NEXT: v_cvt_f16_f32_e32 v4, v4 +; CI-NEXT: v_cvt_f16_f32_e32 v5, v5 +; CI-NEXT: v_cvt_f16_f32_e32 v7, v7 ; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; CI-NEXT: v_or_b32_e32 v2, v3, v2 ; CI-NEXT: v_cvt_f16_f32_e32 v3, v6 ; CI-NEXT: v_cvt_f16_f32_e32 v6, v10 ; CI-NEXT: v_cvt_f16_f32_e32 v9, v13 -; CI-NEXT: v_cvt_f16_f32_e32 v10, v18 +; CI-NEXT: v_cvt_f16_f32_e32 v10, v16 +; CI-NEXT: v_cvt_f32_f16_e32 v3, v3 +; CI-NEXT: v_cvt_f32_f16_e32 v6, v6 +; CI-NEXT: v_cvt_f32_f16_e32 v9, v9 +; CI-NEXT: v_cvt_f16_f32_e32 v13, v17 +; CI-NEXT: v_cvt_f16_f32_e32 v3, v3 +; CI-NEXT: v_cvt_f16_f32_e32 v6, v6 +; CI-NEXT: v_cvt_f16_f32_e32 v9, v9 +; CI-NEXT: v_cvt_f32_f16_e32 v13, v13 ; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; CI-NEXT: v_or_b32_e32 v3, v4, v3 ; CI-NEXT: v_cvt_f16_f32_e32 v4, v8 ; CI-NEXT: v_cvt_f16_f32_e32 v8, v14 -; CI-NEXT: v_cvt_f16_f32_e32 v13, v21 -; CI-NEXT: v_cvt_f16_f32_e32 v14, v26 +; CI-NEXT: buffer_load_dword v14, off, s[0:3], s32 +; CI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:4 +; CI-NEXT: v_cvt_f16_f32_e32 v17, v23 +; CI-NEXT: v_cvt_f32_f16_e32 v4, v4 +; CI-NEXT: v_cvt_f32_f16_e32 v8, v8 +; CI-NEXT: v_cvt_f32_f16_e32 v17, v17 +; CI-NEXT: v_cvt_f16_f32_e32 v4, v4 +; CI-NEXT: v_cvt_f16_f32_e32 v8, v8 ; CI-NEXT: v_lshlrev_b32_e32 v4, 16, v4 ; CI-NEXT: v_or_b32_e32 v4, v5, v4 ; CI-NEXT: v_lshlrev_b32_e32 v5, 16, v6 ; CI-NEXT: v_cvt_f16_f32_e32 v6, v12 ; CI-NEXT: v_or_b32_e32 v5, v7, v5 ; CI-NEXT: v_cvt_f16_f32_e32 v7, v11 -; CI-NEXT: v_cvt_f16_f32_e32 v11, v17 +; CI-NEXT: v_cvt_f16_f32_e32 v11, v15 +; CI-NEXT: v_cvt_f32_f16_e32 v6, v6 +; CI-NEXT: v_cvt_f16_f32_e32 v15, v21 +; CI-NEXT: v_cvt_f32_f16_e32 v7, v7 +; CI-NEXT: v_cvt_f16_f32_e32 v6, v6 +; CI-NEXT: v_cvt_f16_f32_e32 v7, v7 ; CI-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; CI-NEXT: v_cvt_f16_f32_e32 v12, v22 ; CI-NEXT: v_or_b32_e32 v6, v7, v6 ; CI-NEXT: v_lshlrev_b32_e32 v7, 16, v8 -; CI-NEXT: v_cvt_f16_f32_e32 v8, v16 +; CI-NEXT: v_cvt_f16_f32_e32 v8, v19 ; CI-NEXT: v_or_b32_e32 v7, v9, v7 -; CI-NEXT: v_cvt_f16_f32_e32 v9, v15 -; CI-NEXT: v_cvt_f16_f32_e32 v15, v25 +; CI-NEXT: v_cvt_f16_f32_e32 v9, v20 +; CI-NEXT: v_cvt_f32_f16_e32 v12, v8 +; CI-NEXT: v_cvt_f32_f16_e32 v8, v10 +; CI-NEXT: v_cvt_f32_f16_e32 v10, v11 +; CI-NEXT: v_cvt_f16_f32_e32 v11, v18 +; CI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:124 +; CI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:112 +; CI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:116 +; CI-NEXT: v_cvt_f16_f32_e32 v8, v8 +; CI-NEXT: v_cvt_f16_f32_e32 v10, v10 +; CI-NEXT: v_cvt_f32_f16_e32 v11, v11 +; CI-NEXT: v_cvt_f32_f16_e32 v9, v9 ; CI-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; CI-NEXT: v_cvt_f16_f32_e32 v25, v29 -; CI-NEXT: v_or_b32_e32 v8, v9, v8 +; CI-NEXT: v_or_b32_e32 v8, v10, v8 +; CI-NEXT: v_cvt_f16_f32_e32 v10, v11 +; CI-NEXT: v_cvt_f16_f32_e32 v11, v13 +; CI-NEXT: v_cvt_f16_f32_e32 v13, v9 +; CI-NEXT: v_cvt_f16_f32_e32 v12, v12 ; CI-NEXT: v_lshlrev_b32_e32 v9, 16, v10 -; CI-NEXT: v_cvt_f16_f32_e32 v10, v20 ; CI-NEXT: v_or_b32_e32 v9, v11, v9 -; CI-NEXT: v_cvt_f16_f32_e32 v11, v19 -; CI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:4 -; CI-NEXT: buffer_load_dword v17, off, s[0:3], s32 -; CI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:12 -; CI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:8 -; CI-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; CI-NEXT: v_or_b32_e32 v10, v11, v10 -; CI-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; CI-NEXT: v_cvt_f16_f32_e32 v12, v24 +; CI-NEXT: v_lshlrev_b32_e32 v10, 16, v13 +; CI-NEXT: v_cvt_f16_f32_e32 v11, v25 +; CI-NEXT: v_cvt_f16_f32_e32 v13, v22 +; CI-NEXT: v_or_b32_e32 v10, v12, v10 +; CI-NEXT: v_cvt_f16_f32_e32 v12, v26 +; CI-NEXT: v_cvt_f32_f16_e32 v16, v11 +; CI-NEXT: v_cvt_f32_f16_e32 v11, v13 +; CI-NEXT: v_cvt_f32_f16_e32 v13, v15 +; CI-NEXT: v_cvt_f16_f32_e32 v15, v24 +; CI-NEXT: v_cvt_f32_f16_e32 v12, v12 +; CI-NEXT: v_cvt_f16_f32_e32 v11, v11 +; CI-NEXT: v_cvt_f16_f32_e32 v13, v13 +; CI-NEXT: v_cvt_f32_f16_e32 v15, v15 +; CI-NEXT: v_cvt_f16_f32_e32 v22, v30 +; CI-NEXT: v_lshlrev_b32_e32 v11, 16, v11 ; CI-NEXT: v_or_b32_e32 v11, v13, v11 -; CI-NEXT: v_cvt_f16_f32_e32 v13, v23 -; CI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:20 -; CI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:16 -; CI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:28 -; CI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:24 -; CI-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; CI-NEXT: v_cvt_f16_f32_e32 v24, v30 -; CI-NEXT: v_or_b32_e32 v12, v13, v12 -; CI-NEXT: v_lshlrev_b32_e32 v13, 16, v14 -; CI-NEXT: v_or_b32_e32 v13, v15, v13 -; CI-NEXT: v_cvt_f16_f32_e32 v14, v28 +; CI-NEXT: v_cvt_f16_f32_e32 v13, v15 +; CI-NEXT: v_cvt_f16_f32_e32 v15, v17 +; CI-NEXT: v_cvt_f16_f32_e32 v17, v12 +; CI-NEXT: v_cvt_f16_f32_e32 v25, v29 +; CI-NEXT: v_lshlrev_b32_e32 v12, 16, v13 +; CI-NEXT: v_or_b32_e32 v12, v15, v12 +; CI-NEXT: s_waitcnt vmcnt(6) +; CI-NEXT: v_cvt_f16_f32_e32 v15, v31 +; CI-NEXT: v_lshlrev_b32_e32 v13, 16, v17 +; CI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:128 +; CI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:132 +; CI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:120 +; CI-NEXT: v_cvt_f32_f16_e32 v22, v22 +; CI-NEXT: v_cvt_f32_f16_e32 v23, v15 ; CI-NEXT: v_cvt_f16_f32_e32 v15, v27 -; CI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:36 -; CI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:32 -; CI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:44 -; CI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:40 +; CI-NEXT: v_cvt_f32_f16_e32 v25, v25 +; CI-NEXT: s_waitcnt vmcnt(7) +; CI-NEXT: v_cvt_f16_f32_e32 v14, v14 +; CI-NEXT: s_waitcnt vmcnt(6) +; CI-NEXT: v_cvt_f16_f32_e32 v21, v33 +; CI-NEXT: v_cvt_f32_f16_e32 v15, v15 +; CI-NEXT: v_cvt_f16_f32_e32 v22, v22 +; CI-NEXT: v_cvt_f32_f16_e32 v24, v14 +; CI-NEXT: v_cvt_f16_f32_e32 v14, v28 +; CI-NEXT: v_cvt_f16_f32_e32 v15, v15 +; CI-NEXT: v_cvt_f32_f16_e32 v21, v21 +; CI-NEXT: v_cvt_f16_f32_e32 v25, v25 +; CI-NEXT: v_cvt_f32_f16_e32 v14, v14 +; CI-NEXT: v_cvt_f16_f32_e32 v16, v16 +; CI-NEXT: v_cvt_f16_f32_e32 v24, v24 +; CI-NEXT: v_cvt_f16_f32_e32 v28, v23 +; CI-NEXT: v_cvt_f16_f32_e32 v14, v14 +; CI-NEXT: v_or_b32_e32 v13, v16, v13 +; CI-NEXT: v_cvt_f16_f32_e32 v16, v32 +; CI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:12 ; CI-NEXT: v_lshlrev_b32_e32 v14, 16, v14 ; CI-NEXT: v_or_b32_e32 v14, v15, v14 -; CI-NEXT: v_lshlrev_b32_e32 v15, 16, v24 +; CI-NEXT: v_lshlrev_b32_e32 v15, 16, v22 ; CI-NEXT: v_or_b32_e32 v15, v25, v15 -; CI-NEXT: s_waitcnt vmcnt(11) -; CI-NEXT: v_cvt_f16_f32_e32 v16, v16 -; CI-NEXT: s_waitcnt vmcnt(10) -; CI-NEXT: v_cvt_f16_f32_e32 v17, v17 +; CI-NEXT: v_cvt_f16_f32_e32 v25, v21 +; CI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 +; CI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:100 +; CI-NEXT: v_cvt_f32_f16_e32 v16, v16 +; CI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:64 +; CI-NEXT: v_lshlrev_b32_e32 v25, 16, v25 +; CI-NEXT: v_cvt_f16_f32_e32 v27, v16 +; CI-NEXT: v_or_b32_e32 v16, v24, v25 +; CI-NEXT: v_lshlrev_b32_e32 v24, 16, v27 +; CI-NEXT: v_or_b32_e32 v25, v28, v24 ; CI-NEXT: s_waitcnt vmcnt(9) ; CI-NEXT: v_cvt_f16_f32_e32 v18, v18 ; CI-NEXT: s_waitcnt vmcnt(8) ; CI-NEXT: v_cvt_f16_f32_e32 v19, v19 -; CI-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; CI-NEXT: v_or_b32_e32 v16, v17, v16 -; CI-NEXT: v_lshlrev_b32_e32 v17, 16, v18 -; CI-NEXT: v_or_b32_e32 v17, v19, v17 ; CI-NEXT: s_waitcnt vmcnt(7) -; CI-NEXT: v_cvt_f16_f32_e32 v18, v20 +; CI-NEXT: v_cvt_f16_f32_e32 v20, v20 +; CI-NEXT: v_cvt_f32_f16_e32 v18, v18 +; CI-NEXT: v_cvt_f32_f16_e32 v19, v19 +; CI-NEXT: v_cvt_f32_f16_e32 v20, v20 +; CI-NEXT: v_cvt_f16_f32_e32 v18, v18 +; CI-NEXT: v_cvt_f16_f32_e32 v19, v19 +; CI-NEXT: v_cvt_f16_f32_e32 v20, v20 +; CI-NEXT: v_lshlrev_b32_e32 v18, 16, v18 +; CI-NEXT: v_lshlrev_b32_e32 v20, 16, v20 +; CI-NEXT: v_or_b32_e32 v20, v19, v20 +; CI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:20 +; CI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:8 +; CI-NEXT: s_waitcnt vmcnt(8) +; CI-NEXT: v_cvt_f16_f32_e32 v17, v17 +; CI-NEXT: s_waitcnt vmcnt(7) +; CI-NEXT: v_cvt_f16_f32_e32 v26, v26 ; CI-NEXT: s_waitcnt vmcnt(6) -; CI-NEXT: v_cvt_f16_f32_e32 v19, v21 -; CI-NEXT: s_waitcnt vmcnt(5) +; CI-NEXT: v_cvt_f16_f32_e32 v27, v34 +; CI-NEXT: v_cvt_f32_f16_e32 v17, v17 +; CI-NEXT: v_cvt_f32_f16_e32 v26, v26 +; CI-NEXT: v_cvt_f32_f16_e32 v27, v27 +; CI-NEXT: v_cvt_f16_f32_e32 v17, v17 +; CI-NEXT: v_cvt_f16_f32_e32 v26, v26 +; CI-NEXT: v_cvt_f16_f32_e32 v27, v27 +; CI-NEXT: v_lshlrev_b32_e32 v26, 16, v26 +; CI-NEXT: v_or_b32_e32 v17, v17, v26 +; CI-NEXT: v_add_i32_e32 v26, vcc, 0x7c, v0 +; CI-NEXT: v_or_b32_e32 v18, v27, v18 +; CI-NEXT: buffer_store_dword v17, v26, s[0:3], 0 offen +; CI-NEXT: v_add_i32_e32 v17, vcc, 0x78, v0 +; CI-NEXT: buffer_store_dword v18, v17, s[0:3], 0 offen +; CI-NEXT: v_add_i32_e32 v17, vcc, 0x74, v0 +; CI-NEXT: buffer_store_dword v20, v17, s[0:3], 0 offen +; CI-NEXT: v_add_i32_e32 v17, vcc, 0x70, v0 +; CI-NEXT: buffer_store_dword v25, v17, s[0:3], 0 offen +; CI-NEXT: s_waitcnt vmcnt(8) +; CI-NEXT: v_cvt_f16_f32_e32 v21, v21 +; CI-NEXT: s_waitcnt vmcnt(7) ; CI-NEXT: v_cvt_f16_f32_e32 v20, v22 -; CI-NEXT: s_waitcnt vmcnt(4) -; CI-NEXT: v_cvt_f16_f32_e32 v21, v23 -; CI-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; CI-NEXT: v_or_b32_e32 v18, v19, v18 -; CI-NEXT: v_lshlrev_b32_e32 v19, 16, v20 -; CI-NEXT: v_or_b32_e32 v19, v21, v19 -; CI-NEXT: s_waitcnt vmcnt(3) -; CI-NEXT: v_cvt_f16_f32_e32 v20, v26 -; CI-NEXT: s_waitcnt vmcnt(2) -; CI-NEXT: v_cvt_f16_f32_e32 v21, v27 -; CI-NEXT: s_waitcnt vmcnt(1) -; CI-NEXT: v_cvt_f16_f32_e32 v26, v28 -; CI-NEXT: s_waitcnt vmcnt(0) -; CI-NEXT: v_cvt_f16_f32_e32 v27, v29 +; CI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:88 +; CI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:92 +; CI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:80 +; CI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:84 +; CI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:72 +; CI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:76 +; CI-NEXT: v_cvt_f16_f32_e32 v22, v23 +; CI-NEXT: v_cvt_f32_f16_e32 v21, v21 +; CI-NEXT: v_cvt_f32_f16_e32 v20, v20 +; CI-NEXT: s_waitcnt vmcnt(12) +; CI-NEXT: v_cvt_f16_f32_e32 v29, v29 +; CI-NEXT: v_cvt_f32_f16_e32 v22, v22 +; CI-NEXT: v_cvt_f16_f32_e32 v21, v21 +; CI-NEXT: v_cvt_f16_f32_e32 v20, v20 +; CI-NEXT: v_cvt_f32_f16_e32 v29, v29 +; CI-NEXT: v_cvt_f16_f32_e32 v22, v22 ; CI-NEXT: v_lshlrev_b32_e32 v20, 16, v20 ; CI-NEXT: v_or_b32_e32 v20, v21, v20 -; CI-NEXT: v_lshlrev_b32_e32 v21, 16, v26 -; CI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:52 -; CI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:48 -; CI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:60 -; CI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:56 -; CI-NEXT: v_or_b32_e32 v21, v27, v21 -; CI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:132 -; CI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 -; CI-NEXT: s_waitcnt vmcnt(5) -; CI-NEXT: v_cvt_f16_f32_e32 v24, v24 -; CI-NEXT: s_waitcnt vmcnt(4) -; CI-NEXT: v_cvt_f16_f32_e32 v25, v25 -; CI-NEXT: s_waitcnt vmcnt(3) +; CI-NEXT: v_add_i32_e32 v21, vcc, 0x6c, v0 +; CI-NEXT: buffer_store_dword v20, v21, s[0:3], 0 offen +; CI-NEXT: v_lshlrev_b32_e32 v20, 16, v22 +; CI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:24 +; CI-NEXT: v_cvt_f16_f32_e32 v29, v29 +; CI-NEXT: s_waitcnt vmcnt(13) +; CI-NEXT: v_cvt_f16_f32_e32 v19, v19 +; CI-NEXT: s_waitcnt vmcnt(12) +; CI-NEXT: v_cvt_f16_f32_e32 v23, v24 +; CI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:28 +; CI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:16 +; CI-NEXT: v_cvt_f32_f16_e32 v19, v19 +; CI-NEXT: v_cvt_f32_f16_e32 v23, v23 +; CI-NEXT: v_cvt_f16_f32_e32 v19, v19 ; CI-NEXT: v_cvt_f16_f32_e32 v23, v23 -; CI-NEXT: s_waitcnt vmcnt(2) -; CI-NEXT: v_cvt_f16_f32_e32 v22, v22 -; CI-NEXT: s_waitcnt vmcnt(1) +; CI-NEXT: v_lshlrev_b32_e32 v19, 16, v19 +; CI-NEXT: v_or_b32_e32 v20, v23, v20 +; CI-NEXT: s_waitcnt vmcnt(9) +; CI-NEXT: v_cvt_f16_f32_e32 v27, v27 +; CI-NEXT: s_waitcnt vmcnt(8) +; CI-NEXT: v_cvt_f16_f32_e32 v23, v28 +; CI-NEXT: s_waitcnt vmcnt(7) +; CI-NEXT: v_cvt_f16_f32_e32 v17, v17 +; CI-NEXT: s_waitcnt vmcnt(6) +; CI-NEXT: v_cvt_f16_f32_e32 v18, v18 +; CI-NEXT: v_cvt_f32_f16_e32 v27, v27 +; CI-NEXT: v_cvt_f32_f16_e32 v23, v23 +; CI-NEXT: s_waitcnt vmcnt(4) ; CI-NEXT: v_cvt_f16_f32_e32 v26, v26 -; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: v_cvt_f16_f32_e32 v25, v25 ; CI-NEXT: v_cvt_f16_f32_e32 v27, v27 -; CI-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; CI-NEXT: v_or_b32_e32 v24, v25, v24 -; CI-NEXT: v_lshlrev_b32_e32 v26, 16, v26 -; CI-NEXT: v_or_b32_e32 v26, v27, v26 -; CI-NEXT: v_add_i32_e32 v27, vcc, 0x7c, v0 -; CI-NEXT: buffer_store_dword v26, v27, s[0:3], 0 offen -; CI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:124 -; CI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:120 +; CI-NEXT: v_cvt_f16_f32_e32 v23, v23 +; CI-NEXT: v_cvt_f32_f16_e32 v18, v18 +; CI-NEXT: v_cvt_f32_f16_e32 v26, v26 +; CI-NEXT: v_cvt_f32_f16_e32 v17, v17 ; CI-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; CI-NEXT: v_or_b32_e32 v22, v22, v23 -; CI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:88 -; CI-NEXT: s_waitcnt vmcnt(2) +; CI-NEXT: v_or_b32_e32 v23, v27, v23 +; CI-NEXT: v_add_i32_e32 v27, vcc, 0x68, v0 +; CI-NEXT: buffer_store_dword v23, v27, s[0:3], 0 offen +; CI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:32 +; CI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:36 +; CI-NEXT: v_cvt_f32_f16_e32 v25, v25 +; CI-NEXT: v_cvt_f16_f32_e32 v18, v18 ; CI-NEXT: v_cvt_f16_f32_e32 v26, v26 -; CI-NEXT: s_waitcnt vmcnt(1) -; CI-NEXT: v_cvt_f16_f32_e32 v27, v27 +; CI-NEXT: v_cvt_f16_f32_e32 v17, v17 +; CI-NEXT: v_cvt_f16_f32_e32 v25, v25 +; CI-NEXT: v_lshlrev_b32_e32 v18, 16, v18 ; CI-NEXT: v_lshlrev_b32_e32 v26, 16, v26 -; CI-NEXT: v_or_b32_e32 v26, v27, v26 -; CI-NEXT: v_add_i32_e32 v27, vcc, 0x78, v0 -; CI-NEXT: buffer_store_dword v26, v27, s[0:3], 0 offen -; CI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:116 -; CI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:112 -; CI-NEXT: s_waitcnt vmcnt(3) +; CI-NEXT: v_or_b32_e32 v17, v17, v18 +; CI-NEXT: v_add_i32_e32 v18, vcc, 0x64, v0 +; CI-NEXT: v_or_b32_e32 v25, v25, v26 +; CI-NEXT: buffer_store_dword v17, v18, s[0:3], 0 offen +; CI-NEXT: v_add_i32_e32 v17, vcc, 0x60, v0 +; CI-NEXT: buffer_store_dword v25, v17, s[0:3], 0 offen +; CI-NEXT: v_add_i32_e32 v17, vcc, 0x5c, v0 +; CI-NEXT: s_waitcnt vmcnt(5) +; CI-NEXT: v_cvt_f16_f32_e32 v24, v24 +; CI-NEXT: v_cvt_f16_f32_e32 v21, v21 +; CI-NEXT: v_cvt_f32_f16_e32 v24, v24 +; CI-NEXT: v_cvt_f32_f16_e32 v21, v21 +; CI-NEXT: v_cvt_f16_f32_e32 v24, v24 +; CI-NEXT: v_cvt_f16_f32_e32 v21, v21 +; CI-NEXT: v_or_b32_e32 v19, v24, v19 +; CI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:44 +; CI-NEXT: v_cvt_f16_f32_e32 v22, v22 +; CI-NEXT: v_lshlrev_b32_e32 v21, 16, v21 +; CI-NEXT: v_cvt_f32_f16_e32 v22, v22 +; CI-NEXT: v_cvt_f16_f32_e32 v22, v22 +; CI-NEXT: v_or_b32_e32 v21, v22, v21 +; CI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:40 +; CI-NEXT: s_waitcnt vmcnt(5) +; CI-NEXT: v_cvt_f16_f32_e32 v23, v23 +; CI-NEXT: s_waitcnt vmcnt(4) +; CI-NEXT: v_cvt_f16_f32_e32 v27, v27 +; CI-NEXT: v_cvt_f32_f16_e32 v23, v23 +; CI-NEXT: v_cvt_f32_f16_e32 v27, v27 ; CI-NEXT: v_cvt_f16_f32_e32 v23, v23 +; CI-NEXT: v_cvt_f16_f32_e32 v27, v27 +; CI-NEXT: v_lshlrev_b32_e32 v27, 16, v27 ; CI-NEXT: s_waitcnt vmcnt(1) -; CI-NEXT: v_cvt_f16_f32_e32 v26, v26 +; CI-NEXT: v_cvt_f16_f32_e32 v24, v24 +; CI-NEXT: v_cvt_f32_f16_e32 v24, v24 +; CI-NEXT: v_cvt_f16_f32_e32 v24, v24 ; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: v_cvt_f16_f32_e32 v22, v22 +; CI-NEXT: v_cvt_f32_f16_e32 v22, v22 +; CI-NEXT: v_cvt_f16_f32_e32 v28, v22 +; CI-NEXT: v_or_b32_e32 v22, v23, v27 +; CI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:52 +; CI-NEXT: v_lshlrev_b32_e32 v23, 16, v24 +; CI-NEXT: v_or_b32_e32 v23, v28, v23 +; CI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:56 +; CI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:48 +; CI-NEXT: s_waitcnt vmcnt(2) ; CI-NEXT: v_cvt_f16_f32_e32 v27, v27 -; CI-NEXT: v_lshlrev_b32_e32 v26, 16, v26 -; CI-NEXT: v_or_b32_e32 v26, v27, v26 -; CI-NEXT: v_add_i32_e32 v27, vcc, 0x74, v0 -; CI-NEXT: buffer_store_dword v26, v27, s[0:3], 0 offen -; CI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:108 -; CI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:104 ; CI-NEXT: s_waitcnt vmcnt(1) -; CI-NEXT: v_cvt_f16_f32_e32 v25, v26 +; CI-NEXT: v_cvt_f16_f32_e32 v28, v28 ; CI-NEXT: s_waitcnt vmcnt(0) -; CI-NEXT: v_cvt_f16_f32_e32 v26, v27 -; CI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:92 -; CI-NEXT: v_lshlrev_b32_e32 v25, 16, v25 -; CI-NEXT: v_or_b32_e32 v25, v26, v25 -; CI-NEXT: v_add_i32_e32 v26, vcc, 0x70, v0 -; CI-NEXT: buffer_store_dword v25, v26, s[0:3], 0 offen -; CI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:100 -; CI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:96 -; CI-NEXT: s_waitcnt vmcnt(3) +; CI-NEXT: v_cvt_f16_f32_e32 v24, v24 +; CI-NEXT: v_cvt_f32_f16_e32 v27, v27 +; CI-NEXT: v_cvt_f32_f16_e32 v28, v28 +; CI-NEXT: v_cvt_f32_f16_e32 v24, v24 ; CI-NEXT: v_cvt_f16_f32_e32 v27, v27 +; CI-NEXT: v_cvt_f16_f32_e32 v28, v28 +; CI-NEXT: v_cvt_f16_f32_e32 v24, v24 ; CI-NEXT: v_lshlrev_b32_e32 v27, 16, v27 -; CI-NEXT: v_or_b32_e32 v23, v23, v27 -; CI-NEXT: s_waitcnt vmcnt(1) -; CI-NEXT: v_cvt_f16_f32_e32 v25, v25 +; CI-NEXT: v_or_b32_e32 v24, v24, v27 +; CI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:60 ; CI-NEXT: s_waitcnt vmcnt(0) -; CI-NEXT: v_cvt_f16_f32_e32 v26, v26 -; CI-NEXT: v_add_i32_e32 v27, vcc, 0x68, v0 -; CI-NEXT: v_lshlrev_b32_e32 v25, 16, v25 -; CI-NEXT: v_or_b32_e32 v25, v26, v25 -; CI-NEXT: v_add_i32_e32 v26, vcc, 0x6c, v0 -; CI-NEXT: buffer_store_dword v25, v26, s[0:3], 0 offen -; CI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:68 -; CI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:64 -; CI-NEXT: buffer_store_dword v23, v27, s[0:3], 0 offen -; CI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:76 -; CI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:72 -; CI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:84 -; CI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:80 -; CI-NEXT: s_waitcnt vmcnt(3) -; CI-NEXT: v_cvt_f16_f32_e32 v23, v23 -; CI-NEXT: v_cvt_f16_f32_e32 v25, v25 -; CI-NEXT: v_cvt_f16_f32_e32 v26, v26 -; CI-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; CI-NEXT: v_lshlrev_b32_e32 v25, 16, v25 -; CI-NEXT: v_or_b32_e32 v25, v26, v25 -; CI-NEXT: s_waitcnt vmcnt(2) -; CI-NEXT: v_cvt_f16_f32_e32 v26, v27 +; CI-NEXT: v_cvt_f16_f32_e32 v27, v27 +; CI-NEXT: v_cvt_f32_f16_e32 v27, v27 +; CI-NEXT: v_cvt_f16_f32_e32 v27, v27 +; CI-NEXT: v_lshlrev_b32_e32 v27, 16, v27 +; CI-NEXT: v_or_b32_e32 v27, v28, v27 +; CI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:68 ; CI-NEXT: s_waitcnt vmcnt(0) -; CI-NEXT: v_cvt_f16_f32_e32 v27, v29 -; CI-NEXT: v_or_b32_e32 v23, v26, v23 -; CI-NEXT: v_cvt_f16_f32_e32 v26, v28 -; CI-NEXT: v_lshlrev_b32_e32 v26, 16, v26 -; CI-NEXT: v_or_b32_e32 v26, v27, v26 -; CI-NEXT: v_add_i32_e32 v27, vcc, 0x64, v0 -; CI-NEXT: buffer_store_dword v26, v27, s[0:3], 0 offen -; CI-NEXT: v_add_i32_e32 v26, vcc, 0x60, v0 -; CI-NEXT: buffer_store_dword v23, v26, s[0:3], 0 offen -; CI-NEXT: v_add_i32_e32 v23, vcc, 0x5c, v0 -; CI-NEXT: buffer_store_dword v25, v23, s[0:3], 0 offen -; CI-NEXT: v_add_i32_e32 v23, vcc, 0x58, v0 -; CI-NEXT: buffer_store_dword v22, v23, s[0:3], 0 offen -; CI-NEXT: v_add_i32_e32 v22, vcc, 0x54, v0 -; CI-NEXT: buffer_store_dword v24, v22, s[0:3], 0 offen -; CI-NEXT: v_add_i32_e32 v22, vcc, 0x50, v0 -; CI-NEXT: buffer_store_dword v21, v22, s[0:3], 0 offen -; CI-NEXT: v_add_i32_e32 v21, vcc, 0x4c, v0 -; CI-NEXT: buffer_store_dword v20, v21, s[0:3], 0 offen -; CI-NEXT: v_add_i32_e32 v20, vcc, 0x48, v0 -; CI-NEXT: buffer_store_dword v19, v20, s[0:3], 0 offen -; CI-NEXT: v_add_i32_e32 v19, vcc, 0x44, v0 -; CI-NEXT: buffer_store_dword v18, v19, s[0:3], 0 offen -; CI-NEXT: v_add_i32_e32 v18, vcc, 64, v0 -; CI-NEXT: buffer_store_dword v17, v18, s[0:3], 0 offen +; CI-NEXT: v_cvt_f16_f32_e32 v28, v28 +; CI-NEXT: v_cvt_f32_f16_e32 v28, v28 +; CI-NEXT: v_cvt_f16_f32_e32 v28, v28 +; CI-NEXT: v_lshlrev_b32_e32 v28, 16, v28 +; CI-NEXT: v_or_b32_e32 v28, v29, v28 +; CI-NEXT: buffer_store_dword v28, v17, s[0:3], 0 offen +; CI-NEXT: v_add_i32_e32 v17, vcc, 0x58, v0 +; CI-NEXT: buffer_store_dword v27, v17, s[0:3], 0 offen +; CI-NEXT: v_add_i32_e32 v17, vcc, 0x54, v0 +; CI-NEXT: buffer_store_dword v24, v17, s[0:3], 0 offen +; CI-NEXT: v_add_i32_e32 v17, vcc, 0x50, v0 +; CI-NEXT: buffer_store_dword v23, v17, s[0:3], 0 offen +; CI-NEXT: v_add_i32_e32 v17, vcc, 0x4c, v0 +; CI-NEXT: buffer_store_dword v22, v17, s[0:3], 0 offen +; CI-NEXT: v_add_i32_e32 v17, vcc, 0x48, v0 +; CI-NEXT: buffer_store_dword v21, v17, s[0:3], 0 offen +; CI-NEXT: v_add_i32_e32 v17, vcc, 0x44, v0 +; CI-NEXT: buffer_store_dword v19, v17, s[0:3], 0 offen +; CI-NEXT: v_add_i32_e32 v17, vcc, 64, v0 +; CI-NEXT: buffer_store_dword v20, v17, s[0:3], 0 offen ; CI-NEXT: v_add_i32_e32 v17, vcc, 60, v0 ; CI-NEXT: buffer_store_dword v16, v17, s[0:3], 0 offen ; CI-NEXT: v_add_i32_e32 v16, vcc, 56, v0 diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll index c1093a1e89c8..d53c0411ad88 100644 --- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll +++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll @@ -2389,7 +2389,6 @@ define amdgpu_kernel void @test_canonicalize_value_f16_flush(ptr addrspace(1) %a ; GFX6-NEXT: v_mov_b32_e32 v1, s3 ; GFX6-NEXT: s_waitcnt vmcnt(0) ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0 -; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v0 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v2 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc @@ -2471,15 +2470,13 @@ define amdgpu_kernel void @test_canonicalize_value_v2f16_flush(ptr addrspace(1) ; GFX6-NEXT: flat_load_dword v0, v[0:1] ; GFX6-NEXT: v_mov_b32_e32 v3, s3 ; GFX6-NEXT: s_waitcnt vmcnt(0) -; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v0 -; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0 -; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1 -; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0 -; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX6-NEXT: v_or_b32_e32 v4, v1, v0 +; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX6-NEXT: v_or_b32_e32 v4, v0, v1 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v2 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc ; GFX6-NEXT: flat_store_dword v[0:1], v4 @@ -2724,7 +2721,6 @@ define amdgpu_kernel void @test_canonicalize_value_f16_denorm(ptr addrspace(1) % ; GFX6-NEXT: v_mov_b32_e32 v1, s3 ; GFX6-NEXT: s_waitcnt vmcnt(0) ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0 -; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v0 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v2 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc @@ -2807,15 +2803,13 @@ define amdgpu_kernel void @test_canonicalize_value_v2f16_denorm(ptr addrspace(1) ; GFX6-NEXT: flat_load_dword v0, v[0:1] ; GFX6-NEXT: v_mov_b32_e32 v3, s3 ; GFX6-NEXT: s_waitcnt vmcnt(0) -; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v0 -; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0 -; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1 -; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0 -; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX6-NEXT: v_or_b32_e32 v4, v1, v0 +; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX6-NEXT: v_or_b32_e32 v4, v0, v1 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v2 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc ; GFX6-NEXT: flat_store_dword v[0:1], v4 diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll index 78fb89c71e2e..b32630a97b3a 100644 --- a/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll @@ -951,8 +951,6 @@ define half @v_fneg_minnum_f16_ieee(half %a, half %b) #0 { ; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0 ; SI-NEXT: v_cvt_f32_f16_e32 v1, v1 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_max_f32_e32 v0, v0, v1 ; SI-NEXT: s_setpc_b64 s[30:31] ; @@ -1056,7 +1054,6 @@ define half @v_fneg_posk_minnum_f16_ieee(half %a) #0 { ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_max_f32_e32 v0, -4.0, v0 ; SI-NEXT: s_setpc_b64 s[30:31] ; @@ -1110,7 +1107,6 @@ define half @v_fneg_negk_minnum_f16_ieee(half %a) #0 { ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_max_f32_e32 v0, 4.0, v0 ; SI-NEXT: s_setpc_b64 s[30:31] ; @@ -1193,7 +1189,6 @@ define half @v_fneg_neg0_minnum_f16_ieee(half %a) #0 { ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_max_f32_e32 v0, 0, v0 ; SI-NEXT: s_setpc_b64 s[30:31] ; @@ -1222,7 +1217,6 @@ define half @v_fneg_inv2pi_minnum_f16(half %a) #0 { ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_max_f32_e32 v0, 0xbe230000, v0 ; SI-NEXT: s_setpc_b64 s[30:31] ; @@ -1253,7 +1247,6 @@ define half @v_fneg_neg_inv2pi_minnum_f16(half %a) #0 { ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_max_f32_e32 v0, 0xbe230000, v0 ; SI-NEXT: s_setpc_b64 s[30:31] ; @@ -1311,7 +1304,6 @@ define half @v_fneg_0_minnum_foldable_use_f16_ieee(half %a, half %b) #0 { ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; SI-NEXT: v_cvt_f32_f16_e32 v1, v1 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_min_f32_e32 v0, 0, v0 ; SI-NEXT: v_mul_f32_e64 v0, -v0, v1 ; SI-NEXT: s_setpc_b64 s[30:31] @@ -1346,7 +1338,6 @@ define half @v_fneg_inv2pi_minnum_foldable_use_f16(half %a, half %b) #0 { ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; SI-NEXT: v_cvt_f32_f16_e32 v1, v1 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_max_f32_e32 v0, 0xbe230000, v0 ; SI-NEXT: v_mul_f32_e32 v0, v0, v1 ; SI-NEXT: s_setpc_b64 s[30:31] @@ -1413,8 +1404,6 @@ define { half, half } @v_fneg_minnum_multi_use_minnum_f16_ieee(half %a, half %b) ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: v_cvt_f32_f16_e64 v1, -v1 ; SI-NEXT: v_cvt_f32_f16_e64 v0, -v0 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_max_f32_e32 v0, v0, v1 ; SI-NEXT: v_mul_f32_e32 v1, -4.0, v0 ; SI-NEXT: s_setpc_b64 s[30:31] @@ -1494,8 +1483,6 @@ define half @v_fneg_maxnum_f16_ieee(half %a, half %b) #0 { ; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0 ; SI-NEXT: v_cvt_f32_f16_e32 v1, v1 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_min_f32_e32 v0, v0, v1 ; SI-NEXT: s_setpc_b64 s[30:31] ; @@ -1599,7 +1586,6 @@ define half @v_fneg_posk_maxnum_f16_ieee(half %a) #0 { ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_min_f32_e32 v0, -4.0, v0 ; SI-NEXT: s_setpc_b64 s[30:31] ; @@ -1653,7 +1639,6 @@ define half @v_fneg_negk_maxnum_f16_ieee(half %a) #0 { ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_min_f32_e32 v0, 4.0, v0 ; SI-NEXT: s_setpc_b64 s[30:31] ; @@ -1736,7 +1721,6 @@ define half @v_fneg_neg0_maxnum_f16_ieee(half %a) #0 { ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_min_f32_e32 v0, 0, v0 ; SI-NEXT: s_setpc_b64 s[30:31] ; @@ -1792,7 +1776,6 @@ define half @v_fneg_0_maxnum_foldable_use_f16_ieee(half %a, half %b) #0 { ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; SI-NEXT: v_cvt_f32_f16_e32 v1, v1 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_max_f32_e32 v0, 0, v0 ; SI-NEXT: v_mul_f32_e64 v0, -v0, v1 ; SI-NEXT: s_setpc_b64 s[30:31] @@ -1859,8 +1842,6 @@ define { half, half } @v_fneg_maxnum_multi_use_maxnum_f16_ieee(half %a, half %b) ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: v_cvt_f32_f16_e64 v1, -v1 ; SI-NEXT: v_cvt_f32_f16_e64 v0, -v0 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_min_f32_e32 v0, v0, v1 ; SI-NEXT: v_mul_f32_e32 v1, -4.0, v0 ; SI-NEXT: s_setpc_b64 s[30:31] @@ -3980,7 +3961,8 @@ define half @v_fneg_canonicalize_f16(half %a) #0 { ; SI-LABEL: v_fneg_canonicalize_f16: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_xor_b32_e32 v0, 0x80000000, v0 +; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0 +; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; SI-NEXT: s_setpc_b64 s[30:31] ; ; VI-LABEL: v_fneg_canonicalize_f16: diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll index 17f67615c29f..b5440b9c38c9 100644 --- a/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll +++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll @@ -1021,7 +1021,6 @@ define half @v_fneg_inv2pi_minnum_f16(half %a) #0 { ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_max_f32_e32 v0, 0xbe230000, v0 ; SI-NEXT: s_setpc_b64 s[30:31] ; @@ -1043,7 +1042,6 @@ define half @v_fneg_neg_inv2pi_minnum_f16(half %a) #0 { ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_max_f32_e32 v0, 0x3e230000, v0 ; SI-NEXT: s_setpc_b64 s[30:31] ; diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll index ab7ab4de1861..d056a97dc544 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll @@ -32,8 +32,6 @@ define amdgpu_kernel void @maxnum_f16( ; SI-NEXT: s_mov_b32 s1, s5 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; SI-NEXT: v_cvt_f32_f16_e32 v1, v1 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; SI-NEXT: v_max_f32_e32 v0, v0, v1 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: buffer_store_short v0, off, s[0:3], 0 @@ -170,7 +168,6 @@ define amdgpu_kernel void @maxnum_f16_imm_a( ; SI-NEXT: s_mov_b32 s5, s1 ; SI-NEXT: s_waitcnt vmcnt(0) ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_max_f32_e32 v0, 0x40400000, v0 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: buffer_store_short v0, off, s[4:7], 0 @@ -279,7 +276,6 @@ define amdgpu_kernel void @maxnum_f16_imm_b( ; SI-NEXT: s_mov_b32 s5, s1 ; SI-NEXT: s_waitcnt vmcnt(0) ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_max_f32_e32 v0, 4.0, v0 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: buffer_store_short v0, off, s[4:7], 0 @@ -384,21 +380,17 @@ define amdgpu_kernel void @maxnum_v2f16( ; SI-NEXT: s_mov_b32 s6, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_lshr_b32 s1, s2, 16 -; SI-NEXT: v_cvt_f32_f16_e32 v1, s0 -; SI-NEXT: s_lshr_b32 s0, s0, 16 -; SI-NEXT: v_cvt_f32_f16_e32 v2, s0 -; SI-NEXT: v_cvt_f32_f16_e32 v3, s1 -; SI-NEXT: v_cvt_f32_f16_e32 v0, s2 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; SI-NEXT: v_max_f32_e32 v2, v3, v2 -; SI-NEXT: v_cvt_f16_f32_e32 v2, v2 +; SI-NEXT: s_lshr_b32 s3, s0, 16 +; SI-NEXT: v_cvt_f32_f16_e32 v0, s1 +; SI-NEXT: v_cvt_f32_f16_e32 v1, s3 +; SI-NEXT: v_cvt_f32_f16_e32 v2, s2 +; SI-NEXT: v_cvt_f32_f16_e32 v3, s0 ; SI-NEXT: v_max_f32_e32 v0, v0, v1 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 -; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2 -; SI-NEXT: v_or_b32_e32 v0, v0, v1 +; SI-NEXT: v_max_f32_e32 v1, v2, v3 +; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SI-NEXT: v_or_b32_e32 v0, v1, v0 ; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 ; SI-NEXT: s_endpgm ; @@ -497,20 +489,18 @@ define amdgpu_kernel void @maxnum_v2f16_imm_a( ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_load_dword s2, s[2:3], 0x0 -; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_cvt_f32_f16_e32 v0, s2 -; SI-NEXT: s_lshr_b32 s2, s2, 16 +; SI-NEXT: s_lshr_b32 s3, s2, 16 +; SI-NEXT: v_cvt_f32_f16_e32 v0, s3 ; SI-NEXT: v_cvt_f32_f16_e32 v1, s2 +; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_mov_b32 s2, -1 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; SI-NEXT: v_max_f32_e32 v0, 0x40400000, v0 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_max_f32_e32 v1, 4.0, v1 -; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; SI-NEXT: v_max_f32_e32 v0, 4.0, v0 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 -; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; SI-NEXT: v_or_b32_e32 v0, v0, v1 +; SI-NEXT: v_max_f32_e32 v1, 0x40400000, v1 +; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SI-NEXT: v_or_b32_e32 v0, v1, v0 ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -589,20 +579,18 @@ define amdgpu_kernel void @maxnum_v2f16_imm_b( ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_load_dword s2, s[2:3], 0x0 -; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_cvt_f32_f16_e32 v0, s2 -; SI-NEXT: s_lshr_b32 s2, s2, 16 +; SI-NEXT: s_lshr_b32 s3, s2, 16 +; SI-NEXT: v_cvt_f32_f16_e32 v0, s3 ; SI-NEXT: v_cvt_f32_f16_e32 v1, s2 +; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_mov_b32 s2, -1 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; SI-NEXT: v_max_f32_e32 v0, 4.0, v0 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_max_f32_e32 v1, 0x40400000, v1 -; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; SI-NEXT: v_max_f32_e32 v0, 0x40400000, v0 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 -; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; SI-NEXT: v_or_b32_e32 v0, v0, v1 +; SI-NEXT: v_max_f32_e32 v1, 4.0, v1 +; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SI-NEXT: v_or_b32_e32 v0, v1, v0 ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -688,27 +676,21 @@ define amdgpu_kernel void @maxnum_v3f16( ; SI-NEXT: s_mov_b32 s6, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: v_cvt_f32_f16_e32 v0, s3 -; SI-NEXT: v_cvt_f32_f16_e32 v1, s2 -; SI-NEXT: s_lshr_b32 s2, s2, 16 -; SI-NEXT: s_lshr_b32 s3, s0, 16 -; SI-NEXT: v_cvt_f32_f16_e32 v2, s3 +; SI-NEXT: s_lshr_b32 s3, s2, 16 +; SI-NEXT: s_lshr_b32 s8, s0, 16 +; SI-NEXT: v_cvt_f32_f16_e32 v1, s3 +; SI-NEXT: v_cvt_f32_f16_e32 v2, s8 ; SI-NEXT: v_cvt_f32_f16_e32 v3, s2 -; SI-NEXT: v_cvt_f32_f16_e32 v5, s0 -; SI-NEXT: v_cvt_f32_f16_e32 v4, s1 -; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; SI-NEXT: v_max_f32_e32 v2, v3, v2 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v5 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_max_f32_e32 v1, v1, v3 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v4 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; SI-NEXT: v_cvt_f16_f32_e32 v2, v2 -; SI-NEXT: v_max_f32_e32 v0, v0, v3 +; SI-NEXT: v_cvt_f32_f16_e32 v4, s0 +; SI-NEXT: v_cvt_f32_f16_e32 v5, s1 +; SI-NEXT: v_max_f32_e32 v1, v1, v2 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; SI-NEXT: v_max_f32_e32 v2, v3, v4 +; SI-NEXT: v_max_f32_e32 v0, v0, v5 +; SI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 -; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; SI-NEXT: v_or_b32_e32 v1, v1, v2 +; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; SI-NEXT: v_or_b32_e32 v1, v2, v1 ; SI-NEXT: buffer_store_short v0, off, s[4:7], 0 offset:4 ; SI-NEXT: buffer_store_dword v1, off, s[4:7], 0 ; SI-NEXT: s_endpgm @@ -837,25 +819,17 @@ define amdgpu_kernel void @maxnum_v4f16( ; SI-NEXT: v_cvt_f32_f16_e32 v2, s6 ; SI-NEXT: s_lshr_b32 s6, s7, 16 ; SI-NEXT: v_cvt_f32_f16_e32 v3, s6 +; SI-NEXT: v_cvt_f32_f16_e32 v4, s4 ; SI-NEXT: s_lshr_b32 s6, s5, 16 +; SI-NEXT: s_lshr_b32 s4, s4, 16 ; SI-NEXT: v_cvt_f32_f16_e32 v5, s6 +; SI-NEXT: v_cvt_f32_f16_e32 v7, s4 ; SI-NEXT: v_cvt_f32_f16_e32 v1, s7 -; SI-NEXT: v_cvt_f32_f16_e32 v4, s4 -; SI-NEXT: s_lshr_b32 s4, s4, 16 -; SI-NEXT: v_cvt_f32_f16_e32 v7, s5 -; SI-NEXT: v_cvt_f32_f16_e32 v6, s4 -; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3 +; SI-NEXT: v_cvt_f32_f16_e32 v6, s5 ; SI-NEXT: v_max_f32_e32 v3, v3, v5 -; SI-NEXT: v_mul_f32_e32 v5, 1.0, v7 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_max_f32_e32 v1, v1, v5 -; SI-NEXT: v_mul_f32_e32 v5, 1.0, v6 -; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; SI-NEXT: v_max_f32_e32 v2, v2, v5 -; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 +; SI-NEXT: v_max_f32_e32 v2, v2, v7 ; SI-NEXT: v_cvt_f16_f32_e32 v3, v3 +; SI-NEXT: v_max_f32_e32 v1, v1, v6 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; SI-NEXT: v_max_f32_e32 v0, v0, v4 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 @@ -986,20 +960,16 @@ define amdgpu_kernel void @fmax_v4f16_imm_a( ; SI-NEXT: v_cvt_f32_f16_e32 v1, s5 ; SI-NEXT: s_lshr_b32 s5, s5, 16 ; SI-NEXT: v_cvt_f32_f16_e32 v0, s4 -; SI-NEXT: v_cvt_f32_f16_e32 v2, s5 ; SI-NEXT: s_lshr_b32 s4, s4, 16 +; SI-NEXT: v_cvt_f32_f16_e32 v2, s5 ; SI-NEXT: v_cvt_f32_f16_e32 v3, s4 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; SI-NEXT: v_max_f32_e32 v2, 4.0, v2 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3 ; SI-NEXT: v_max_f32_e32 v1, 0x40400000, v1 +; SI-NEXT: v_max_f32_e32 v0, 0x41000000, v0 +; SI-NEXT: v_max_f32_e32 v2, 4.0, v2 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; SI-NEXT: v_max_f32_e32 v3, 2.0, v3 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; SI-NEXT: v_cvt_f16_f32_e32 v3, v3 -; SI-NEXT: v_max_f32_e32 v0, 0x41000000, v0 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; SI-NEXT: v_or_b32_e32 v1, v1, v2 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll index b7370ce0fde1..f934a2de9247 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll @@ -32,8 +32,6 @@ define amdgpu_kernel void @minnum_f16_ieee( ; SI-NEXT: s_mov_b32 s1, s5 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; SI-NEXT: v_cvt_f32_f16_e32 v1, v1 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 ; SI-NEXT: v_min_f32_e32 v0, v0, v1 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: buffer_store_short v0, off, s[0:3], 0 @@ -197,7 +195,6 @@ define amdgpu_kernel void @minnum_f16_imm_a( ; SI-NEXT: s_mov_b32 s5, s1 ; SI-NEXT: s_waitcnt vmcnt(0) ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_min_f32_e32 v0, 0x40400000, v0 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: buffer_store_short v0, off, s[4:7], 0 @@ -305,7 +302,6 @@ define amdgpu_kernel void @minnum_f16_imm_b( ; SI-NEXT: s_mov_b32 s5, s1 ; SI-NEXT: s_waitcnt vmcnt(0) ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_min_f32_e32 v0, 4.0, v0 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: buffer_store_short v0, off, s[4:7], 0 @@ -409,21 +405,17 @@ define amdgpu_kernel void @minnum_v2f16_ieee( ; SI-NEXT: s_mov_b32 s6, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_lshr_b32 s1, s2, 16 -; SI-NEXT: v_cvt_f32_f16_e32 v1, s0 -; SI-NEXT: s_lshr_b32 s0, s0, 16 -; SI-NEXT: v_cvt_f32_f16_e32 v2, s0 -; SI-NEXT: v_cvt_f32_f16_e32 v3, s1 -; SI-NEXT: v_cvt_f32_f16_e32 v0, s2 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; SI-NEXT: v_min_f32_e32 v2, v3, v2 -; SI-NEXT: v_cvt_f16_f32_e32 v2, v2 +; SI-NEXT: s_lshr_b32 s3, s0, 16 +; SI-NEXT: v_cvt_f32_f16_e32 v0, s1 +; SI-NEXT: v_cvt_f32_f16_e32 v1, s3 +; SI-NEXT: v_cvt_f32_f16_e32 v2, s2 +; SI-NEXT: v_cvt_f32_f16_e32 v3, s0 ; SI-NEXT: v_min_f32_e32 v0, v0, v1 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 -; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2 -; SI-NEXT: v_or_b32_e32 v0, v0, v1 +; SI-NEXT: v_min_f32_e32 v1, v2, v3 +; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SI-NEXT: v_or_b32_e32 v0, v1, v0 ; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 ; SI-NEXT: s_endpgm ; @@ -556,20 +548,18 @@ define amdgpu_kernel void @minnum_v2f16_imm_a( ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_load_dword s2, s[2:3], 0x0 -; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_cvt_f32_f16_e32 v0, s2 -; SI-NEXT: s_lshr_b32 s2, s2, 16 +; SI-NEXT: s_lshr_b32 s3, s2, 16 +; SI-NEXT: v_cvt_f32_f16_e32 v0, s3 ; SI-NEXT: v_cvt_f32_f16_e32 v1, s2 +; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_mov_b32 s2, -1 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; SI-NEXT: v_min_f32_e32 v0, 0x40400000, v0 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_min_f32_e32 v1, 4.0, v1 -; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; SI-NEXT: v_min_f32_e32 v0, 4.0, v0 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 -; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; SI-NEXT: v_or_b32_e32 v0, v0, v1 +; SI-NEXT: v_min_f32_e32 v1, 0x40400000, v1 +; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SI-NEXT: v_or_b32_e32 v0, v1, v0 ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -647,20 +637,18 @@ define amdgpu_kernel void @minnum_v2f16_imm_b( ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_load_dword s2, s[2:3], 0x0 -; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_cvt_f32_f16_e32 v0, s2 -; SI-NEXT: s_lshr_b32 s2, s2, 16 +; SI-NEXT: s_lshr_b32 s3, s2, 16 +; SI-NEXT: v_cvt_f32_f16_e32 v0, s3 ; SI-NEXT: v_cvt_f32_f16_e32 v1, s2 +; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_mov_b32 s2, -1 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; SI-NEXT: v_min_f32_e32 v0, 4.0, v0 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_min_f32_e32 v1, 0x40400000, v1 -; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; SI-NEXT: v_min_f32_e32 v0, 0x40400000, v0 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 -; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; SI-NEXT: v_or_b32_e32 v0, v0, v1 +; SI-NEXT: v_min_f32_e32 v1, 4.0, v1 +; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SI-NEXT: v_or_b32_e32 v0, v1, v0 ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -745,27 +733,21 @@ define amdgpu_kernel void @minnum_v3f16( ; SI-NEXT: s_mov_b32 s6, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: v_cvt_f32_f16_e32 v0, s3 -; SI-NEXT: v_cvt_f32_f16_e32 v1, s2 -; SI-NEXT: s_lshr_b32 s2, s2, 16 -; SI-NEXT: s_lshr_b32 s3, s0, 16 -; SI-NEXT: v_cvt_f32_f16_e32 v2, s3 +; SI-NEXT: s_lshr_b32 s3, s2, 16 +; SI-NEXT: s_lshr_b32 s8, s0, 16 +; SI-NEXT: v_cvt_f32_f16_e32 v1, s3 +; SI-NEXT: v_cvt_f32_f16_e32 v2, s8 ; SI-NEXT: v_cvt_f32_f16_e32 v3, s2 -; SI-NEXT: v_cvt_f32_f16_e32 v5, s0 -; SI-NEXT: v_cvt_f32_f16_e32 v4, s1 -; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; SI-NEXT: v_min_f32_e32 v2, v3, v2 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v5 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_min_f32_e32 v1, v1, v3 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v4 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 -; SI-NEXT: v_cvt_f16_f32_e32 v2, v2 -; SI-NEXT: v_min_f32_e32 v0, v0, v3 +; SI-NEXT: v_cvt_f32_f16_e32 v4, s0 +; SI-NEXT: v_cvt_f32_f16_e32 v5, s1 +; SI-NEXT: v_min_f32_e32 v1, v1, v2 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 +; SI-NEXT: v_min_f32_e32 v2, v3, v4 +; SI-NEXT: v_min_f32_e32 v0, v0, v5 +; SI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 -; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; SI-NEXT: v_or_b32_e32 v1, v1, v2 +; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; SI-NEXT: v_or_b32_e32 v1, v2, v1 ; SI-NEXT: buffer_store_short v0, off, s[4:7], 0 offset:4 ; SI-NEXT: buffer_store_dword v1, off, s[4:7], 0 ; SI-NEXT: s_endpgm @@ -893,25 +875,17 @@ define amdgpu_kernel void @minnum_v4f16( ; SI-NEXT: v_cvt_f32_f16_e32 v2, s6 ; SI-NEXT: s_lshr_b32 s6, s7, 16 ; SI-NEXT: v_cvt_f32_f16_e32 v3, s6 +; SI-NEXT: v_cvt_f32_f16_e32 v4, s4 ; SI-NEXT: s_lshr_b32 s6, s5, 16 +; SI-NEXT: s_lshr_b32 s4, s4, 16 ; SI-NEXT: v_cvt_f32_f16_e32 v5, s6 +; SI-NEXT: v_cvt_f32_f16_e32 v7, s4 ; SI-NEXT: v_cvt_f32_f16_e32 v1, s7 -; SI-NEXT: v_cvt_f32_f16_e32 v4, s4 -; SI-NEXT: s_lshr_b32 s4, s4, 16 -; SI-NEXT: v_cvt_f32_f16_e32 v7, s5 -; SI-NEXT: v_cvt_f32_f16_e32 v6, s4 -; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3 +; SI-NEXT: v_cvt_f32_f16_e32 v6, s5 ; SI-NEXT: v_min_f32_e32 v3, v3, v5 -; SI-NEXT: v_mul_f32_e32 v5, 1.0, v7 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_min_f32_e32 v1, v1, v5 -; SI-NEXT: v_mul_f32_e32 v5, 1.0, v6 -; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; SI-NEXT: v_min_f32_e32 v2, v2, v5 -; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 +; SI-NEXT: v_min_f32_e32 v2, v2, v7 ; SI-NEXT: v_cvt_f16_f32_e32 v3, v3 +; SI-NEXT: v_min_f32_e32 v1, v1, v6 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; SI-NEXT: v_min_f32_e32 v0, v0, v4 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 @@ -1041,20 +1015,16 @@ define amdgpu_kernel void @fmin_v4f16_imm_a( ; SI-NEXT: v_cvt_f32_f16_e32 v1, s5 ; SI-NEXT: s_lshr_b32 s5, s5, 16 ; SI-NEXT: v_cvt_f32_f16_e32 v0, s4 -; SI-NEXT: v_cvt_f32_f16_e32 v2, s5 ; SI-NEXT: s_lshr_b32 s4, s4, 16 +; SI-NEXT: v_cvt_f32_f16_e32 v2, s5 ; SI-NEXT: v_cvt_f32_f16_e32 v3, s4 -; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1 -; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; SI-NEXT: v_min_f32_e32 v2, 4.0, v2 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3 ; SI-NEXT: v_min_f32_e32 v1, 0x40400000, v1 +; SI-NEXT: v_min_f32_e32 v0, 0x41000000, v0 +; SI-NEXT: v_min_f32_e32 v2, 4.0, v2 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; SI-NEXT: v_min_f32_e32 v3, 2.0, v3 -; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; SI-NEXT: v_cvt_f16_f32_e32 v3, v3 -; SI-NEXT: v_min_f32_e32 v0, 0x41000000, v0 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; SI-NEXT: v_or_b32_e32 v1, v1, v2 diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll index fb3e79b2cf29..5b7f0e72b70d 100644 --- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll +++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll @@ -951,56 +951,70 @@ define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %s ; SDAG-GFX1100-LABEL: v_mad_mix_v3f32_clamp_postcvt: ; SDAG-GFX1100: ; %bb.0: ; SDAG-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SDAG-GFX1100-NEXT: v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] ; SDAG-GFX1100-NEXT: v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1] -; SDAG-GFX1100-NEXT: v_fma_mixlo_f16 v3, v0, v2, v4 op_sel_hi:[1,1,1] clamp ; SDAG-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; SDAG-GFX1100-NEXT: v_pack_b32_f16 v1, v1, 0 -; SDAG-GFX1100-NEXT: v_fma_mixhi_f16 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp +; SDAG-GFX1100-NEXT: v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] +; SDAG-GFX1100-NEXT: v_pack_b32_f16 v0, v1, 0 ; SDAG-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; SDAG-GFX1100-NEXT: v_pk_max_f16 v1, v1, v1 clamp -; SDAG-GFX1100-NEXT: v_mov_b32_e32 v0, v3 +; SDAG-GFX1100-NEXT: v_pk_max_f16 v1, v6, 0 +; SDAG-GFX1100-NEXT: v_pk_max_f16 v2, v0, 0 +; SDAG-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; SDAG-GFX1100-NEXT: v_pk_min_f16 v0, v1, 1.0 op_sel_hi:[1,0] +; SDAG-GFX1100-NEXT: v_pk_min_f16 v1, v2, 1.0 op_sel_hi:[1,0] ; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31] ; ; SDAG-GFX900-LABEL: v_mad_mix_v3f32_clamp_postcvt: ; SDAG-GFX900: ; %bb.0: ; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SDAG-GFX900-NEXT: v_mad_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] ; SDAG-GFX900-NEXT: v_mad_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1] -; SDAG-GFX900-NEXT: v_mad_mixlo_f16 v3, v0, v2, v4 op_sel_hi:[1,1,1] clamp ; SDAG-GFX900-NEXT: v_pack_b32_f16 v1, v1, 0 -; SDAG-GFX900-NEXT: v_mad_mixhi_f16 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp -; SDAG-GFX900-NEXT: v_pk_max_f16 v1, v1, v1 clamp -; SDAG-GFX900-NEXT: v_mov_b32_e32 v0, v3 +; SDAG-GFX900-NEXT: v_mad_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] +; SDAG-GFX900-NEXT: v_pk_max_f16 v1, v1, 0 +; SDAG-GFX900-NEXT: v_pk_max_f16 v0, v6, 0 +; SDAG-GFX900-NEXT: v_pk_min_f16 v0, v0, 1.0 op_sel_hi:[1,0] +; SDAG-GFX900-NEXT: v_pk_min_f16 v1, v1, 1.0 op_sel_hi:[1,0] ; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31] ; ; SDAG-GFX906-LABEL: v_mad_mix_v3f32_clamp_postcvt: ; SDAG-GFX906: ; %bb.0: ; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SDAG-GFX906-NEXT: v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] ; SDAG-GFX906-NEXT: v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1] -; SDAG-GFX906-NEXT: v_fma_mixlo_f16 v3, v0, v2, v4 op_sel_hi:[1,1,1] clamp ; SDAG-GFX906-NEXT: v_pack_b32_f16 v1, v1, 0 -; SDAG-GFX906-NEXT: v_fma_mixhi_f16 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp -; SDAG-GFX906-NEXT: v_pk_max_f16 v1, v1, v1 clamp -; SDAG-GFX906-NEXT: v_mov_b32_e32 v0, v3 +; SDAG-GFX906-NEXT: v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] +; SDAG-GFX906-NEXT: v_pk_max_f16 v1, v1, 0 +; SDAG-GFX906-NEXT: v_pk_max_f16 v0, v6, 0 +; SDAG-GFX906-NEXT: v_pk_min_f16 v0, v0, 1.0 op_sel_hi:[1,0] +; SDAG-GFX906-NEXT: v_pk_min_f16 v1, v1, 1.0 op_sel_hi:[1,0] ; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31] ; ; SDAG-VI-LABEL: v_mad_mix_v3f32_clamp_postcvt: ; SDAG-VI: ; %bb.0: ; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1 ; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v3, v3 ; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2 ; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v4, v4 -; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1 -; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v3, v3 ; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v5, v5 ; SDAG-VI-NEXT: v_mac_f32_e32 v8, v6, v7 ; SDAG-VI-NEXT: v_mac_f32_e32 v4, v0, v2 -; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v8 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD -; SDAG-VI-NEXT: v_cvt_f16_f32_e64 v2, v4 clamp ; SDAG-VI-NEXT: v_mac_f32_e32 v5, v1, v3 -; SDAG-VI-NEXT: v_cvt_f16_f32_e64 v1, v5 clamp +; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v0, v8 +; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v1, v4 +; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v2, v5 +; SDAG-VI-NEXT: v_max_f16_e32 v0, 0, v0 +; SDAG-VI-NEXT: v_max_f16_e32 v3, 0, v1 +; SDAG-VI-NEXT: v_max_f16_e32 v1, 0, v2 +; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x3c00 +; SDAG-VI-NEXT: v_min_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; SDAG-VI-NEXT: v_min_f16_e32 v2, 1.0, v3 +; SDAG-VI-NEXT: v_min_f16_e32 v1, 1.0, v1 ; SDAG-VI-NEXT: v_or_b32_e32 v0, v2, v0 ; SDAG-VI-NEXT: s_setpc_b64 s[30:31] ; @@ -1139,63 +1153,80 @@ define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %s } define <4 x half> @v_mad_mix_v4f32_clamp_postcvt(<4 x half> %src0, <4 x half> %src1, <4 x half> %src2) #0 { -; GFX1100-LABEL: v_mad_mix_v4f32_clamp_postcvt: -; GFX1100: ; %bb.0: -; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX1100-NEXT: v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp -; GFX1100-NEXT: v_fma_mixlo_f16 v7, v1, v3, v5 op_sel_hi:[1,1,1] clamp -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1100-NEXT: v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp -; GFX1100-NEXT: v_fma_mixhi_f16 v7, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1100-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7 -; GFX1100-NEXT: s_setpc_b64 s[30:31] +; SDAG-GFX1100-LABEL: v_mad_mix_v4f32_clamp_postcvt: +; SDAG-GFX1100: ; %bb.0: +; SDAG-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SDAG-GFX1100-NEXT: v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] +; SDAG-GFX1100-NEXT: v_fma_mixlo_f16 v7, v1, v3, v5 op_sel_hi:[1,1,1] +; SDAG-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; SDAG-GFX1100-NEXT: v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] +; SDAG-GFX1100-NEXT: v_fma_mixhi_f16 v7, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] +; SDAG-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; SDAG-GFX1100-NEXT: v_pk_max_f16 v0, v6, 0 +; SDAG-GFX1100-NEXT: v_pk_max_f16 v1, v7, 0 +; SDAG-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; SDAG-GFX1100-NEXT: v_pk_min_f16 v0, v0, 1.0 op_sel_hi:[1,0] +; SDAG-GFX1100-NEXT: v_pk_min_f16 v1, v1, 1.0 op_sel_hi:[1,0] +; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31] ; -; GFX900-LABEL: v_mad_mix_v4f32_clamp_postcvt: -; GFX900: ; %bb.0: -; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX900-NEXT: v_mad_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp -; GFX900-NEXT: v_mad_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp -; GFX900-NEXT: v_mad_mixlo_f16 v2, v1, v3, v5 op_sel_hi:[1,1,1] clamp -; GFX900-NEXT: v_mad_mixhi_f16 v2, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp -; GFX900-NEXT: v_mov_b32_e32 v0, v6 -; GFX900-NEXT: v_mov_b32_e32 v1, v2 -; GFX900-NEXT: s_setpc_b64 s[30:31] +; SDAG-GFX900-LABEL: v_mad_mix_v4f32_clamp_postcvt: +; SDAG-GFX900: ; %bb.0: +; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SDAG-GFX900-NEXT: v_mad_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] +; SDAG-GFX900-NEXT: v_mad_mixlo_f16 v7, v1, v3, v5 op_sel_hi:[1,1,1] +; SDAG-GFX900-NEXT: v_mad_mixhi_f16 v7, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] +; SDAG-GFX900-NEXT: v_mad_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] +; SDAG-GFX900-NEXT: v_pk_max_f16 v1, v7, 0 +; SDAG-GFX900-NEXT: v_pk_max_f16 v0, v6, 0 +; SDAG-GFX900-NEXT: v_pk_min_f16 v0, v0, 1.0 op_sel_hi:[1,0] +; SDAG-GFX900-NEXT: v_pk_min_f16 v1, v1, 1.0 op_sel_hi:[1,0] +; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31] ; -; GFX906-LABEL: v_mad_mix_v4f32_clamp_postcvt: -; GFX906: ; %bb.0: -; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX906-NEXT: v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp -; GFX906-NEXT: v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp -; GFX906-NEXT: v_fma_mixlo_f16 v2, v1, v3, v5 op_sel_hi:[1,1,1] clamp -; GFX906-NEXT: v_fma_mixhi_f16 v2, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp -; GFX906-NEXT: v_mov_b32_e32 v0, v6 -; GFX906-NEXT: v_mov_b32_e32 v1, v2 -; GFX906-NEXT: s_setpc_b64 s[30:31] +; SDAG-GFX906-LABEL: v_mad_mix_v4f32_clamp_postcvt: +; SDAG-GFX906: ; %bb.0: +; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SDAG-GFX906-NEXT: v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] +; SDAG-GFX906-NEXT: v_fma_mixlo_f16 v7, v1, v3, v5 op_sel_hi:[1,1,1] +; SDAG-GFX906-NEXT: v_fma_mixhi_f16 v7, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] +; SDAG-GFX906-NEXT: v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] +; SDAG-GFX906-NEXT: v_pk_max_f16 v1, v7, 0 +; SDAG-GFX906-NEXT: v_pk_max_f16 v0, v6, 0 +; SDAG-GFX906-NEXT: v_pk_min_f16 v0, v0, 1.0 op_sel_hi:[1,0] +; SDAG-GFX906-NEXT: v_pk_min_f16 v1, v1, 1.0 op_sel_hi:[1,0] +; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31] ; ; SDAG-VI-LABEL: v_mad_mix_v4f32_clamp_postcvt: ; SDAG-VI: ; %bb.0: ; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0 +; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1 -; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2 +; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0 +; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v3, v3 -; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v10, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v11, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v5, v5 +; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2 +; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v10, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v11, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v4, v4 +; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v5, v5 ; SDAG-VI-NEXT: v_mac_f32_e32 v10, v7, v9 ; SDAG-VI-NEXT: v_mac_f32_e32 v11, v6, v8 -; SDAG-VI-NEXT: v_mac_f32_e32 v5, v1, v3 ; SDAG-VI-NEXT: v_mac_f32_e32 v4, v0, v2 -; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v11 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD -; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v1, v10 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD -; SDAG-VI-NEXT: v_cvt_f16_f32_e64 v2, v4 clamp -; SDAG-VI-NEXT: v_cvt_f16_f32_e64 v3, v5 clamp +; SDAG-VI-NEXT: v_mac_f32_e32 v5, v1, v3 +; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v0, v10 +; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v1, v11 +; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v2, v4 +; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v3, v5 +; SDAG-VI-NEXT: v_max_f16_e32 v0, 0, v0 +; SDAG-VI-NEXT: v_max_f16_e32 v1, 0, v1 +; SDAG-VI-NEXT: v_max_f16_e32 v2, 0, v2 +; SDAG-VI-NEXT: v_max_f16_e32 v3, 0, v3 +; SDAG-VI-NEXT: v_mov_b32_e32 v4, 0x3c00 +; SDAG-VI-NEXT: v_min_f16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; SDAG-VI-NEXT: v_min_f16_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; SDAG-VI-NEXT: v_min_f16_e32 v3, 1.0, v3 +; SDAG-VI-NEXT: v_min_f16_e32 v2, 1.0, v2 ; SDAG-VI-NEXT: v_or_b32_e32 v0, v2, v0 ; SDAG-VI-NEXT: v_or_b32_e32 v1, v3, v1 ; SDAG-VI-NEXT: s_setpc_b64 s[30:31] @@ -1241,6 +1272,40 @@ define <4 x half> @v_mad_mix_v4f32_clamp_postcvt(<4 x half> %src0, <4 x half> %s ; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v3, v3 clamp ; SDAG-CI-NEXT: s_setpc_b64 s[30:31] ; +; GISEL-GFX1100-LABEL: v_mad_mix_v4f32_clamp_postcvt: +; GISEL-GFX1100: ; %bb.0: +; GISEL-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GISEL-GFX1100-NEXT: v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp +; GISEL-GFX1100-NEXT: v_fma_mixlo_f16 v7, v1, v3, v5 op_sel_hi:[1,1,1] clamp +; GISEL-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GISEL-GFX1100-NEXT: v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp +; GISEL-GFX1100-NEXT: v_fma_mixhi_f16 v7, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp +; GISEL-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GISEL-GFX1100-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7 +; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31] +; +; GISEL-GFX900-LABEL: v_mad_mix_v4f32_clamp_postcvt: +; GISEL-GFX900: ; %bb.0: +; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GISEL-GFX900-NEXT: v_mad_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp +; GISEL-GFX900-NEXT: v_mad_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp +; GISEL-GFX900-NEXT: v_mad_mixlo_f16 v2, v1, v3, v5 op_sel_hi:[1,1,1] clamp +; GISEL-GFX900-NEXT: v_mad_mixhi_f16 v2, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp +; GISEL-GFX900-NEXT: v_mov_b32_e32 v0, v6 +; GISEL-GFX900-NEXT: v_mov_b32_e32 v1, v2 +; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GISEL-GFX906-LABEL: v_mad_mix_v4f32_clamp_postcvt: +; GISEL-GFX906: ; %bb.0: +; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GISEL-GFX906-NEXT: v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp +; GISEL-GFX906-NEXT: v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp +; GISEL-GFX906-NEXT: v_fma_mixlo_f16 v2, v1, v3, v5 op_sel_hi:[1,1,1] clamp +; GISEL-GFX906-NEXT: v_fma_mixhi_f16 v2, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp +; GISEL-GFX906-NEXT: v_mov_b32_e32 v0, v6 +; GISEL-GFX906-NEXT: v_mov_b32_e32 v1, v2 +; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31] +; ; GISEL-VI-LABEL: v_mad_mix_v4f32_clamp_postcvt: ; GISEL-VI: ; %bb.0: ; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -- GitLab From c18e1215c4f387058961651148be730144d3537b Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Wed, 13 Mar 2024 20:15:29 +0800 Subject: [PATCH 0113/1407] [InstCombine] Simplify `zext nneg i1 X` to zero (#85043) Alive2: https://alive2.llvm.org/ce/z/Wm6kCk --- .../InstCombine/InstCombineCasts.cpp | 4 +++ llvm/test/Transforms/InstCombine/zext.ll | 31 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp index 45afa6363ae0..a9817f1af8c1 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp @@ -1121,6 +1121,10 @@ Instruction *InstCombinerImpl::visitZExt(ZExtInst &Zext) { Value *Src = Zext.getOperand(0); Type *SrcTy = Src->getType(), *DestTy = Zext.getType(); + // zext nneg bool x -> 0 + if (SrcTy->isIntOrIntVectorTy(1) && Zext.hasNonNeg()) + return replaceInstUsesWith(Zext, Constant::getNullValue(Zext.getType())); + // Try to extend the entire expression tree to the wide destination type. unsigned BitsToClear; if (shouldChangeType(SrcTy, DestTy) && diff --git a/llvm/test/Transforms/InstCombine/zext.ll b/llvm/test/Transforms/InstCombine/zext.ll index edbd4850fb11..88cd9c70af40 100644 --- a/llvm/test/Transforms/InstCombine/zext.ll +++ b/llvm/test/Transforms/InstCombine/zext.ll @@ -836,3 +836,34 @@ define i64 @zext_nneg_demanded_constant(i8 %a) nounwind { %c = and i64 %b, 254 ret i64 %c } + +define i32 @zext_nneg_i1(i1 %x) { +; CHECK-LABEL: @zext_nneg_i1( +; CHECK-NEXT: entry: +; CHECK-NEXT: ret i32 0 +; +entry: + %res = zext nneg i1 %x to i32 + ret i32 %res +} + +define <2 x i32> @zext_nneg_i1_vec(<2 x i1> %x) { +; CHECK-LABEL: @zext_nneg_i1_vec( +; CHECK-NEXT: entry: +; CHECK-NEXT: ret <2 x i32> zeroinitializer +; +entry: + %res = zext nneg <2 x i1> %x to <2 x i32> + ret <2 x i32> %res +} + +define i32 @zext_nneg_i2(i2 %x) { +; CHECK-LABEL: @zext_nneg_i2( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[RES:%.*]] = zext nneg i2 [[X:%.*]] to i32 +; CHECK-NEXT: ret i32 [[RES]] +; +entry: + %res = zext nneg i2 %x to i32 + ret i32 %res +} -- GitLab From 203757776826cfd164c537048ec90f5ada50cae2 Mon Sep 17 00:00:00 2001 From: Jacek Caban Date: Wed, 13 Mar 2024 13:27:20 +0100 Subject: [PATCH 0114/1407] [llvm-ar] Use COFF archive format for COFF targets. (#82898) Detect COFF files by default and allow specifying it with --format argument. This is important for ARM64EC, which uses a separated symbol map for EC symbols. Since K_COFF is mostly compatible with K_GNU, this shouldn't really make a difference for other targets. This originally landed as #82642, but was reverted due to test failures in tests using no symbol table. Since COFF symbol can't express it, fallback to GNU format in that case. --- llvm/docs/CommandGuide/llvm-ar.rst | 2 +- llvm/docs/ReleaseNotes.rst | 3 + llvm/include/llvm/Object/Archive.h | 1 + llvm/lib/Object/Archive.cpp | 15 ++-- llvm/lib/Object/ArchiveWriter.cpp | 28 ++++---- llvm/test/tools/llvm-ar/coff-symtab.test | 91 ++++++++++++++++++++++++ llvm/test/tools/llvm-ar/no-symtab.yaml | 32 +++++++++ llvm/tools/llvm-ar/llvm-ar.cpp | 22 ++++-- 8 files changed, 171 insertions(+), 23 deletions(-) create mode 100644 llvm/test/tools/llvm-ar/coff-symtab.test create mode 100644 llvm/test/tools/llvm-ar/no-symtab.yaml diff --git a/llvm/docs/CommandGuide/llvm-ar.rst b/llvm/docs/CommandGuide/llvm-ar.rst index 03d5b9e41ada..63b3a519550b 100644 --- a/llvm/docs/CommandGuide/llvm-ar.rst +++ b/llvm/docs/CommandGuide/llvm-ar.rst @@ -261,7 +261,7 @@ Other .. option:: --format= - This option allows for default, gnu, darwin or bsd ```` to be selected. + This option allows for default, gnu, darwin, bsd or coff ```` to be selected. When creating an ``archive`` with the default ````, :program:``llvm-ar`` will attempt to infer it from the input files and fallback to the default toolchain target if unable to do so. diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst index b34a5f31c5eb..7be51730663b 100644 --- a/llvm/docs/ReleaseNotes.rst +++ b/llvm/docs/ReleaseNotes.rst @@ -153,6 +153,9 @@ Changes to the LLVM tools if it's not specified with the ``--format`` argument and cannot be inferred from input files. +* llvm-ar now allows specifying COFF archive format with ``--format`` argument + and uses it by default for COFF targets. + * llvm-objcopy now supports ``--set-symbol-visibility`` and ``--set-symbols-visibility`` options for ELF input to change the visibility of symbols. diff --git a/llvm/include/llvm/Object/Archive.h b/llvm/include/llvm/Object/Archive.h index f71630054dc6..a3165c3235e0 100644 --- a/llvm/include/llvm/Object/Archive.h +++ b/llvm/include/llvm/Object/Archive.h @@ -339,6 +339,7 @@ public: Kind kind() const { return (Kind)Format; } bool isThin() const { return IsThin; } static object::Archive::Kind getDefaultKind(); + static object::Archive::Kind getDefaultKindForTriple(Triple &T); child_iterator child_begin(Error &Err, bool SkipInternal = true) const; child_iterator child_end() const; diff --git a/llvm/lib/Object/Archive.cpp b/llvm/lib/Object/Archive.cpp index 9000e9aa81ff..6139d9996bda 100644 --- a/llvm/lib/Object/Archive.cpp +++ b/llvm/lib/Object/Archive.cpp @@ -969,12 +969,19 @@ Archive::Archive(MemoryBufferRef Source, Error &Err) Err = Error::success(); } +object::Archive::Kind Archive::getDefaultKindForTriple(Triple &T) { + if (T.isOSDarwin()) + return object::Archive::K_DARWIN; + if (T.isOSAIX()) + return object::Archive::K_AIXBIG; + if (T.isOSWindows()) + return object::Archive::K_COFF; + return object::Archive::K_GNU; +} + object::Archive::Kind Archive::getDefaultKind() { Triple HostTriple(sys::getDefaultTargetTriple()); - return HostTriple.isOSDarwin() - ? object::Archive::K_DARWIN - : (HostTriple.isOSAIX() ? object::Archive::K_AIXBIG - : object::Archive::K_GNU); + return getDefaultKindForTriple(HostTriple); } Archive::child_iterator Archive::child_begin(Error &Err, diff --git a/llvm/lib/Object/ArchiveWriter.cpp b/llvm/lib/Object/ArchiveWriter.cpp index e0629747b40c..aa57e55de70c 100644 --- a/llvm/lib/Object/ArchiveWriter.cpp +++ b/llvm/lib/Object/ArchiveWriter.cpp @@ -62,12 +62,16 @@ object::Archive::Kind NewArchiveMember::detectKindFromObject() const { Expected> OptionalObject = object::ObjectFile::createObjectFile(MemBufferRef); - if (OptionalObject) - return isa(**OptionalObject) - ? object::Archive::K_DARWIN - : (isa(**OptionalObject) - ? object::Archive::K_AIXBIG - : object::Archive::K_GNU); + if (OptionalObject) { + if (isa(**OptionalObject)) + return object::Archive::K_DARWIN; + if (isa(**OptionalObject)) + return object::Archive::K_AIXBIG; + if (isa(**OptionalObject) || + isa(**OptionalObject)) + return object::Archive::K_COFF; + return object::Archive::K_GNU; + } // Squelch the error in case we had a non-object file. consumeError(OptionalObject.takeError()); @@ -80,10 +84,7 @@ object::Archive::Kind NewArchiveMember::detectKindFromObject() const { MemBufferRef, file_magic::bitcode, &Context)) { auto &IRObject = cast(**ObjOrErr); auto TargetTriple = Triple(IRObject.getTargetTriple()); - return TargetTriple.isOSDarwin() - ? object::Archive::K_DARWIN - : (TargetTriple.isOSAIX() ? object::Archive::K_AIXBIG - : object::Archive::K_GNU); + return object::Archive::getDefaultKindForTriple(TargetTriple); } else { // Squelch the error in case this was not a SymbolicFile. consumeError(ObjOrErr.takeError()); @@ -976,10 +977,12 @@ static Error writeArchiveToStream(raw_ostream &Out, SmallString<0> StringTableBuf; raw_svector_ostream StringTable(StringTableBuf); SymMap SymMap; + bool ShouldWriteSymtab = WriteSymtab != SymtabWritingMode::NoSymtab; // COFF symbol map uses 16-bit indexes, so we can't use it if there are too - // many members. - if (isCOFFArchive(Kind) && NewMembers.size() > 0xfffe) + // many members. COFF format also requires symbol table presence, so use + // GNU format when NoSymtab is requested. + if (isCOFFArchive(Kind) && (NewMembers.size() > 0xfffe || !ShouldWriteSymtab)) Kind = object::Archive::K_GNU; // In the scenario when LLVMContext is populated SymbolicFile will contain a @@ -1008,7 +1011,6 @@ static Error writeArchiveToStream(raw_ostream &Out, uint64_t LastMemberHeaderOffset = 0; uint64_t NumSyms = 0; uint64_t NumSyms32 = 0; // Store symbol number of 32-bit member files. - bool ShouldWriteSymtab = WriteSymtab != SymtabWritingMode::NoSymtab; for (const auto &M : Data) { // Record the start of the member's offset diff --git a/llvm/test/tools/llvm-ar/coff-symtab.test b/llvm/test/tools/llvm-ar/coff-symtab.test new file mode 100644 index 000000000000..4f7270d9e2c6 --- /dev/null +++ b/llvm/test/tools/llvm-ar/coff-symtab.test @@ -0,0 +1,91 @@ +Verify that llvm-ar uses COFF archive format by ensuring that archive map is sorted. + +RUN: rm -rf %t.dir && split-file %s %t.dir && cd %t.dir + +RUN: yaml2obj coff-symtab.yaml -o coff-symtab.obj +RUN: llvm-ar crs out.a coff-symtab.obj +RUN: llvm-nm --print-armap out.a | FileCheck %s + +RUN: llvm-as coff-symtab.ll -o coff-symtab.bc +RUN: llvm-ar crs out2.a coff-symtab.bc +RUN: llvm-nm --print-armap out2.a | FileCheck %s + +RUN: yaml2obj elf.yaml -o coff-symtab.o +RUN: llvm-ar crs --format coff out3.a coff-symtab.o +RUN: llvm-nm --print-armap out3.a | FileCheck %s + +Create an empty archive with no symbol map, add a COFF file to it and check that the output archive is a COFF archive. + +RUN: llvm-ar rcS out4.a +RUN: llvm-ar rs out4.a coff-symtab.obj +RUN: llvm-nm --print-armap out4.a | FileCheck %s + +CHECK: Archive map +CHECK-NEXT: a in coff-symtab +CHECK-NEXT: b in coff-symtab +CHECK-NEXT: c in coff-symtab +CHECK-EMPTY: + +#--- coff-symtab.yaml +--- !COFF +header: + Machine: IMAGE_FILE_MACHINE_UNKNOWN + Characteristics: [ ] +sections: + - Name: .text + Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ] + Alignment: 4 + SectionData: '' +symbols: + - Name: b + Value: 0 + SectionNumber: 1 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_FUNCTION + StorageClass: IMAGE_SYM_CLASS_EXTERNAL + - Name: c + Value: 0 + SectionNumber: 1 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_FUNCTION + StorageClass: IMAGE_SYM_CLASS_EXTERNAL + - Name: a + Value: 0 + SectionNumber: 1 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_FUNCTION + StorageClass: IMAGE_SYM_CLASS_EXTERNAL +... + + +#--- coff-symtab.ll +target triple = "x86_64-unknown-windows-msvc" + +define void @b() { ret void } +define void @c() { ret void } +define void @a() { ret void } + +#--- elf.yaml +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data : ELFDATA2LSB + Type: ET_REL + Machine: EM_X86_64 +Sections: + - Name: .text + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_EXECINSTR ] + AddressAlign: 0x0000000000000004 + Content: '' +Symbols: + - Name: b + Binding: STB_GLOBAL + Section: .text + - Name: c + Binding: STB_GLOBAL + Section: .text + - Name: a + Binding: STB_GLOBAL + Section: .text +... diff --git a/llvm/test/tools/llvm-ar/no-symtab.yaml b/llvm/test/tools/llvm-ar/no-symtab.yaml new file mode 100644 index 000000000000..7370c9b32355 --- /dev/null +++ b/llvm/test/tools/llvm-ar/no-symtab.yaml @@ -0,0 +1,32 @@ +## Create archives with no symtab in various formats and check that we can read them. + +# RUN: yaml2obj %s -o %t.o +# RUN: rm -f %t.*.a + +# RUN: llvm-ar --format=gnu rcS %t.gnu.a %t.o +# RUN: llvm-ar --format=coff rcS %t.coff.a %t.o +# RUN: llvm-ar --format=darwin rcS %t.darwin.a %t.o +# RUN: llvm-ar --format=bsd rcS %t.bsd.a %t.o +# RUN: llvm-ar --format=bigarchive rcS %t.bigarchive.a %t.o + +# RUN: llvm-nm --print-armap %t.gnu.a | FileCheck %s +# RUN: llvm-nm --print-armap %t.coff.a | FileCheck %s +# RUN: llvm-nm --print-armap %t.darwin.a | FileCheck %s +# RUN: llvm-nm --print-armap %t.bsd.a | FileCheck %s +# RUN: llvm-nm --print-armap %t.bigarchive.a | FileCheck %s + +# CHECK-NOT: Archive map + +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_REL + Machine: EM_X86_64 +Sections: + - Name: .text + Type: SHT_PROGBITS +Symbols: + - Name: symbol + Binding: STB_GLOBAL + Section: .text diff --git a/llvm/tools/llvm-ar/llvm-ar.cpp b/llvm/tools/llvm-ar/llvm-ar.cpp index 81cb2a21daf1..294b8531b08f 100644 --- a/llvm/tools/llvm-ar/llvm-ar.cpp +++ b/llvm/tools/llvm-ar/llvm-ar.cpp @@ -82,6 +82,7 @@ static void printArHelp(StringRef ToolName) { =darwin - darwin =bsd - bsd =bigarchive - big archive (AIX OS) + =coff - coff --plugin= - ignored for compatibility -h --help - display this help and exit --output - the directory to extract archive members to @@ -193,7 +194,7 @@ static SmallVector PositionalArgs; static bool MRI; namespace { -enum Format { Default, GNU, BSD, DARWIN, BIGARCHIVE, Unknown }; +enum Format { Default, GNU, COFF, BSD, DARWIN, BIGARCHIVE, Unknown }; } static Format FormatType = Default; @@ -1025,14 +1026,21 @@ static void performWriteOperation(ArchiveOperation Operation, Kind = object::Archive::K_GNU; else if (OldArchive) { Kind = OldArchive->kind(); - if (Kind == object::Archive::K_BSD) { - auto InferredKind = object::Archive::K_BSD; + std::optional AltKind; + if (Kind == object::Archive::K_BSD) + AltKind = object::Archive::K_DARWIN; + else if (Kind == object::Archive::K_GNU && !OldArchive->hasSymbolTable()) + // If there is no symbol table, we can't tell GNU from COFF format + // from the old archive type. + AltKind = object::Archive::K_COFF; + if (AltKind) { + auto InferredKind = Kind; if (NewMembersP && !NewMembersP->empty()) InferredKind = NewMembersP->front().detectKindFromObject(); else if (!NewMembers.empty()) InferredKind = NewMembers.front().detectKindFromObject(); - if (InferredKind == object::Archive::K_DARWIN) - Kind = object::Archive::K_DARWIN; + if (InferredKind == AltKind) + Kind = *AltKind; } } else if (NewMembersP) Kind = !NewMembersP->empty() ? NewMembersP->front().detectKindFromObject() @@ -1044,6 +1052,9 @@ static void performWriteOperation(ArchiveOperation Operation, case GNU: Kind = object::Archive::K_GNU; break; + case COFF: + Kind = object::Archive::K_COFF; + break; case BSD: if (Thin) fail("only the gnu format has a thin mode"); @@ -1376,6 +1387,7 @@ static int ar_main(int argc, char **argv) { .Case("darwin", DARWIN) .Case("bsd", BSD) .Case("bigarchive", BIGARCHIVE) + .Case("coff", COFF) .Default(Unknown); if (FormatType == Unknown) fail(std::string("Invalid format ") + Match); -- GitLab From 9e406ef4f4b089f88e74b2713f0fbee51b9537d6 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Wed, 13 Mar 2024 09:00:53 -0400 Subject: [PATCH 0115/1407] [libc++] Remove _LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT (#83928) This was slated for removal in LLVM 19. --- libcxx/docs/ReleaseNotes/19.rst | 2 +- libcxx/include/variant | 32 +++---------------- .../variant.variant/variant.assign/T.pass.cpp | 15 ++------- .../variant.assign/conv.pass.cpp | 8 ++--- .../variant.variant/variant.ctor/T.pass.cpp | 15 ++------- .../variant.ctor/conv.pass.cpp | 8 ++--- libcxx/test/support/variant_test_helpers.h | 10 ------ 7 files changed, 18 insertions(+), 72 deletions(-) diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst index 04f16610f811..2b62a36ca8e5 100644 --- a/libcxx/docs/ReleaseNotes/19.rst +++ b/libcxx/docs/ReleaseNotes/19.rst @@ -65,7 +65,7 @@ Deprecations and Removals provided, and such a base template is bound to be incorrect for some types, which could currently cause unexpected behavior while going undetected. -- TODO: The ``_LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT`` macro that changed the behavior for narrowing conversions +- The ``_LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT`` macro that changed the behavior for narrowing conversions in ``std::variant`` has been removed in LLVM 19. - TODO: The ``_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS`` macro has been removed in LLVM 19. diff --git a/libcxx/include/variant b/libcxx/include/variant index d1eea52f0a93..59d7f9b740f3 100644 --- a/libcxx/include/variant +++ b/libcxx/include/variant @@ -347,13 +347,13 @@ inline constexpr size_t variant_npos = static_cast(-1); template _LIBCPP_HIDE_FROM_ABI constexpr auto __choose_index_type() { -#ifdef _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION +# ifdef _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION if constexpr (_NumAlternatives < numeric_limits::max()) return static_cast(0); else if constexpr (_NumAlternatives < numeric_limits::max()) return static_cast(0); else -#endif // _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION +# endif // _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION return static_cast(0); } @@ -1085,13 +1085,9 @@ struct __narrowing_check { }; template -using __check_for_narrowing _LIBCPP_NODEBUG = typename _If< -# ifdef _LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT - false && -# endif - is_arithmetic<_Dest>::value, - __narrowing_check, - __no_narrowing_check >::template _Apply<_Dest, _Source>; +using __check_for_narrowing _LIBCPP_NODEBUG = + typename _If< is_arithmetic<_Dest>::value, __narrowing_check, __no_narrowing_check >::template _Apply<_Dest, + _Source>; template struct __overload { @@ -1099,24 +1095,6 @@ struct __overload { auto operator()(_Tp, _Up&&) const -> __check_for_narrowing<_Tp, _Up>; }; -// TODO(LLVM-19): Remove all occurrences of this macro. -# ifdef _LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT -template -struct __overload_bool { - template > - auto operator()(bool, _Up&&) const -> enable_if_t, __type_identity<_Tp>>; -}; - -template -struct __overload : __overload_bool {}; -template -struct __overload : __overload_bool {}; -template -struct __overload : __overload_bool {}; -template -struct __overload : __overload_bool {}; -# endif - template struct __all_overloads : _Bases... { void operator()() const; diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/T.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/T.pass.cpp index b3fc2021a6b2..b38b10d89dfd 100644 --- a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/T.pass.cpp +++ b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/T.pass.cpp @@ -134,8 +134,7 @@ void test_T_assignment_sfinae() { } { using V = std::variant; - static_assert(std::is_assignable::value == VariantAllowsNarrowingConversions, - "no matching operator="); + static_assert(!std::is_assignable::value, "no matching operator="); } { using V = std::variant, bool>; @@ -144,12 +143,8 @@ void test_T_assignment_sfinae() { struct X { operator void*(); }; - static_assert(!std::is_assignable::value, - "no boolean conversion in operator="); -#ifndef _LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT - static_assert(std::is_assignable::value, - "converted to bool in operator="); -#endif + static_assert(!std::is_assignable::value, "no boolean conversion in operator="); + static_assert(std::is_assignable::value, "converted to bool in operator="); } { struct X {}; @@ -188,7 +183,6 @@ void test_T_assignment_basic() { assert(v.index() == 1); assert(std::get<1>(v) == 43); } -#ifndef TEST_VARIANT_ALLOWS_NARROWING_CONVERSIONS { std::variant v; v = 42; @@ -198,7 +192,6 @@ void test_T_assignment_basic() { assert(v.index() == 0); assert(std::get<0>(v) == 43); } -#endif { std::variant v = true; v = "bar"; @@ -299,13 +292,11 @@ void test_T_assignment_performs_assignment() { } void test_T_assignment_vector_bool() { -#ifndef _LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT std::vector vec = {true}; std::variant v; v = vec[0]; assert(v.index() == 0); assert(std::get<0>(v) == true); -#endif } int main(int, char**) { diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/conv.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/conv.pass.cpp index 246309c01b4d..90e405d57750 100644 --- a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/conv.pass.cpp +++ b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/conv.pass.cpp @@ -25,18 +25,16 @@ int main(int, char**) { static_assert(!std::is_assignable, int>::value, ""); static_assert(!std::is_assignable, int>::value, ""); - static_assert(std::is_assignable, int>::value == VariantAllowsNarrowingConversions, ""); + static_assert(!std::is_assignable, int>::value, ""); - static_assert(std::is_assignable, int>::value == VariantAllowsNarrowingConversions, ""); - static_assert(std::is_assignable, int>::value == VariantAllowsNarrowingConversions, ""); + static_assert(!std::is_assignable, int>::value, ""); + static_assert(!std::is_assignable, int>::value, ""); static_assert(!std::is_assignable, int>::value, ""); static_assert(!std::is_assignable, decltype("meow")>::value, ""); static_assert(!std::is_assignable, decltype("meow")>::value, ""); -#ifndef _LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT static_assert(std::is_assignable, std::true_type>::value, ""); -#endif static_assert(!std::is_assignable, std::unique_ptr >::value, ""); static_assert(!std::is_assignable, decltype(nullptr)>::value, ""); diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/T.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/T.pass.cpp index 89fd646878ee..6b7de8888849 100644 --- a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/T.pass.cpp +++ b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/T.pass.cpp @@ -68,8 +68,7 @@ void test_T_ctor_sfinae() { } { using V = std::variant; - static_assert(std::is_constructible::value == VariantAllowsNarrowingConversions, - "no matching constructor"); + static_assert(!std::is_constructible::value, "no matching constructor"); } { using V = std::variant, bool>; @@ -78,12 +77,8 @@ void test_T_ctor_sfinae() { struct X { operator void*(); }; - static_assert(!std::is_constructible::value, - "no boolean conversion in constructor"); -#ifndef _LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT - static_assert(std::is_constructible::value, - "converted to bool in constructor"); -#endif + static_assert(!std::is_constructible::value, "no boolean conversion in constructor"); + static_assert(std::is_constructible::value, "converted to bool in constructor"); } { struct X {}; @@ -128,13 +123,11 @@ void test_T_ctor_basic() { static_assert(v.index() == 1, ""); static_assert(std::get<1>(v) == 42, ""); } -#ifndef TEST_VARIANT_ALLOWS_NARROWING_CONVERSIONS { constexpr std::variant v(42); static_assert(v.index() == 1, ""); static_assert(std::get<1>(v) == 42, ""); } -#endif { std::variant v = "foo"; assert(v.index() == 0); @@ -202,12 +195,10 @@ void test_construction_with_repeated_types() { } void test_vector_bool() { -#ifndef _LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT std::vector vec = {true}; std::variant v = vec[0]; assert(v.index() == 0); assert(std::get<0>(v) == true); -#endif } int main(int, char**) { diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/conv.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/conv.pass.cpp index 7fb44ff40765..0b8eeed1eac8 100644 --- a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/conv.pass.cpp +++ b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/conv.pass.cpp @@ -24,18 +24,16 @@ int main(int, char**) { static_assert(!std::is_constructible, int>::value, ""); static_assert(!std::is_constructible, int>::value, ""); - static_assert(std::is_constructible, int>::value == VariantAllowsNarrowingConversions, ""); + static_assert(!std::is_constructible, int>::value, ""); - static_assert(std::is_constructible, int>::value == VariantAllowsNarrowingConversions, ""); - static_assert(std::is_constructible, int>::value == VariantAllowsNarrowingConversions, ""); + static_assert(!std::is_constructible, int>::value, ""); + static_assert(!std::is_constructible, int>::value, ""); static_assert(!std::is_constructible, int>::value, ""); static_assert(!std::is_constructible, decltype("meow")>::value, ""); static_assert(!std::is_constructible, decltype("meow")>::value, ""); -#ifndef _LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT static_assert(std::is_constructible, std::true_type>::value, ""); -#endif static_assert(!std::is_constructible, std::unique_ptr >::value, ""); static_assert(!std::is_constructible, decltype(nullptr)>::value, ""); diff --git a/libcxx/test/support/variant_test_helpers.h b/libcxx/test/support/variant_test_helpers.h index c174cba32840..345e32170e58 100644 --- a/libcxx/test/support/variant_test_helpers.h +++ b/libcxx/test/support/variant_test_helpers.h @@ -24,16 +24,6 @@ // FIXME: Currently the variant tests are disabled using this macro. #define TEST_VARIANT_HAS_NO_REFERENCES -// TODO(LLVM-19): Remove TEST_VARIANT_ALLOWS_NARROWING_CONVERSIONS -#ifdef _LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT -# define TEST_VARIANT_ALLOWS_NARROWING_CONVERSIONS -#endif -#ifdef TEST_VARIANT_ALLOWS_NARROWING_CONVERSIONS -constexpr bool VariantAllowsNarrowingConversions = true; -#else -constexpr bool VariantAllowsNarrowingConversions = false; -#endif - #ifndef TEST_HAS_NO_EXCEPTIONS struct CopyThrows { CopyThrows() = default; -- GitLab From 5c3d001668ec6117045a9750a1f9d7e3995adfee Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Wed, 13 Mar 2024 18:34:23 +0530 Subject: [PATCH 0116/1407] AMDGPU: Don't use table for metadata docs, and fix section headers (#85046) --- llvm/docs/AMDGPUUsage.rst | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index fd9ad7fac19a..fe37e85c2a40 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -1312,24 +1312,30 @@ The AMDGPU backend implements the following LLVM IR intrinsics. List AMDGPU intrinsics. +.. _amdgpu_metadata: + LLVM IR Metadata ------------------- +================ + +The AMDGPU backend implements the following target custom LLVM IR +metadata. + +.. _amdgpu_last_use: -The AMDGPU backend implements the following LLVM IR metadata. +'``amdgpu.last.use``' Metadata +------------------------------ + +Sets TH_LOAD_LU temporal hint on load instructions that support it. +Takes priority over nontemporal hint (TH_LOAD_NT). This takes no +arguments. + +.. code-block:: llvm -.. list-table:: AMDGPU LLVM IR Metatdata - :name: amdgpu-llvm-ir-metadata-table + %val = load i32, ptr %in, align 4, !amdgpu.last.use !{} - * - Metadata Name - - Description - - Values - * - !amdgpu.last.use - - Sets TH_LOAD_LU temporal hint on load instructions that support it. - Takes priority over nontemporal hint (TH_LOAD_NT). - - {} LLVM IR Attributes ------------------- +================== The AMDGPU backend supports the following LLVM IR attributes. @@ -1451,7 +1457,7 @@ The AMDGPU backend supports the following LLVM IR attributes. ======================================= ========================================================== Calling Conventions -------------------- +=================== The AMDGPU backend supports the following calling conventions: -- GitLab From 9fa866020395b215ece6140c2fedc7c31950272c Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Wed, 13 Mar 2024 13:11:45 +0000 Subject: [PATCH 0117/1407] [AMDGPU] Test new GFX12 opcode name buffer_atomic_min_num_f32 The old name buffer_atomic_min_f32 is still tested as part of the alias tests. --- llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf.s | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf.s b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf.s index 08ec5b3f6a52..efeaf8339f69 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf.s @@ -3970,70 +3970,70 @@ buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607 dlc buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc // GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction -buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 +buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 // GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f] -buffer_atomic_min_f32 v255, off, s[8:11], s3 offset:8388607 +buffer_atomic_min_num_f32 v255, off, s[8:11], s3 offset:8388607 // GFX12: encoding: [0x03,0x40,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f] -buffer_atomic_min_f32 v5, off, s[12:15], s3 offset:8388607 +buffer_atomic_min_num_f32 v5, off, s[12:15], s3 offset:8388607 // GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f] -buffer_atomic_min_f32 v5, off, s[96:99], s3 offset:8388607 +buffer_atomic_min_num_f32 v5, off, s[96:99], s3 offset:8388607 // GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f] -buffer_atomic_min_f32 v5, off, s[8:11], s101 offset:8388607 +buffer_atomic_min_num_f32 v5, off, s[8:11], s101 offset:8388607 // GFX12: encoding: [0x65,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f] -buffer_atomic_min_f32 v5, off, s[8:11], m0 offset:8388607 +buffer_atomic_min_num_f32 v5, off, s[8:11], m0 offset:8388607 // GFX12: encoding: [0x7d,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f] -buffer_atomic_min_f32 v5, off, s[8:11], 0 offset:8388607 +buffer_atomic_min_num_f32 v5, off, s[8:11], 0 offset:8388607 // GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction -buffer_atomic_min_f32 v5, off, s[8:11], -1 offset:8388607 +buffer_atomic_min_num_f32 v5, off, s[8:11], -1 offset:8388607 // GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction -buffer_atomic_min_f32 v5, off, s[8:11], 0.5 offset:8388607 +buffer_atomic_min_num_f32 v5, off, s[8:11], 0.5 offset:8388607 // GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction -buffer_atomic_min_f32 v5, off, s[8:11], -4.0 offset:8388607 +buffer_atomic_min_num_f32 v5, off, s[8:11], -4.0 offset:8388607 // GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction -buffer_atomic_min_f32 v5, v0, s[8:11], s3 idxen offset:8388607 +buffer_atomic_min_num_f32 v5, v0, s[8:11], s3 idxen offset:8388607 // GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f] -buffer_atomic_min_f32 v5, v0, s[8:11], s3 offen offset:8388607 +buffer_atomic_min_num_f32 v5, v0, s[8:11], s3 offen offset:8388607 // GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f] -buffer_atomic_min_f32 v5, off, s[8:11], s3 +buffer_atomic_min_num_f32 v5, off, s[8:11], s3 // GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00] -buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:0 +buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:0 // GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00] -buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:7 +buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:7 // GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00] -buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN +buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN // GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f] -buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE +buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE // GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f] -buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV +buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV // GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f] -buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 glc +buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 glc // GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction -buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 slc +buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 slc // GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction -buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 dlc +buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 dlc // GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction -buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc +buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc // GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 -- GitLab From e48d5a838f69e0a8e0ae95a8aed1a8809f45465a Mon Sep 17 00:00:00 2001 From: NagyDonat Date: Tue, 12 Mar 2024 18:24:26 +0100 Subject: [PATCH 0118/1407] Reapply "[analyzer] Accept C library functions from the `std` namespace" This reapplies f32b04d4ea91ad1018c25a1d4178cc4392d34968i, after fixing the use-after-free of ASTUnit in the unittest. https://github.com/llvm/llvm-project/pull/84469#issuecomment-1992163439 Co-authored-by: Balazs Benics --- .../Core/PathSensitive/CallDescription.h | 8 +- .../StaticAnalyzer/Core/CheckerContext.cpp | 8 +- clang/unittests/StaticAnalyzer/CMakeLists.txt | 1 + .../StaticAnalyzer/IsCLibraryFunctionTest.cpp | 84 +++++++++++++++++++ .../clang/unittests/StaticAnalyzer/BUILD.gn | 1 + 5 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h index 3432d2648633..b4e1636130ca 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h @@ -41,12 +41,8 @@ public: /// - We also accept calls where the number of arguments or parameters is /// greater than the specified value. /// For the exact heuristics, see CheckerContext::isCLibraryFunction(). - /// Note that functions whose declaration context is not a TU (e.g. - /// methods, functions in namespaces) are not accepted as C library - /// functions. - /// FIXME: If I understand it correctly, this discards calls where C++ code - /// refers a C library function through the namespace `std::` via headers - /// like . + /// (This mode only matches functions that are declared either directly + /// within a TU or in the namespace `std`.) CLibrary, /// Matches "simple" functions that are not methods. (Static methods are diff --git a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp index d6d4cec9dd3d..1a9bff529e9b 100644 --- a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp +++ b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp @@ -87,9 +87,11 @@ bool CheckerContext::isCLibraryFunction(const FunctionDecl *FD, if (!II) return false; - // Look through 'extern "C"' and anything similar invented in the future. - // If this function is not in TU directly, it is not a C library function. - if (!FD->getDeclContext()->getRedeclContext()->isTranslationUnit()) + // C library functions are either declared directly within a TU (the common + // case) or they are accessed through the namespace `std` (when they are used + // in C++ via headers like ). + const DeclContext *DC = FD->getDeclContext()->getRedeclContext(); + if (!(DC->isTranslationUnit() || DC->isStdNamespace())) return false; // If this function is not externally visible, it is not a C library function. diff --git a/clang/unittests/StaticAnalyzer/CMakeLists.txt b/clang/unittests/StaticAnalyzer/CMakeLists.txt index 775f0f8486b8..db56e77331b8 100644 --- a/clang/unittests/StaticAnalyzer/CMakeLists.txt +++ b/clang/unittests/StaticAnalyzer/CMakeLists.txt @@ -11,6 +11,7 @@ add_clang_unittest(StaticAnalysisTests CallEventTest.cpp ConflictingEvalCallsTest.cpp FalsePositiveRefutationBRVisitorTest.cpp + IsCLibraryFunctionTest.cpp NoStateChangeFuncVisitorTest.cpp ParamRegionTest.cpp RangeSetTest.cpp diff --git a/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp b/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp new file mode 100644 index 000000000000..31ff13f428da --- /dev/null +++ b/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp @@ -0,0 +1,84 @@ +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/ASTMatchers/ASTMatchers.h" +#include "clang/Analysis/AnalysisDeclContext.h" +#include "clang/Frontend/ASTUnit.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" +#include "clang/Tooling/Tooling.h" +#include "gtest/gtest.h" + +#include + +using namespace clang; +using namespace ento; +using namespace ast_matchers; + +class IsCLibraryFunctionTest : public testing::Test { +public: + const FunctionDecl *getFunctionDecl() const { return Result; } + + testing::AssertionResult buildAST(StringRef Code) { + ASTUnit = tooling::buildASTFromCode(Code); + if (!ASTUnit) + return testing::AssertionFailure() << "AST construction failed"; + + ASTContext &Context = ASTUnit->getASTContext(); + if (Context.getDiagnostics().hasErrorOccurred()) + return testing::AssertionFailure() << "Compilation error"; + + auto Matches = ast_matchers::match(functionDecl().bind("fn"), Context); + if (Matches.empty()) + return testing::AssertionFailure() << "No function declaration found"; + + if (Matches.size() > 1) + return testing::AssertionFailure() + << "Multiple function declarations found"; + + Result = Matches[0].getNodeAs("fn"); + return testing::AssertionSuccess(); + } + +private: + std::unique_ptr ASTUnit; + const FunctionDecl *Result = nullptr; +}; + +TEST_F(IsCLibraryFunctionTest, AcceptsGlobal) { + ASSERT_TRUE(buildAST(R"cpp(void fun();)cpp")); + EXPECT_TRUE(CheckerContext::isCLibraryFunction(getFunctionDecl())); +} + +TEST_F(IsCLibraryFunctionTest, AcceptsExternCGlobal) { + ASSERT_TRUE(buildAST(R"cpp(extern "C" { void fun(); })cpp")); + EXPECT_TRUE(CheckerContext::isCLibraryFunction(getFunctionDecl())); +} + +TEST_F(IsCLibraryFunctionTest, RejectsNoInlineNoExternalLinkage) { + // Functions that are neither inlined nor externally visible cannot be C library functions. + ASSERT_TRUE(buildAST(R"cpp(static void fun();)cpp")); + EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl())); +} + +TEST_F(IsCLibraryFunctionTest, RejectsAnonymousNamespace) { + ASSERT_TRUE(buildAST(R"cpp(namespace { void fun(); })cpp")); + EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl())); +} + +TEST_F(IsCLibraryFunctionTest, AcceptsStdNamespace) { + ASSERT_TRUE(buildAST(R"cpp(namespace std { void fun(); })cpp")); + EXPECT_TRUE(CheckerContext::isCLibraryFunction(getFunctionDecl())); +} + +TEST_F(IsCLibraryFunctionTest, RejectsOtherNamespaces) { + ASSERT_TRUE(buildAST(R"cpp(namespace stdx { void fun(); })cpp")); + EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl())); +} + +TEST_F(IsCLibraryFunctionTest, RejectsClassStatic) { + ASSERT_TRUE(buildAST(R"cpp(class A { static void fun(); };)cpp")); + EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl())); +} + +TEST_F(IsCLibraryFunctionTest, RejectsClassMember) { + ASSERT_TRUE(buildAST(R"cpp(class A { void fun(); };)cpp")); + EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl())); +} diff --git a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn index 01c2b6ced336..9c240cff1816 100644 --- a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn @@ -19,6 +19,7 @@ unittest("StaticAnalysisTests") { "CallEventTest.cpp", "ConflictingEvalCallsTest.cpp", "FalsePositiveRefutationBRVisitorTest.cpp", + "IsCLibraryFunctionTest.cpp", "NoStateChangeFuncVisitorTest.cpp", "ParamRegionTest.cpp", "RangeSetTest.cpp", -- GitLab From 83d178843f4322159b6469f430a8a241b8672d6d Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Wed, 13 Mar 2024 21:52:40 +0800 Subject: [PATCH 0119/1407] [InstCombine] Set zero_is_poison for ctlz/cttz if they are only used as shift amounts (#85035) Alive2: https://alive2.llvm.org/ce/z/r-67t9 It would improve the codegen if the target doesn't provide a defined value for ctlz/cttz with zero. --- .../InstCombine/InstCombineCalls.cpp | 5 + .../Transforms/InstCombine/shift-cttz-ctlz.ll | 93 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 llvm/test/Transforms/InstCombine/shift-cttz-ctlz.ll diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index f5f3716d390d..694b18017bab 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -504,6 +504,11 @@ static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) { return IC.replaceInstUsesWith(II, ConstantInt::getNullValue(II.getType())); } + // If ctlz/cttz is only used as a shift amount, set is_zero_poison to true. + if (II.hasOneUse() && match(Op1, m_Zero()) && + match(II.user_back(), m_Shift(m_Value(), m_Specific(&II)))) + return IC.replaceOperand(II, 1, IC.Builder.getTrue()); + Constant *C; if (IsTZ) { diff --git a/llvm/test/Transforms/InstCombine/shift-cttz-ctlz.ll b/llvm/test/Transforms/InstCombine/shift-cttz-ctlz.ll new file mode 100644 index 000000000000..2b2f820c9a09 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/shift-cttz-ctlz.ll @@ -0,0 +1,93 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes=instcombine -S | FileCheck %s + +define i32 @shl_cttz_false(i32 %x, i32 %y) { +; CHECK-LABEL: define i32 @shl_cttz_false( +; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CTTZ:%.*]] = call i32 @llvm.cttz.i32(i32 [[Y]], i1 true), !range [[RNG0:![0-9]+]] +; CHECK-NEXT: [[RES:%.*]] = shl i32 [[X]], [[CTTZ]] +; CHECK-NEXT: ret i32 [[RES]] +; +entry: + %cttz = call i32 @llvm.cttz.i32(i32 %y, i1 false) + %res = shl i32 %x, %cttz + ret i32 %res +} + +define i32 @shl_ctlz_false(i32 %x, i32 %y) { +; CHECK-LABEL: define i32 @shl_ctlz_false( +; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CTTZ:%.*]] = call i32 @llvm.ctlz.i32(i32 [[Y]], i1 true), !range [[RNG0]] +; CHECK-NEXT: [[RES:%.*]] = shl i32 [[X]], [[CTTZ]] +; CHECK-NEXT: ret i32 [[RES]] +; +entry: + %cttz = call i32 @llvm.ctlz.i32(i32 %y, i1 false) + %res = shl i32 %x, %cttz + ret i32 %res +} + +define i32 @lshr_cttz_false(i32 %x, i32 %y) { +; CHECK-LABEL: define i32 @lshr_cttz_false( +; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CTTZ:%.*]] = call i32 @llvm.cttz.i32(i32 [[Y]], i1 true), !range [[RNG0]] +; CHECK-NEXT: [[RES:%.*]] = lshr i32 [[X]], [[CTTZ]] +; CHECK-NEXT: ret i32 [[RES]] +; +entry: + %cttz = call i32 @llvm.cttz.i32(i32 %y, i1 false) + %res = lshr i32 %x, %cttz + ret i32 %res +} + +define i32 @ashr_cttz_false(i32 %x, i32 %y) { +; CHECK-LABEL: define i32 @ashr_cttz_false( +; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CTTZ:%.*]] = call i32 @llvm.cttz.i32(i32 [[Y]], i1 true), !range [[RNG0]] +; CHECK-NEXT: [[RES:%.*]] = ashr i32 [[X]], [[CTTZ]] +; CHECK-NEXT: ret i32 [[RES]] +; +entry: + %cttz = call i32 @llvm.cttz.i32(i32 %y, i1 false) + %res = ashr i32 %x, %cttz + ret i32 %res +} + +define i32 @shl_cttz_false_multiuse(i32 %x, i32 %y) { +; CHECK-LABEL: define i32 @shl_cttz_false_multiuse( +; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CTTZ:%.*]] = call i32 @llvm.cttz.i32(i32 [[Y]], i1 false), !range [[RNG0]] +; CHECK-NEXT: call void @use(i32 [[CTTZ]]) +; CHECK-NEXT: [[RES:%.*]] = shl i32 [[X]], [[CTTZ]] +; CHECK-NEXT: ret i32 [[RES]] +; +entry: + %cttz = call i32 @llvm.cttz.i32(i32 %y, i1 false) + call void @use(i32 %cttz) + %res = shl i32 %x, %cttz + ret i32 %res +} + +define i32 @shl_cttz_as_lhs(i32 %x, i32 %y) { +; CHECK-LABEL: define i32 @shl_cttz_as_lhs( +; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CTTZ:%.*]] = call i32 @llvm.cttz.i32(i32 [[Y]], i1 false), !range [[RNG0]] +; CHECK-NEXT: [[RES:%.*]] = shl i32 [[CTTZ]], [[X]] +; CHECK-NEXT: ret i32 [[RES]] +; +entry: + %cttz = call i32 @llvm.cttz.i32(i32 %y, i1 false) + %res = shl i32 %cttz, %x + ret i32 %res +} + +declare void @use(i32) +;. +; CHECK: [[RNG0]] = !{i32 0, i32 33} +;. -- GitLab From 960b4aa6dab69125778f230c4c94f2d19c96cc87 Mon Sep 17 00:00:00 2001 From: Jake Egan Date: Wed, 13 Mar 2024 09:54:35 -0400 Subject: [PATCH 0120/1407] [AIX][ClangRepl] Disable new test on AIX This new test fails on the AIX bot with error `LLVM ERROR: Incompatible object format!`. Disable for now to investigate. Same as 86337beca2e6f939127cd3e088ec80c0cf4a0a64. --- clang/unittests/Interpreter/InterpreterExtensionsTest.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp index 77fd1b4e1981..1cf564b5671b 100644 --- a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp +++ b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp @@ -56,7 +56,11 @@ public: void resetExecutor() { Interpreter::ResetExecutor(); } }; +ifdef _AIX +TEST(InterpreterExtensionsTest, DISABLED_ExecutorCreateReset) { +#else TEST(InterpreterExtensionsTest, ExecutorCreateReset) { +#endif // Make sure we can create the executer on the platform. if (!HostSupportsJit()) GTEST_SKIP(); -- GitLab From 424e0a825fe4d9e3bf98b63ef86edbc4fa5e3799 Mon Sep 17 00:00:00 2001 From: Jake Egan Date: Wed, 13 Mar 2024 09:57:31 -0400 Subject: [PATCH 0121/1407] [ClangRepl] Add missing hashtag Hashtag missing from commit 960b4aa6dab69125778f230c4c94f2d19c96cc87 --- clang/unittests/Interpreter/InterpreterExtensionsTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp index 1cf564b5671b..b7708616fd24 100644 --- a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp +++ b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp @@ -56,7 +56,7 @@ public: void resetExecutor() { Interpreter::ResetExecutor(); } }; -ifdef _AIX +#ifdef _AIX TEST(InterpreterExtensionsTest, DISABLED_ExecutorCreateReset) { #else TEST(InterpreterExtensionsTest, ExecutorCreateReset) { -- GitLab From 2cf2bc472da87bb4bf971b1448e05b9e3bd983dc Mon Sep 17 00:00:00 2001 From: Sirraide Date: Wed, 13 Mar 2024 14:59:55 +0100 Subject: [PATCH 0122/1407] [Clang] [CodeGen] Fix codegen bug in constant initialisation in C23 mode (#84981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consider the following code: ```c bool const inf = (1.0/0.0); ``` When trying to emit the initialiser of this variable in C23, we end up hitting a code path in codegen in `VarDecl::evaluateValueImpl()` where we check for `IsConstantInitialization && (Ctx.getLangOpts().CPlusPlus || Ctx.getLangOpts().C23)`, and if that is the case and we emitted any notes, constant evaluation fails, and as a result, codegen issues this error: ``` :1:12: error: cannot compile this static initializer yet 1 | bool const inf = (1.0/0.0); | ``` As a fix, only fail in C23 mode if we’re initialising a `constexpr` variable. This fixes #84784. --- clang/lib/AST/Decl.cpp | 11 +++++++---- clang/test/CodeGen/const-init.c | 3 +++ clang/test/Sema/const-init.c | 6 ++++++ 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 clang/test/Sema/const-init.c diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp index 8626f04012f7..95900afdd2c5 100644 --- a/clang/lib/AST/Decl.cpp +++ b/clang/lib/AST/Decl.cpp @@ -2577,11 +2577,14 @@ APValue *VarDecl::evaluateValueImpl(SmallVectorImpl &Notes, bool Result = Init->EvaluateAsInitializer(Eval->Evaluated, Ctx, this, Notes, IsConstantInitialization); - // In C++/C23, this isn't a constant initializer if we produced notes. In that - // case, we can't keep the result, because it may only be correct under the - // assumption that the initializer is a constant context. + // In C++, or in C23 if we're initialising a 'constexpr' variable, this isn't + // a constant initializer if we produced notes. In that case, we can't keep + // the result, because it may only be correct under the assumption that the + // initializer is a constant context. if (IsConstantInitialization && - (Ctx.getLangOpts().CPlusPlus || Ctx.getLangOpts().C23) && !Notes.empty()) + (Ctx.getLangOpts().CPlusPlus || + (isConstexpr() && Ctx.getLangOpts().C23)) && + !Notes.empty()) Result = false; // Ensure the computed APValue is cleaned up later if evaluation succeeded, diff --git a/clang/test/CodeGen/const-init.c b/clang/test/CodeGen/const-init.c index 0e4fc4ad48af..ad3e9551199a 100644 --- a/clang/test/CodeGen/const-init.c +++ b/clang/test/CodeGen/const-init.c @@ -216,3 +216,6 @@ int PR4517_x2 = PR4517_arrc[PR4517_idx]; // CHECK: @PR4517_x = global i32 42, align 4 // CHECK: @PR4517_idx = constant i32 1, align 4 // CHECK: @PR4517_x2 = global i32 42, align 4 + +// CHECK: @GH84784_inf = constant i8 1 +_Bool const GH84784_inf = (1.0/0.0); diff --git a/clang/test/Sema/const-init.c b/clang/test/Sema/const-init.c new file mode 100644 index 000000000000..5b07ede747eb --- /dev/null +++ b/clang/test/Sema/const-init.c @@ -0,0 +1,6 @@ +// RUN: %clang_cc1 -fsyntax-only -verify -std=c23 %s + +// Division by 0 here is an error iff the variable is 'constexpr'. +const _Bool inf1 = (1.0/0.0 == __builtin_inf()); +constexpr _Bool inf2 = (1.0/0.0 == __builtin_inf()); // expected-error {{must be initialized by a constant expression}} expected-note {{division by zero}} +constexpr _Bool inf3 = __builtin_inf() == __builtin_inf(); -- GitLab From 390f28702fad7b704d026b5c3e9a6030cecab01b Mon Sep 17 00:00:00 2001 From: mahesh-attarde <145317060+mahesh-attarde@users.noreply.github.com> Date: Wed, 13 Mar 2024 07:03:15 -0700 Subject: [PATCH 0123/1407] [CodeGen][Tablegen] Fix uninitialized var and shift overflow. (#84896) Fix uninitialized var and shift overflow. --- llvm/include/llvm/CodeGen/AccelTable.h | 2 +- llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp | 3 ++- llvm/utils/TableGen/DecoderEmitter.cpp | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/llvm/include/llvm/CodeGen/AccelTable.h b/llvm/include/llvm/CodeGen/AccelTable.h index 6ee817a7124d..cff8fcbaf2cd 100644 --- a/llvm/include/llvm/CodeGen/AccelTable.h +++ b/llvm/include/llvm/CodeGen/AccelTable.h @@ -353,7 +353,7 @@ public: dwarf::Index Index; dwarf::Form Form; }; - DebugNamesAbbrev(uint32_t DieTag) : DieTag(DieTag) {} + DebugNamesAbbrev(uint32_t DieTag) : DieTag(DieTag), Number(0) {} /// Add attribute encoding to an abbreviation. void addAttribute(const DebugNamesAbbrev::AttributeEncoding &Attr) { AttrVect.push_back(Attr); diff --git a/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp b/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp index 55cdc3c92864..2e8e7d0a88af 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp @@ -369,7 +369,8 @@ void AppleAccelTableWriter::emit() const { DWARF5AccelTableData::DWARF5AccelTableData(const DIE &Die, const uint32_t UnitID, const bool IsTU) - : OffsetVal(&Die), DieTag(Die.getTag()), IsTU(IsTU), UnitID(UnitID) {} + : OffsetVal(&Die), DieTag(Die.getTag()), AbbrevNumber(0), IsTU(IsTU), + UnitID(UnitID) {} void Dwarf5AccelTableWriter::Header::emit(Dwarf5AccelTableWriter &Ctx) { assert(CompUnitCount > 0 && "Index must have at least one CU."); diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp index dd78dc02159b..628bff520a12 100644 --- a/llvm/utils/TableGen/DecoderEmitter.cpp +++ b/llvm/utils/TableGen/DecoderEmitter.cpp @@ -934,7 +934,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table, unsigned Shift = 0; do { OS << ", " << (unsigned)*I; - Value += (*I & 0x7f) << Shift; + Value += ((uint64_t)(*I & 0x7f)) << Shift; Shift += 7; } while (*I++ >= 128); if (Value > 127) { @@ -947,7 +947,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table, Shift = 0; do { OS << ", " << (unsigned)*I; - Value += (*I & 0x7f) << Shift; + Value += ((uint64_t)(*I & 0x7f)) << Shift; Shift += 7; } while (*I++ >= 128); if (Value > 127) { -- GitLab From eb21ee49cff081911d99d29ba887c1715fc2b8fc Mon Sep 17 00:00:00 2001 From: David Spickett Date: Wed, 13 Mar 2024 14:14:24 +0000 Subject: [PATCH 0124/1407] [lldb][test] Disable other runlocker test on AArch64 Linux Flaky on the bot: https://lab.llvm.org/buildbot/#/builders/96/builds/54435 --- lldb/test/API/python_api/run_locker/TestRunLocker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lldb/test/API/python_api/run_locker/TestRunLocker.py b/lldb/test/API/python_api/run_locker/TestRunLocker.py index 10832840ac09..4e0dd26bff70 100644 --- a/lldb/test/API/python_api/run_locker/TestRunLocker.py +++ b/lldb/test/API/python_api/run_locker/TestRunLocker.py @@ -15,6 +15,8 @@ class TestRunLocker(TestBase): NO_DEBUG_INFO_TESTCASE = True @expectedFailureAll(oslist=["windows"]) + # Is flaky on Linux AArch64 buildbot. + @skipIf(oslist=["linux"], archs=["aarch64"]) def test_run_locker(self): """Test that the run locker is set correctly when we launch""" self.build() -- GitLab From e77324decf74e8203fdee53e53c1866319ebf47c Mon Sep 17 00:00:00 2001 From: Zepp Date: Wed, 13 Mar 2024 22:25:29 +0800 Subject: [PATCH 0125/1407] [Clang] [Docs] Add reference to documentation of `SysVABIAttr` (#85022) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We for some reason already had documentation for this attribute, but just weren’t linking to it. --- clang/include/clang/Basic/Attr.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index 63efd85dcd4e..67d87eca16ed 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -2934,7 +2934,7 @@ def Suppress : DeclOrStmtAttr { def SysVABI : DeclOrTypeAttr { let Spellings = [GCC<"sysv_abi">]; // let Subjects = [Function, ObjCMethod]; - let Documentation = [Undocumented]; + let Documentation = [SysVABIDocs]; } def ThisCall : DeclOrTypeAttr { -- GitLab From 37b5eb0a0a75bdf69b96b902417906da31c88dc3 Mon Sep 17 00:00:00 2001 From: Zaara Syeda <95926691+syzaara@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:26:31 -0400 Subject: [PATCH 0126/1407] [AIX][TOC] Add -mtocdata/-mno-tocdata options on AIX (#67999) This patch enables support that the XL compiler had for AIX under -qdatalocal/-qdataimported. --- clang/docs/UsersManual.rst | 61 ++++++++++ clang/include/clang/Basic/CodeGenOptions.h | 9 ++ .../clang/Basic/DiagnosticDriverKinds.td | 3 + .../clang/Basic/DiagnosticFrontendKinds.td | 2 + clang/include/clang/Driver/Options.td | 21 ++++ clang/lib/CodeGen/CGDecl.cpp | 1 + clang/lib/CodeGen/CodeGenModule.cpp | 26 +++++ clang/lib/CodeGen/Targets/PPC.cpp | 59 ++++++++++ clang/lib/Driver/ToolChains/AIX.cpp | 87 ++++++++++++++ clang/lib/Frontend/CompilerInstance.cpp | 5 + .../test/CodeGen/PowerPC/toc-data-attribute.c | 50 ++++++++ .../CodeGen/PowerPC/toc-data-attribute.cpp | 39 +++++++ .../CodeGen/PowerPC/toc-data-diagnostics.c | 68 +++++++++++ .../PowerPC/toc-data-structs-arrays.cpp | 65 +++++++++++ clang/test/Driver/toc-conf.c | 30 +++++ clang/test/Driver/tocdata-cc1.c | 16 +++ llvm/include/llvm/ADT/STLExtras.h | 13 +++ llvm/lib/MC/MCSectionXCOFF.cpp | 3 +- llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp | 2 + llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp | 34 ------ llvm/lib/Target/PowerPC/PPCSubtarget.cpp | 22 ++++ llvm/lib/Target/PowerPC/PPCSubtarget.h | 2 + .../CodeGen/PowerPC/toc-data-large-array.ll | 16 +++ .../CodeGen/PowerPC/toc-data-large-array2.ll | 8 ++ .../CodeGen/PowerPC/toc-data-struct-array.ll | 110 ++++++++++++++++++ 25 files changed, 716 insertions(+), 36 deletions(-) create mode 100644 clang/test/CodeGen/PowerPC/toc-data-attribute.c create mode 100644 clang/test/CodeGen/PowerPC/toc-data-attribute.cpp create mode 100644 clang/test/CodeGen/PowerPC/toc-data-diagnostics.c create mode 100644 clang/test/CodeGen/PowerPC/toc-data-structs-arrays.cpp create mode 100644 clang/test/Driver/toc-conf.c create mode 100644 clang/test/Driver/tocdata-cc1.c create mode 100644 llvm/test/CodeGen/PowerPC/toc-data-large-array.ll create mode 100644 llvm/test/CodeGen/PowerPC/toc-data-large-array2.ll create mode 100644 llvm/test/CodeGen/PowerPC/toc-data-struct-array.ll diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst index 7391e4cf3a9a..7a63d720241a 100644 --- a/clang/docs/UsersManual.rst +++ b/clang/docs/UsersManual.rst @@ -4227,7 +4227,68 @@ Clang expects the GCC executable "gcc.exe" compiled for AIX ^^^ +TOC Data Transformation +""""""""""""""""""""""" +TOC data transformation is off by default (``-mno-tocdata``). +When ``-mtocdata`` is specified, the TOC data transformation will be applied to +all suitable variables with static storage duration, including static data +members of classes and block-scope static variables (if not marked as exceptions, +see further below). +Suitable variables must: + +- have complete types +- be independently generated (i.e., not placed in a pool) +- be at most as large as a pointer +- not be aligned more strictly than a pointer +- not be structs containing flexible array members +- not have internal linkage +- not have aliases +- not have section attributes +- not be thread local storage + +The TOC data transformation results in the variable, not its address, +being placed in the TOC. This eliminates the need to load the address of the +variable from the TOC. + +Note: +If the TOC data transformation is applied to a variable whose definition +is imported, the linker will generate fixup code for reading or writing to the +variable. + +When multiple toc-data options are used, the last option used has the affect. +For example: -mno-tocdata=g5,g1 -mtocdata=g1,g2 -mno-tocdata=g2 -mtocdata=g3,g4 +results in -mtocdata=g1,g3,g4 + +Names of variables not having external linkage will be ignored. + +**Options:** + +.. option:: -mno-tocdata + + This is the default behaviour. Only variables explicitly specified with + ``-mtocdata=`` will have the TOC data transformation applied. + +.. option:: -mtocdata + + Apply the TOC data transformation to all suitable variables with static + storage duration (including static data members of classes and block-scope + static variables) that are not explicitly specified with ``-mno-tocdata=``. + +.. option:: -mno-tocdata= + + Can be used in conjunction with ``-mtocdata`` to mark the comma-separated + list of external linkage variables, specified using their mangled names, as + exceptions to ``-mtocdata``. + +.. option:: -mtocdata= + + Apply the TOC data transformation to the comma-separated list of external + linkage variables, specified using their mangled names, if they are suitable. + Emit diagnostics for all unsuitable variables specified. + +Default Visibility Export Mapping +""""""""""""""""""""""""""""""""" The ``-mdefault-visibility-export-mapping=`` option can be used to control mapping of default visibility to an explicit shared object export (i.e. XCOFF exported visibility). Three values are provided for the option: diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h index 3f8fe385fef3..cf29e576ef32 100644 --- a/clang/include/clang/Basic/CodeGenOptions.h +++ b/clang/include/clang/Basic/CodeGenOptions.h @@ -404,6 +404,15 @@ public: /// List of pass builder callbacks. std::vector> PassBuilderCallbacks; + /// List of global variables explicitly specified by the user as toc-data. + std::vector TocDataVarsUserSpecified; + + /// List of global variables that over-ride the toc-data default. + std::vector NoTocDataVars; + + /// Flag for all global variables to be treated as toc-data. + bool AllTocData; + /// Path to allowlist file specifying which objects /// (files, functions) should exclusively be instrumented /// by sanitizer coverage pass. diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td index 1bc9885849d5..e33a1f4c45b9 100644 --- a/clang/include/clang/Basic/DiagnosticDriverKinds.td +++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td @@ -587,6 +587,9 @@ def warn_drv_unsupported_gpopt : Warning< "ignoring '-mgpopt' option as it cannot be used with %select{|the implicit" " usage of }0-mabicalls">, InGroup; +def warn_drv_unsupported_tocdata: Warning< + "ignoring '-mtocdata' as it is only supported for -mcmodel=small">, + InGroup; def warn_drv_unsupported_sdata : Warning< "ignoring '-msmall-data-limit=' with -mcmodel=large for -fpic or RV64">, InGroup; diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index dcd2c19fb7ee..794a0a82be6d 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -94,6 +94,8 @@ def err_fe_backend_error_attr : def warn_fe_backend_warning_attr : Warning<"call to '%0' declared with 'warning' attribute: %1">, BackendInfo, InGroup; +def warn_toc_unsupported_type : Warning<"-mtocdata option is ignored " + "for %0 because %1">, InGroup; def err_fe_invalid_code_complete_file : Error< "cannot locate code-completion file %0">, DefaultFatal; diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index aca8c9b0d548..1fac7b6f0093 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -3609,6 +3609,27 @@ def fpass_plugin_EQ : Joined<["-"], "fpass-plugin=">, MetaVarName<"">, HelpText<"Load pass plugin from a dynamic shared object file (only with new pass manager).">, MarshallingInfoStringVector>; +defm tocdata : BoolOption<"m","tocdata", + CodeGenOpts<"AllTocData">, DefaultFalse, + PosFlag, + NegFlag, + BothFlags<[TargetSpecific], [ClangOption, CLOption]>>, Group; +def mtocdata_EQ : CommaJoined<["-"], "mtocdata=">, + Visibility<[ClangOption, CC1Option]>, + Flags<[TargetSpecific]>, Group, + HelpText<"Specifies a list of variables to which the TOC data transformation" + "will be applied.">, + MarshallingInfoStringVector>; +def mno_tocdata_EQ : CommaJoined<["-"], "mno-tocdata=">, + Visibility<[ClangOption, CC1Option]>, + Flags<[TargetSpecific]>, Group, + HelpText<"Specifies a list of variables to be exempt from the TOC data" + "transformation.">, + MarshallingInfoStringVector>; defm preserve_as_comments : BoolFOption<"preserve-as-comments", CodeGenOpts<"PreserveAsmComments">, DefaultTrue, NegFlag(&D), GV, *this); // Make sure the result is of the correct type. LangAS ExpectedAS = Ty.getAddressSpace(); diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 967319bdfc45..8ceecff28cbc 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -626,6 +626,26 @@ static bool checkAliasedGlobal( return true; } +// Emit a warning if toc-data attribute is requested for global variables that +// have aliases and remove the toc-data attribute. +static void checkAliasForTocData(llvm::GlobalVariable *GVar, + const CodeGenOptions &CodeGenOpts, + DiagnosticsEngine &Diags, + SourceLocation Location) { + if (GVar->hasAttribute("toc-data")) { + auto GVId = GVar->getName(); + // Is this a global variable specified by the user as local? + if ((llvm::binary_search(CodeGenOpts.TocDataVarsUserSpecified, GVId))) { + Diags.Report(Location, diag::warn_toc_unsupported_type) + << GVId << "the variable has an alias"; + } + llvm::AttributeSet CurrAttributes = GVar->getAttributes(); + llvm::AttributeSet NewAttributes = + CurrAttributes.removeAttribute(GVar->getContext(), "toc-data"); + GVar->setAttributes(NewAttributes); + } +} + void CodeGenModule::checkAliases() { // Check if the constructed aliases are well formed. It is really unfortunate // that we have to do this in CodeGen, but we only construct mangled names @@ -652,6 +672,12 @@ void CodeGenModule::checkAliases() { continue; } + if (getContext().getTargetInfo().getTriple().isOSAIX()) + if (const llvm::GlobalVariable *GVar = + dyn_cast(GV)) + checkAliasForTocData(const_cast(GVar), + getCodeGenOpts(), Diags, Location); + llvm::Constant *Aliasee = IsIFunc ? cast(Alias)->getResolver() : cast(Alias)->getAliasee(); diff --git a/clang/lib/CodeGen/Targets/PPC.cpp b/clang/lib/CodeGen/Targets/PPC.cpp index 40dddde508c1..00b04723f17d 100644 --- a/clang/lib/CodeGen/Targets/PPC.cpp +++ b/clang/lib/CodeGen/Targets/PPC.cpp @@ -8,6 +8,7 @@ #include "ABIInfoImpl.h" #include "TargetInfo.h" +#include "clang/Basic/DiagnosticFrontend.h" using namespace clang; using namespace clang::CodeGen; @@ -145,6 +146,9 @@ public: bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const override; + + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + CodeGen::CodeGenModule &M) const override; }; } // namespace @@ -265,6 +269,61 @@ bool AIXTargetCodeGenInfo::initDwarfEHRegSizeTable( return PPC_initDwarfEHRegSizeTable(CGF, Address, Is64Bit, /*IsAIX*/ true); } +void AIXTargetCodeGenInfo::setTargetAttributes( + const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const { + if (!isa(GV)) + return; + + auto *GVar = dyn_cast(GV); + auto GVId = GV->getName(); + + // Is this a global variable specified by the user as toc-data? + bool UserSpecifiedTOC = + llvm::binary_search(M.getCodeGenOpts().TocDataVarsUserSpecified, GVId); + // Assumes the same variable cannot be in both TocVarsUserSpecified and + // NoTocVars. + if (UserSpecifiedTOC || + ((M.getCodeGenOpts().AllTocData) && + !llvm::binary_search(M.getCodeGenOpts().NoTocDataVars, GVId))) { + const unsigned long PointerSize = + GV->getParent()->getDataLayout().getPointerSizeInBits() / 8; + auto *VarD = dyn_cast(D); + assert(VarD && "Invalid declaration of global variable."); + + ASTContext &Context = D->getASTContext(); + unsigned Alignment = Context.toBits(Context.getDeclAlign(D)) / 8; + const auto *Ty = VarD->getType().getTypePtr(); + const RecordDecl *RDecl = + Ty->isRecordType() ? Ty->getAs()->getDecl() : nullptr; + + bool EmitDiagnostic = UserSpecifiedTOC && GV->hasExternalLinkage(); + auto reportUnsupportedWarning = [&](bool ShouldEmitWarning, StringRef Msg) { + if (ShouldEmitWarning) + M.getDiags().Report(D->getLocation(), diag::warn_toc_unsupported_type) + << GVId << Msg; + }; + if (!Ty || Ty->isIncompleteType()) + reportUnsupportedWarning(EmitDiagnostic, "of incomplete type"); + else if (RDecl && RDecl->hasFlexibleArrayMember()) + reportUnsupportedWarning(EmitDiagnostic, + "it contains a flexible array member"); + else if (VarD->getTLSKind() != VarDecl::TLS_None) + reportUnsupportedWarning(EmitDiagnostic, "of thread local storage"); + else if (PointerSize < Context.getTypeInfo(VarD->getType()).Width / 8) + reportUnsupportedWarning(EmitDiagnostic, + "variable is larger than a pointer"); + else if (PointerSize < Alignment) + reportUnsupportedWarning(EmitDiagnostic, + "variable is aligned wider than a pointer"); + else if (D->hasAttr()) + reportUnsupportedWarning(EmitDiagnostic, + "variable has a section attribute"); + else if (GV->hasExternalLinkage() || + (M.getCodeGenOpts().AllTocData && !GV->hasLocalLinkage())) + GVar->addAttribute("toc-data"); + } +} + // PowerPC-32 namespace { /// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information. diff --git a/clang/lib/Driver/ToolChains/AIX.cpp b/clang/lib/Driver/ToolChains/AIX.cpp index 3c7049a99982..6e089903a315 100644 --- a/clang/lib/Driver/ToolChains/AIX.cpp +++ b/clang/lib/Driver/ToolChains/AIX.cpp @@ -433,6 +433,88 @@ void AIX::AddCXXStdlibLibArgs(const llvm::opt::ArgList &Args, llvm_unreachable("Unexpected C++ library type; only libc++ is supported."); } +// This function processes all the mtocdata options to build the final +// simplified toc data options to pass to CC1. +static void addTocDataOptions(const llvm::opt::ArgList &Args, + llvm::opt::ArgStringList &CC1Args, + const Driver &D) { + + // Check the global toc-data setting. The default is -mno-tocdata. + // To enable toc-data globally, -mtocdata must be specified. + // Additionally, it must be last to take effect. + const bool TOCDataGloballyinEffect = [&Args]() { + if (const Arg *LastArg = + Args.getLastArg(options::OPT_mtocdata, options::OPT_mno_tocdata)) + return LastArg->getOption().matches(options::OPT_mtocdata); + else + return false; + }(); + + // Currently only supported for small code model. + if (TOCDataGloballyinEffect && + (Args.getLastArgValue(options::OPT_mcmodel_EQ).equals("large") || + Args.getLastArgValue(options::OPT_mcmodel_EQ).equals("medium"))) { + D.Diag(clang::diag::warn_drv_unsupported_tocdata); + return; + } + + enum TOCDataSetting { + AddressInTOC = 0, // Address of the symbol stored in the TOC. + DataInTOC = 1 // Symbol defined in the TOC. + }; + + const TOCDataSetting DefaultTocDataSetting = + TOCDataGloballyinEffect ? DataInTOC : AddressInTOC; + + // Process the list of variables in the explicitly specified options + // -mtocdata= and -mno-tocdata= to see which variables are opposite to + // the global setting of tocdata in TOCDataGloballyinEffect. + // Those that have the opposite setting to TOCDataGloballyinEffect, are added + // to ExplicitlySpecifiedGlobals. + llvm::StringSet<> ExplicitlySpecifiedGlobals; + for (const auto Arg : + Args.filtered(options::OPT_mtocdata_EQ, options::OPT_mno_tocdata_EQ)) { + TOCDataSetting ArgTocDataSetting = + Arg->getOption().matches(options::OPT_mtocdata_EQ) ? DataInTOC + : AddressInTOC; + + if (ArgTocDataSetting != DefaultTocDataSetting) + for (const char *Val : Arg->getValues()) + ExplicitlySpecifiedGlobals.insert(Val); + else + for (const char *Val : Arg->getValues()) + ExplicitlySpecifiedGlobals.erase(Val); + } + + auto buildExceptionList = [](const llvm::StringSet<> &ExplicitValues, + const char *OptionSpelling) { + std::string Option(OptionSpelling); + bool IsFirst = true; + for (const auto &E : ExplicitValues) { + if (!IsFirst) + Option += ","; + + IsFirst = false; + Option += E.first(); + } + return Option; + }; + + // Pass the final tocdata options to CC1 consisting of the default + // tocdata option (-mtocdata/-mno-tocdata) along with the list + // option (-mno-tocdata=/-mtocdata=) if there are any explicitly specified + // variables which would be exceptions to the default setting. + const char *TocDataGlobalOption = + TOCDataGloballyinEffect ? "-mtocdata" : "-mno-tocdata"; + CC1Args.push_back(TocDataGlobalOption); + + const char *TocDataListOption = + TOCDataGloballyinEffect ? "-mno-tocdata=" : "-mtocdata="; + if (!ExplicitlySpecifiedGlobals.empty()) + CC1Args.push_back(Args.MakeArgString(llvm::Twine( + buildExceptionList(ExplicitlySpecifiedGlobals, TocDataListOption)))); +} + void AIX::addClangTargetOptions( const llvm::opt::ArgList &Args, llvm::opt::ArgStringList &CC1Args, Action::OffloadKind DeviceOffloadingKind) const { @@ -440,6 +522,11 @@ void AIX::addClangTargetOptions( Args.AddLastArg(CC1Args, options::OPT_mdefault_visibility_export_mapping_EQ); Args.addOptInFlag(CC1Args, options::OPT_mxcoff_roptr, options::OPT_mno_xcoff_roptr); + // Forward last mtocdata/mno_tocdata options to -cc1. + if (Args.hasArg(options::OPT_mtocdata_EQ, options::OPT_mno_tocdata_EQ, + options::OPT_mtocdata)) + addTocDataOptions(Args, CC1Args, getDriver()); + if (Args.hasFlag(options::OPT_fxl_pragma_pack, options::OPT_fno_xl_pragma_pack, true)) CC1Args.push_back("-fxl-pragma-pack"); diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index ec4e68209d65..019f847ccbaa 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -1047,6 +1047,11 @@ bool CompilerInstance::ExecuteAction(FrontendAction &Act) { if (getFrontendOpts().ShowStats || !getFrontendOpts().StatsFile.empty()) llvm::EnableStatistics(false); + // Sort vectors containing toc data and no toc data variables to facilitate + // binary search later. + llvm::sort(getCodeGenOpts().TocDataVarsUserSpecified); + llvm::sort(getCodeGenOpts().NoTocDataVars); + for (const FrontendInputFile &FIF : getFrontendOpts().Inputs) { // Reset the ID tables if we are reusing the SourceManager and parsing // regular files. diff --git a/clang/test/CodeGen/PowerPC/toc-data-attribute.c b/clang/test/CodeGen/PowerPC/toc-data-attribute.c new file mode 100644 index 000000000000..db23d74759ee --- /dev/null +++ b/clang/test/CodeGen/PowerPC/toc-data-attribute.c @@ -0,0 +1,50 @@ +// RUN: %clang_cc1 %s -triple powerpc-ibm-aix-xcoff -S -mtocdata=f,g,h,i,j,k,l,m,n,o,p -emit-llvm -o - 2>&1 | FileCheck %s -check-prefixes=COMMON,CHECK32 --match-full-lines +// RUN: %clang_cc1 %s -triple powerpc-ibm-aix-xcoff -S -mtocdata -emit-llvm -o - 2>&1 | FileCheck %s -check-prefixes=COMMON,CHECK32 --match-full-lines + +// RUN: %clang_cc1 %s -triple powerpc64-ibm-aix-xcoff -S -mtocdata=f,g,h,i,j,k,l,m,n,o,p -emit-llvm -o - 2>&1 | FileCheck %s -check-prefixes=COMMON,CHECK64 --match-full-lines +// RUN: %clang_cc1 %s -triple powerpc64-ibm-aix-xcoff -S -mtocdata -emit-llvm -o - 2>&1 | FileCheck %s -check-prefixes=COMMON,CHECK64 --match-full-lines + +extern int f; +long long g = 5; +const char *h = "h"; +int *i; +int __attribute__((aligned(128))) j = 0; +float k = 100.00; +double l = 2.5; +int m __attribute__((section("foo"))) = 10; +__thread int n; + +extern int p[]; + +struct SomeStruct; +extern struct SomeStruct o; + +static int func_a() { + return g+(int)h[0]+*i+j+k+l+m+n+p[0]; +} + +int func_b() { + f = 1; + return func_a(); +} + +struct SomeStruct* getAddress(void) { + return &o; +} + +// CHECK32: @g = global i64 5, align 8 +// CHECK64: @g = global i64 5, align 8 #0 +// COMMON: {{.*}} = private unnamed_addr constant [2 x i8] c"h\00", align 1 +// COMMON: @h = global {{...*}} #0 +// COMMON: @j = global i32 0, align 128 +// COMMON: @k = global float 1.000000e+02, align 4 #0 +// CHECK32: @l = global double 2.500000e+00, align 8 +// CHECK64: @l = global double 2.500000e+00, align 8 #0 +// COMMON: @m = global i32 10, section "foo", align 4 +// COMMON: @f = external global i32, align 4 #0 +// COMMON: @o = external global %struct.SomeStruct, align 1 +// CHECK32: @i = global ptr null, align 4 #0 +// CHECK64: @i = global ptr null, align 8 #0 +// COMMON: @n = thread_local global i32 0, align 4 +// COMMON: @p = external global [0 x i32], align 4 +// COMMON: attributes #0 = { "toc-data" } diff --git a/clang/test/CodeGen/PowerPC/toc-data-attribute.cpp b/clang/test/CodeGen/PowerPC/toc-data-attribute.cpp new file mode 100644 index 000000000000..8183e3b727e7 --- /dev/null +++ b/clang/test/CodeGen/PowerPC/toc-data-attribute.cpp @@ -0,0 +1,39 @@ +// RUN: %clang_cc1 %s -triple powerpc-ibm-aix-xcoff -S -mtocdata -emit-llvm -o - 2>&1 | FileCheck %s -check-prefixes=COMMON,ALLTOC +// RUN: %clang_cc1 %s -triple powerpc-ibm-aix-xcoff -S -mtocdata=n,_ZN11MyNamespace10myVariableE,_ZL1s,_ZZ4testvE7counter -emit-llvm -o - 2>&1 | FileCheck %s -check-prefixes=COMMON,TOCLIST +// RUN: %clang_cc1 %s -triple powerpc64-ibm-aix-xcoff -S -mtocdata -emit-llvm -o - 2>&1 | FileCheck %s -check-prefixes=COMMON,ALLTOC +// RUN: %clang_cc1 %s -triple powerpc64-ibm-aix-xcoff -S -mtocdata=n,_ZN11MyNamespace10myVariableE,_ZL1s,_ZZ4testvE7counter -emit-llvm -o - 2>&1 | FileCheck %s -check-prefixes=COMMON,TOCLIST + +extern int n; +static int s = 100; + +inline int test() { + static int counter = 0; + counter++; + return counter; +} + +int a () { + n = test(); + return 0; +} + +namespace MyNamespace { + int myVariable = 10; +} + +int b(int x) { + using namespace MyNamespace; + return x + myVariable; +} + +int c(int x) { + s += x; + return s; +} + +// COMMON: @n = external global i32, align 4 #0 +// COMMON: @_ZN11MyNamespace10myVariableE = global i32 10, align 4 #0 +// COMMON-NOT: @_ZL1s = internal global i32 100, align 4 #0 +// ALLTOC: @_ZZ4testvE7counter = linkonce_odr global i32 0, align 4 #0 +// TOCLIST-NOT: @_ZZ4testvE7counter = linkonce_odr global i32 0, align 4 #0 +// COMMON: attributes #0 = { "toc-data" } diff --git a/clang/test/CodeGen/PowerPC/toc-data-diagnostics.c b/clang/test/CodeGen/PowerPC/toc-data-diagnostics.c new file mode 100644 index 000000000000..ba8955530e46 --- /dev/null +++ b/clang/test/CodeGen/PowerPC/toc-data-diagnostics.c @@ -0,0 +1,68 @@ +// RUN: %clang_cc1 %s -triple=powerpc-ibm-aix-xcoff -S -mtocdata=h,g,f,e,d,c,b,a,globalOneWithAlias,globalTwoWithAlias,ll,t3 -verify -emit-llvm -o - | FileCheck %s -check-prefix=CHECK --match-full-lines +// RUN: %clang_cc1 %s -triple=powerpc-ibm-aix-xcoff -S -mtocdata -verify=none -emit-llvm -o - | FileCheck %s -check-prefix=CHECK --match-full-lines + +// none-no-diagnostics + +struct large_struct { + int x; + short y; + short z; + char c; +}; + +struct large_struct a; // expected-warning {{-mtocdata option is ignored for a because variable is larger than a pointer}} +long long b = 5; // expected-warning {{-mtocdata option is ignored for b because variable is larger than a pointer}} +int __attribute__((aligned(128))) c = 0; // expected-warning {{-mtocdata option is ignored for c because variable is aligned wider than a pointer}} +double d = 2.5; // expected-warning {{-mtocdata option is ignored for d because variable is larger than a pointer}} +int e __attribute__((section("foo"))) = 10; // expected-warning {{-mtocdata option is ignored for e because variable has a section attribute}} +__thread int f; // expected-warning {{-mtocdata option is ignored for f because of thread local storage}} + +struct SomeStruct; +extern struct SomeStruct g; // expected-warning {{-mtocdata option is ignored for g because of incomplete type}} + +extern int h[]; // expected-warning {{-mtocdata option is ignored for h because of incomplete type}} + +struct ty3 { + int A; + char C[]; +}; +struct ty3 t3 = { 4, "fo" }; // expected-warning {{-mtocdata option is ignored for t3 because it contains a flexible array member}} + +int globalOneWithAlias = 10; +__attribute__((__alias__("globalOneWithAlias"))) extern int aliasOne; // expected-warning {{-mtocdata option is ignored for globalOneWithAlias because the variable has an alias}} +__attribute__((__alias__("globalTwoWithAlias"))) extern int aliasTwo; // expected-warning {{-mtocdata option is ignored for globalTwoWithAlias because the variable has an alias}} +int globalTwoWithAlias = 20; + + +int func() { + return a.x+b+c+d+e+f+h[0]; +} + +struct SomeStruct* getAddress(void) { + return &g; +} + +int test() { + return globalOneWithAlias + globalTwoWithAlias + aliasOne + aliasTwo; +} + +long long test2() { + static long long ll = 5; + ll++; + return ll; +} + +// CHECK: @b = global i64 5, align 8 +// CHECK: @c = global i32 0, align 128 +// CHECK: @d = global double 2.500000e+00, align 8 +// CHECK: @e = global i32 10, section "foo", align 4 +// CHECK: @globalOneWithAlias = global i32 10, align 4 +// CHECK: @globalTwoWithAlias = global i32 20, align 4 +// CHECK: @a = global %struct.large_struct zeroinitializer, align 4 +// CHECK: @f = thread_local global i32 0, align 4 +// CHECK: @h = external global [0 x i32], align 4 +// CHECK: @g = external global %struct.SomeStruct, align 1 +// CHECK: @test2.ll = internal global i64 5, align 8 +// CHECK: @aliasOne = alias i32, ptr @globalOneWithAlias +// CHECK: @aliasTwo = alias i32, ptr @globalTwoWithAlias +// CHECK-NOT: attributes #0 = { "toc-data" } diff --git a/clang/test/CodeGen/PowerPC/toc-data-structs-arrays.cpp b/clang/test/CodeGen/PowerPC/toc-data-structs-arrays.cpp new file mode 100644 index 000000000000..a717995cdceb --- /dev/null +++ b/clang/test/CodeGen/PowerPC/toc-data-structs-arrays.cpp @@ -0,0 +1,65 @@ +// RUN: %clang_cc1 %s -triple powerpc-ibm-aix-xcoff -S -mtocdata=a4,a5,a8,a9,b,c,d,e,v -emit-llvm -o - 2>&1 \ +// RUN: | FileCheck %s -check-prefixes=CHECK32 --match-full-lines +// RUN: %clang_cc1 %s -triple powerpc-ibm-aix-xcoff -S -mtocdata -emit-llvm -o - 2>&1 \ +// RUN: | FileCheck %s -check-prefixes=CHECK32 --match-full-lines + +// RUN: %clang_cc1 %s -triple powerpc64-ibm-aix-xcoff -S -mtocdata=a4,a5,a8,a9,b,c,d,e,v -emit-llvm -o - 2>&1 \ +// RUN: | FileCheck %s -check-prefixes=CHECK64 --match-full-lines +// RUN: %clang_cc1 %s -triple powerpc64-ibm-aix-xcoff -S -mtocdata -emit-llvm -o - 2>&1 \ +// RUN: | FileCheck %s -check-prefixes=CHECK64 --match-full-lines + +struct size4_struct { + int x; +}; + +struct size5_struct { + int x; + char c; +}; + +struct size8_struct { + int x; + short y; + short z; +}; + +struct size9_struct { + int x; + short y; + short z; + char c; +}; + +struct size4_struct a4; +struct size5_struct a5; +struct size8_struct a8; +struct size9_struct a9; + +short b[2]; +short c[3]; +short d[4]; +short e[5]; + +int func_a() { + return a4.x+a5.x+a8.x+a9.x+b[0]+c[0]+d[0]+e[0]; +} + +// CHECK32: @a4 = global %struct.size4_struct zeroinitializer, align 4 #0 +// CHECK32: @a5 = global %struct.size5_struct zeroinitializer, align 4 +// CHECK32: @a8 = global %struct.size8_struct zeroinitializer, align 4 +// CHECK32: @a9 = global %struct.size9_struct zeroinitializer, align 4 +// CHECK32: @b = global [2 x i16] zeroinitializer, align 2 #0 +// CHECK32: @c = global [3 x i16] zeroinitializer, align 2 +// CHECK32: @d = global [4 x i16] zeroinitializer, align 2 +// CHECK32: @e = global [5 x i16] zeroinitializer, align 2 +// CHECK32: attributes #0 = { "toc-data" } + +// CHECK64: @a4 = global %struct.size4_struct zeroinitializer, align 4 #0 +// CHECK64: @a5 = global %struct.size5_struct zeroinitializer, align 4 #0 +// CHECK64: @a8 = global %struct.size8_struct zeroinitializer, align 4 #0 +// CHECK64: @a9 = global %struct.size9_struct zeroinitializer, align 4 +// CHECK64: @b = global [2 x i16] zeroinitializer, align 2 #0 +// CHECK64: @c = global [3 x i16] zeroinitializer, align 2 #0 +// CHECK64: @d = global [4 x i16] zeroinitializer, align 2 #0 +// CHECK64: @e = global [5 x i16] zeroinitializer, align 2 +// CHECK64: attributes #0 = { "toc-data" } diff --git a/clang/test/Driver/toc-conf.c b/clang/test/Driver/toc-conf.c new file mode 100644 index 000000000000..80d92ee1a90b --- /dev/null +++ b/clang/test/Driver/toc-conf.c @@ -0,0 +1,30 @@ +// RUN: %clang %s --target=powerpc-unknown-aix -mno-tocdata -mtocdata -mno-tocdata -### 2>&1 | FileCheck %s -check-prefix=CHECK-FLAG1 +// RUN: %clang %s --target=powerpc-unknown-aix -mno-tocdata -mtocdata -mno-tocdata -mtocdata -### 2>&1 | FileCheck %s -check-prefix=CHECK-FLAG2 +// RUN: %clang %s --target=powerpc-unknown-aix -mtocdata=g1,g2 -mno-tocdata=g2 -mtocdata=g3,g4 -mno-tocdata=g5,g1 -### 2>&1 | FileCheck %s -check-prefix=CHECK-EQCONF +// RUN: %clang %s --target=powerpc-unknown-aix -mtocdata=g1 -mtocdata -mno-tocdata -mtocdata=g2,g3 -mno-tocdata=g4,g5,g3 -### 2>&1 | FileCheck %s -check-prefix=CHECK-CONF1 +// RUN: %clang %s --target=powerpc-unknown-aix -mno-tocdata=g1 -mno-tocdata -mtocdata -### 2>&1 | FileCheck %s -check-prefix=CHECK-CONF2 + +int g1, g4, g5; +extern int g2; +int g3 = 0; +void func() { + g2 = 0; +} + +// CHECK-FLAG1-NOT: warning: +// CHECK-FLAG1: "-cc1"{{.*}}" "-mno-tocdata" + +// CHECK-FLAG2-NOT: warning: +// CHECK-FLAG2: "-cc1"{{.*}}" "-mtocdata" + +// CHECK-EQCONF-NOT: warning: +// CHECK-EQCONF: "-cc1"{{.*}}" "-mno-tocdata" +// CHECK-EQCONF: "-mtocdata=g3,g4" + +// CHECK-CONF1-NOT: warning: +// CHECK-CONF1: "-cc1"{{.*}}" "-mno-tocdata" +// CHECK-CONF1: "-mtocdata=g2,g1" + +// CHECK-CONF2-NOT: warning: +// CHECK-CONF2: "-cc1"{{.*}}" "-mtocdata" +// CHECK-CONF2: "-mno-tocdata=g1" diff --git a/clang/test/Driver/tocdata-cc1.c b/clang/test/Driver/tocdata-cc1.c new file mode 100644 index 000000000000..fe0d97ea02db --- /dev/null +++ b/clang/test/Driver/tocdata-cc1.c @@ -0,0 +1,16 @@ +// RUN: %clang -### --target=powerpc-ibm-aix-xcoff -mcmodel=medium -mtocdata %s 2>&1 \ +// RUN: | FileCheck -check-prefix=CHECK-NOTOC %s +// RUN: %clang -### --target=powerpc-ibm-aix-xcoff -mcmodel=large -mtocdata %s 2>&1 \ +// RUN: | FileCheck -check-prefix=CHECK-NOTOC %s +// RUN: %clang -### --target=powerpc-ibm-aix-xcoff -mtocdata %s 2>&1 \ +// RUN: | FileCheck -check-prefix=CHECK-TOC %s +// RUN: %clang -### --target=powerpc64-ibm-aix-xcoff -mcmodel=medium -mtocdata %s 2>&1 \ +// RUN: | FileCheck -check-prefix=CHECK-NOTOC %s +// RUN: %clang -### --target=powerpc64-ibm-aix-xcoff -mcmodel=large -mtocdata %s 2>&1 \ +// RUN: | FileCheck -check-prefix=CHECK-NOTOC %s +// RUN: %clang -### --target=powerpc64-ibm-aix-xcoff -mtocdata %s 2>&1 \ +// RUN: | FileCheck -check-prefix=CHECK-TOC %s +// CHECK-NOTOC: warning: ignoring '-mtocdata' as it is only supported for -mcmodel=small +// CHECK-NOTOC-NOT: "-cc1"{{.*}}" "-mtocdata" +// CHECK-TOC: "-cc1"{{.*}}" "-mtocdata" +// CHECK-TOC-NOT: warning: ignoring '-mtocdata' as it is only supported for -mcmodel=small diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h index 5ac549c44756..02a3074ae1f0 100644 --- a/llvm/include/llvm/ADT/STLExtras.h +++ b/llvm/include/llvm/ADT/STLExtras.h @@ -1945,6 +1945,19 @@ auto partition(R &&Range, UnaryPredicate P) { return std::partition(adl_begin(Range), adl_end(Range), P); } +/// Provide wrappers to std::binary_search which take ranges instead of having +/// to pass begin/end explicitly. +template auto binary_search(R &&Range, T &&Value) { + return std::binary_search(adl_begin(Range), adl_end(Range), + std::forward(Value)); +} + +template +auto binary_search(R &&Range, T &&Value, Compare C) { + return std::binary_search(adl_begin(Range), adl_end(Range), + std::forward(Value), C); +} + /// Provide wrappers to std::lower_bound which take ranges instead of having to /// pass begin/end explicitly. template auto lower_bound(R &&Range, T &&Value) { diff --git a/llvm/lib/MC/MCSectionXCOFF.cpp b/llvm/lib/MC/MCSectionXCOFF.cpp index 95d32e3580e3..609ef0962930 100644 --- a/llvm/lib/MC/MCSectionXCOFF.cpp +++ b/llvm/lib/MC/MCSectionXCOFF.cpp @@ -87,8 +87,7 @@ void MCSectionXCOFF::printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, if (getKind().isCommon() && !getKind().isBSSLocal()) return; - assert((getKind().isBSSExtern() || getKind().isBSSLocal()) && - "Unexepected section kind for toc-data"); + assert(getKind().isBSS() && "Unexpected section kind for toc-data"); printCsectDirective(OS); return; } diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp index 9396ca22dacf..6f33b16f045a 100644 --- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -2659,6 +2659,8 @@ void PPCAIXAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) { // If the Global Variable has the toc-data attribute, it needs to be emitted // when we emit the .toc section. if (GV->hasAttribute("toc-data")) { + unsigned PointerSize = GV->getParent()->getDataLayout().getPointerSize(); + Subtarget->tocDataChecks(PointerSize, GV); TOCDataGlobalVars.push_back(GV); return; } diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp index 9e5f0b36616d..2462cbb19282 100644 --- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp @@ -521,40 +521,6 @@ static bool hasTocDataAttr(SDValue Val, unsigned PointerSize) { if (!GV->hasAttribute("toc-data")) return false; - - // TODO: These asserts should be updated as more support for the toc data - // transformation is added (struct support, etc.). - - assert( - PointerSize >= GV->getAlign().valueOrOne().value() && - "GlobalVariables with an alignment requirement stricter than TOC entry " - "size not supported by the toc data transformation."); - - Type *GVType = GV->getValueType(); - - assert(GVType->isSized() && "A GlobalVariable's size must be known to be " - "supported by the toc data transformation."); - - if (GVType->isVectorTy()) - report_fatal_error("A GlobalVariable of Vector type is not currently " - "supported by the toc data transformation."); - - if (GVType->isArrayTy()) - report_fatal_error("A GlobalVariable of Array type is not currently " - "supported by the toc data transformation."); - - if (GVType->isStructTy()) - report_fatal_error("A GlobalVariable of Struct type is not currently " - "supported by the toc data transformation."); - - assert(GVType->getPrimitiveSizeInBits() <= PointerSize * 8 && - "A GlobalVariable with size larger than a TOC entry is not currently " - "supported by the toc data transformation."); - - if (GV->hasPrivateLinkage()) - report_fatal_error("A GlobalVariable with private linkage is not " - "currently supported by the toc data transformation."); - return true; } diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp index 5380ec1c4c0d..884f2f5c57b2 100644 --- a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp +++ b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp @@ -185,6 +185,28 @@ bool PPCSubtarget::enableSubRegLiveness() const { return UseSubRegLiveness; } +void PPCSubtarget::tocDataChecks(unsigned PointerSize, + const GlobalVariable *GV) const { + // TODO: These asserts should be updated as more support for the toc data + // transformation is added (struct support, etc.). + assert( + PointerSize >= GV->getAlign().valueOrOne().value() && + "GlobalVariables with an alignment requirement stricter than TOC entry " + "size not supported by the toc data transformation."); + + Type *GVType = GV->getValueType(); + assert(GVType->isSized() && "A GlobalVariable's size must be known to be " + "supported by the toc data transformation."); + if (GV->getParent()->getDataLayout().getTypeSizeInBits(GVType) > + PointerSize * 8) + report_fatal_error( + "A GlobalVariable with size larger than a TOC entry is not currently " + "supported by the toc data transformation."); + if (GV->hasPrivateLinkage()) + report_fatal_error("A GlobalVariable with private linkage is not " + "currently supported by the toc data transformation."); +} + bool PPCSubtarget::isGVIndirectSymbol(const GlobalValue *GV) const { // Large code model always uses the TOC even for local symbols. if (TM.getCodeModel() == CodeModel::Large) diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.h b/llvm/lib/Target/PowerPC/PPCSubtarget.h index 306a52dca836..d913f22bd5ba 100644 --- a/llvm/lib/Target/PowerPC/PPCSubtarget.h +++ b/llvm/lib/Target/PowerPC/PPCSubtarget.h @@ -245,6 +245,8 @@ public: /// True if the GV will be accessed via an indirect symbol. bool isGVIndirectSymbol(const GlobalValue *GV) const; + void tocDataChecks(unsigned PointerSize, const GlobalVariable *GV) const; + /// True if the ABI is descriptor based. bool usesFunctionDescriptors() const { // Both 32-bit and 64-bit AIX are descriptor based. For ELF only the 64-bit diff --git a/llvm/test/CodeGen/PowerPC/toc-data-large-array.ll b/llvm/test/CodeGen/PowerPC/toc-data-large-array.ll new file mode 100644 index 000000000000..90f40d9f0fec --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/toc-data-large-array.ll @@ -0,0 +1,16 @@ +; RUN: not --crash llc -mtriple powerpc-ibm-aix-xcoff < %s 2>&1 | FileCheck %s --check-prefix CHECK-ERROR +; RUN: not --crash llc -mtriple powerpc64-ibm-aix-xcoff < %s 2>&1 | FileCheck %s --check-prefix CHECK-ERROR + +@a = global [5 x i16] zeroinitializer, align 2 #0 + +; Function Attrs: noinline +define i16 @foo() #1 { +entry: + %0 = load i16, ptr @a, align 2 + ret i16 %0 +} + +attributes #0 = { "toc-data" } +attributes #1 = { noinline } + +; CHECK-ERROR: LLVM ERROR: A GlobalVariable with size larger than a TOC entry is not currently supported by the toc data transformation. diff --git a/llvm/test/CodeGen/PowerPC/toc-data-large-array2.ll b/llvm/test/CodeGen/PowerPC/toc-data-large-array2.ll new file mode 100644 index 000000000000..f870e996a401 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/toc-data-large-array2.ll @@ -0,0 +1,8 @@ +; RUN: not --crash llc -mtriple powerpc-ibm-aix-xcoff < %s 2>&1 | FileCheck %s --check-prefix CHECK-ERROR +; RUN: not --crash llc -mtriple powerpc64-ibm-aix-xcoff < %s 2>&1 | FileCheck %s --check-prefix CHECK-ERROR + +@a = global [5 x i16] zeroinitializer, align 2 #0 + +attributes #0 = { "toc-data" } + +; CHECK-ERROR: LLVM ERROR: A GlobalVariable with size larger than a TOC entry is not currently supported by the toc data transformation. diff --git a/llvm/test/CodeGen/PowerPC/toc-data-struct-array.ll b/llvm/test/CodeGen/PowerPC/toc-data-struct-array.ll new file mode 100644 index 000000000000..a5c9a8b909d1 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/toc-data-struct-array.ll @@ -0,0 +1,110 @@ +; RUN: llc -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck %s --check-prefix CHECK +; RUN: llc -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s --check-prefix CHECK + +; RUN: llc -filetype=obj -mtriple powerpc-ibm-aix-xcoff < %s -o %t32.o +; RUN: llvm-readobj %t32.o --syms | FileCheck %s --check-prefix=OBJ32 +; RUN: llc -filetype=obj -mtriple powerpc64-ibm-aix-xcoff < %s -o %t64.o +; RUN: llvm-readobj %t64.o --syms | FileCheck %s --check-prefix=OBJ64 + +%struct.small_struct = type { i16 } + +@a = global %struct.small_struct zeroinitializer, align 2 #0 +@b = global [2 x i16] zeroinitializer, align 2 #0 + +; Function Attrs: noinline +define i16 @foo() #1 { +entry: + %0 = load i16, ptr @a, align 2 + %1 = load i16, ptr @b, align 2 + %add = add nsw i16 %0, %1 + ret i16 %add +} + +attributes #0 = { "toc-data" } +attributes #1 = { noinline } + +; CHECK: .toc +; CHECK-NEXT: .csect a[TD],2 +; CHECK-NEXT: .globl a[TD] # @a +; CHECK-NEXT: .align 1 +; CHECK-NEXT: .space 2 +; CHECK-NEXT: .csect b[TD],2 +; CHECK-NEXT: .globl b[TD] # @b +; CHECK-NEXT: .align 1 +; CHECK-NEXT: .space 4 + +; OBJ32: Symbol { +; OBJ32: Name: a +; OBJ32-NEXT: Value (RelocatableAddress): 0x3C +; OBJ32-NEXT: Section: .data +; OBJ32-NEXT: Type: 0x0 +; OBJ32-NEXT: StorageClass: C_EXT (0x2) +; OBJ32-NEXT: NumberOfAuxEntries: 1 +; OBJ32-NEXT: CSECT Auxiliary Entry { +; OBJ32-NEXT: Index: {{[0-9]+}} +; OBJ32-NEXT: SectionLen: 2 +; OBJ32-NEXT: ParameterHashIndex: 0x0 +; OBJ32-NEXT: TypeChkSectNum: 0x0 +; OBJ32-NEXT: SymbolAlignmentLog2: 2 +; OBJ32-NEXT: SymbolType: XTY_SD (0x1) +; OBJ32-NEXT: StorageMappingClass: XMC_TD (0x10) +; OBJ32-NEXT: StabInfoIndex: 0x0 +; OBJ32-NEXT: StabSectNum: 0x0 +; OBJ32-NEXT: } +; OBJ32-NEXT: } +; OBJ32-NEXT: Symbol { +; OBJ32: Name: b +; OBJ32-NEXT: Value (RelocatableAddress): 0x40 +; OBJ32-NEXT: Section: .data +; OBJ32-NEXT: Type: 0x0 +; OBJ32-NEXT: StorageClass: C_EXT (0x2) +; OBJ32-NEXT: NumberOfAuxEntries: 1 +; OBJ32-NEXT: CSECT Auxiliary Entry { +; OBJ32-NEXT: Index: {{[0-9]+}} +; OBJ32-NEXT: SectionLen: 4 +; OBJ32-NEXT: ParameterHashIndex: 0x0 +; OBJ32-NEXT: TypeChkSectNum: 0x0 +; OBJ32-NEXT: SymbolAlignmentLog2: 2 +; OBJ32-NEXT: SymbolType: XTY_SD (0x1) +; OBJ32-NEXT: StorageMappingClass: XMC_TD (0x10) +; OBJ32-NEXT: StabInfoIndex: 0x0 +; OBJ32-NEXT: StabSectNum: 0x0 +; OBJ32-NEXT: } +; OBJ32-NEXT: } + +; OBJ64: Symbol { +; OBJ64: Name: a +; OBJ64-NEXT: Value (RelocatableAddress): 0x48 +; OBJ64-NEXT: Section: .data +; OBJ64-NEXT: Type: 0x0 +; OBJ64-NEXT: StorageClass: C_EXT (0x2) +; OBJ64-NEXT: NumberOfAuxEntries: 1 +; OBJ64-NEXT: CSECT Auxiliary Entry { +; OBJ64-NEXT: Index: {{[0-9]+}} +; OBJ64-NEXT: SectionLen: 2 +; OBJ64-NEXT: ParameterHashIndex: 0x0 +; OBJ64-NEXT: TypeChkSectNum: 0x0 +; OBJ64-NEXT: SymbolAlignmentLog2: 2 +; OBJ64-NEXT: SymbolType: XTY_SD (0x1) +; OBJ64-NEXT: StorageMappingClass: XMC_TD (0x10) +; OBJ64-NEXT: Auxiliary Type: AUX_CSECT (0xFB) +; OBJ64-NEXT: } +; OBJ64-NEXT: } +; OBJ64-NEXT: Symbol { +; OBJ64: Name: b +; OBJ64-NEXT: Value (RelocatableAddress): 0x4C +; OBJ64-NEXT: Section: .data +; OBJ64-NEXT: Type: 0x0 +; OBJ64-NEXT: StorageClass: C_EXT (0x2) +; OBJ64-NEXT: NumberOfAuxEntries: 1 +; OBJ64-NEXT: CSECT Auxiliary Entry { +; OBJ64-NEXT: Index: {{[0-9]+}} +; OBJ64-NEXT: SectionLen: 4 +; OBJ64-NEXT: ParameterHashIndex: 0x0 +; OBJ64-NEXT: TypeChkSectNum: 0x0 +; OBJ64-NEXT: SymbolAlignmentLog2: 2 +; OBJ64-NEXT: SymbolType: XTY_SD (0x1) +; OBJ64-NEXT: StorageMappingClass: XMC_TD (0x10) +; OBJ64-NEXT: Auxiliary Type: AUX_CSECT (0xFB) +; OBJ64-NEXT: } +; OBJ64-NEXT: } -- GitLab From 1402c016ffe860446552a959e9fc4696c39392f9 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 13 Mar 2024 14:29:59 +0000 Subject: [PATCH 0127/1407] [VPlan] Use VPBuilder to create BranchOnCond in VPHCFGBuilder. This simplifies the code to create the recipe slightly as well as properly retaining the debug location of the input IR. --- .../Transforms/Vectorize/VPlanHCFGBuilder.cpp | 3 +- .../LoopVectorize/dbg-outer-loop-vect.ll | 40 +++++++++---------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp index 6474a9697dce..877b5d438115 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp @@ -296,8 +296,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, // recipes. if (Br->isConditional()) { VPValue *Cond = getOrCreateVPOperand(Br->getCondition()); - VPBB->appendRecipe( - new VPInstruction(VPInstruction::BranchOnCond, {Cond})); + VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst); } // Skip the rest of the Instruction processing for Branch instructions. diff --git a/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll b/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll index 7f209634fe05..2c665a417ab5 100644 --- a/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll +++ b/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll @@ -7,7 +7,7 @@ define void @foo(ptr %h) !dbg !4 { ; CHECK-LABEL: define void @foo( ; CHECK-SAME: ptr [[H:%.*]]) !dbg [[DBG4:![0-9]+]] { ; CHECK-NEXT: entry: -; CHECK-NEXT: call void @llvm.dbg.value(metadata i64 0, metadata [[META11:![0-9]+]], metadata !DIExpression()), !dbg [[DBG20:![0-9]+]] +; CHECK-NEXT: tail call void @llvm.dbg.value(metadata i64 0, metadata [[META11:![0-9]+]], metadata !DIExpression()), !dbg [[DBG20:![0-9]+]] ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !dbg [[DBG21:![0-9]+]] ; CHECK: vector.ph: ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]], !dbg [[DBG21]] @@ -27,15 +27,15 @@ define void @foo(ptr %h) !dbg !4 { ; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> , <4 x ptr> [[TMP3]], i32 4, <4 x i1> ), !dbg [[DBG22]] ; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], , !dbg [[DBG24:![0-9]+]] ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], , !dbg [[DBG25:![0-9]+]] -; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0 -; CHECK-NEXT: br i1 [[TMP6]], label [[FOR_COND_CLEANUP32]], label [[FOR_COND5_PREHEADER1]] +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0, !dbg [[DBG26:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP6]], label [[FOR_COND_CLEANUP32]], label [[FOR_COND5_PREHEADER1]], !dbg [[DBG26]] ; CHECK: for.cond.cleanup32: -; CHECK-NEXT: [[TMP7:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], , !dbg [[DBG26:![0-9]+]] -; CHECK-NEXT: [[TMP8:%.*]] = icmp eq <4 x i64> [[TMP7]], , !dbg [[DBG27:![0-9]+]] +; CHECK-NEXT: [[TMP7:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], , !dbg [[DBG27:![0-9]+]] +; CHECK-NEXT: [[TMP8:%.*]] = icmp eq <4 x i64> [[TMP7]], , !dbg [[DBG28:![0-9]+]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 -; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]], !dbg [[DBG21]] ; CHECK: scalar.ph: @@ -43,8 +43,8 @@ define void @foo(ptr %h) !dbg !4 { ; CHECK-NEXT: br label [[FOR_COND1_PREHEADER:%.*]], !dbg [[DBG21]] ; CHECK: for.cond1.preheader: ; CHECK-NEXT: [[I_023:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC13:%.*]], [[FOR_COND_CLEANUP3:%.*]] ] -; CHECK-NEXT: call void @llvm.dbg.value(metadata i64 [[I_023]], metadata [[META11]], metadata !DIExpression()), !dbg [[DBG20]] -; CHECK-NEXT: br label [[FOR_COND5_PREHEADER:%.*]], !dbg [[DBG32:![0-9]+]] +; CHECK-NEXT: tail call void @llvm.dbg.value(metadata i64 [[I_023]], metadata [[META11]], metadata !DIExpression()), !dbg [[DBG20]] +; CHECK-NEXT: br label [[FOR_COND5_PREHEADER:%.*]], !dbg [[DBG26]] ; CHECK: for.cond5.preheader: ; CHECK-NEXT: [[L_022:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER]] ], [ [[INC10:%.*]], [[FOR_COND5_PREHEADER]] ] ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[H]], i64 [[L_022]] @@ -57,11 +57,11 @@ define void @foo(ptr %h) !dbg !4 { ; CHECK-NEXT: store i32 3, ptr [[ARRAYIDX_3]], align 4, !dbg [[DBG22]] ; CHECK-NEXT: [[INC10]] = add nuw nsw i64 [[L_022]], 1, !dbg [[DBG24]] ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC10]], 5, !dbg [[DBG25]] -; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP3]], label [[FOR_COND5_PREHEADER]], !dbg [[DBG32]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP3]], label [[FOR_COND5_PREHEADER]], !dbg [[DBG26]] ; CHECK: for.cond.cleanup3: -; CHECK-NEXT: [[INC13]] = add nuw nsw i64 [[I_023]], 1, !dbg [[DBG26]] -; CHECK-NEXT: call void @llvm.dbg.value(metadata i64 [[INC13]], metadata [[META11]], metadata !DIExpression()), !dbg [[DBG20]] -; CHECK-NEXT: [[EXITCOND24_NOT:%.*]] = icmp eq i64 [[INC13]], 23, !dbg [[DBG27]] +; CHECK-NEXT: [[INC13]] = add nuw nsw i64 [[I_023]], 1, !dbg [[DBG27]] +; CHECK-NEXT: tail call void @llvm.dbg.value(metadata i64 [[INC13]], metadata [[META11]], metadata !DIExpression()), !dbg [[DBG20]] +; CHECK-NEXT: [[EXITCOND24_NOT:%.*]] = icmp eq i64 [[INC13]], 23, !dbg [[DBG28]] ; CHECK-NEXT: br i1 [[EXITCOND24_NOT]], label [[EXIT]], label [[FOR_COND1_PREHEADER]], !dbg [[DBG21]], !llvm.loop [[LOOP34:![0-9]+]] ; CHECK: exit: ; CHECK-NEXT: ret void, !dbg [[DBG35:![0-9]+]] @@ -163,14 +163,14 @@ declare void @llvm.dbg.value(metadata, metadata, metadata) ; CHECK: [[META23]] = distinct !DILexicalBlock(scope: [[META18]], file: [[META1]], line: 12, column: 7) ; CHECK: [[DBG24]] = !DILocation(line: 11, column: 32, scope: [[META19]]) ; CHECK: [[DBG25]] = !DILocation(line: 11, column: 26, scope: [[META19]]) -; CHECK: [[DBG26]] = !DILocation(line: 10, column: 30, scope: [[META16]]) -; CHECK: [[DBG27]] = !DILocation(line: 10, column: 24, scope: [[META16]]) -; CHECK: [[LOOP28]] = distinct !{[[LOOP28]], [[DBG21]], [[META29:![0-9]+]], [[META30:![0-9]+]], [[META31:![0-9]+]]} -; CHECK: [[META29]] = !DILocation(line: 13, column: 13, scope: [[META12]]) -; CHECK: [[META30]] = !{!"llvm.loop.isvectorized", i32 1} -; CHECK: [[META31]] = !{!"llvm.loop.unroll.runtime.disable"} -; CHECK: [[DBG32]] = !DILocation(line: 11, column: 5, scope: [[META15]]) +; CHECK: [[DBG26]] = !DILocation(line: 11, column: 5, scope: [[META15]]) +; CHECK: [[DBG27]] = !DILocation(line: 10, column: 30, scope: [[META16]]) +; CHECK: [[DBG28]] = !DILocation(line: 10, column: 24, scope: [[META16]]) +; CHECK: [[LOOP29]] = distinct !{[[LOOP29]], [[DBG21]], [[META30:![0-9]+]], [[META31:![0-9]+]], [[META32:![0-9]+]]} +; CHECK: [[META30]] = !DILocation(line: 13, column: 13, scope: [[META12]]) +; CHECK: [[META31]] = !{!"llvm.loop.isvectorized", i32 1} +; CHECK: [[META32]] = !{!"llvm.loop.unroll.runtime.disable"} ; CHECK: [[DBG33]] = !DILocation(line: 13, column: 2, scope: [[META23]]) -; CHECK: [[LOOP34]] = distinct !{[[LOOP34]], [[DBG21]], [[META29]], [[META30]]} +; CHECK: [[LOOP34]] = distinct !{[[LOOP34]], [[DBG21]], [[META30]], [[META31]]} ; CHECK: [[DBG35]] = !DILocation(line: 14, column: 1, scope: [[DBG4]]) ;. -- GitLab From 4e49ee55c587637e17dec7a72b9ce86d85f8f241 Mon Sep 17 00:00:00 2001 From: David Spickett Date: Wed, 13 Mar 2024 14:29:43 +0000 Subject: [PATCH 0128/1407] [lldb][Test] Disable ConcurrentVFork tests on Arm/AArch64 Linux They are either flaky, or not cleaning up after themselves. See https://github.com/llvm/llvm-project/issues/85084. --- .../fork/concurrent_vfork/TestConcurrentVFork.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lldb/test/API/functionalities/fork/concurrent_vfork/TestConcurrentVFork.py b/lldb/test/API/functionalities/fork/concurrent_vfork/TestConcurrentVFork.py index 2dcbb728549f..1790bd497f4e 100644 --- a/lldb/test/API/functionalities/fork/concurrent_vfork/TestConcurrentVFork.py +++ b/lldb/test/API/functionalities/fork/concurrent_vfork/TestConcurrentVFork.py @@ -48,6 +48,8 @@ class TestConcurrentVFork(TestBase): self.expect("continue", patterns=[r"exited with status = 1[0-4]"]) @skipUnlessPlatform(["linux"]) + # See https://github.com/llvm/llvm-project/issues/85084. + @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_parent_vfork_no_exec(self): """ Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-parent. @@ -56,6 +58,8 @@ class TestConcurrentVFork(TestBase): self.follow_parent_helper(use_fork=False, call_exec=False) @skipUnlessPlatform(["linux"]) + # See https://github.com/llvm/llvm-project/issues/85084. + @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_parent_fork_no_exec(self): """ Make sure that debugging concurrent fork() from multiple threads won't crash lldb during follow-parent. @@ -64,6 +68,8 @@ class TestConcurrentVFork(TestBase): self.follow_parent_helper(use_fork=True, call_exec=False) @skipUnlessPlatform(["linux"]) + # See https://github.com/llvm/llvm-project/issues/85084. + @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_parent_vfork_call_exec(self): """ Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-parent. @@ -72,6 +78,8 @@ class TestConcurrentVFork(TestBase): self.follow_parent_helper(use_fork=False, call_exec=True) @skipUnlessPlatform(["linux"]) + # See https://github.com/llvm/llvm-project/issues/85084. + @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_parent_fork_call_exec(self): """ Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-parent. @@ -80,6 +88,8 @@ class TestConcurrentVFork(TestBase): self.follow_parent_helper(use_fork=True, call_exec=True) @skipUnlessPlatform(["linux"]) + # See https://github.com/llvm/llvm-project/issues/85084. + @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_child_vfork_no_exec(self): """ Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-child. @@ -88,6 +98,8 @@ class TestConcurrentVFork(TestBase): self.follow_child_helper(use_fork=False, call_exec=False) @skipUnlessPlatform(["linux"]) + # See https://github.com/llvm/llvm-project/issues/85084. + @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_child_fork_no_exec(self): """ Make sure that debugging concurrent fork() from multiple threads won't crash lldb during follow-child. @@ -96,6 +108,8 @@ class TestConcurrentVFork(TestBase): self.follow_child_helper(use_fork=True, call_exec=False) @skipUnlessPlatform(["linux"]) + # See https://github.com/llvm/llvm-project/issues/85084. + @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_child_vfork_call_exec(self): """ Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-child. @@ -104,6 +118,8 @@ class TestConcurrentVFork(TestBase): self.follow_child_helper(use_fork=False, call_exec=True) @skipUnlessPlatform(["linux"]) + # See https://github.com/llvm/llvm-project/issues/85084. + @skipIf(oslist=["linux"], archs=["aarch64", "arm"]) def test_follow_child_fork_call_exec(self): """ Make sure that debugging concurrent fork() from multiple threads won't crash lldb during follow-child. -- GitLab From 63180ba444dc09fb9e85fdb98af56b2fc86f6027 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 13 Mar 2024 14:02:46 +0000 Subject: [PATCH 0129/1407] [DAG] Use SelectionDAG::getNOT helper where possible. NFC. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 87033e824aa5..40b078a201ac 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -2799,8 +2799,7 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) { // Limit this to after legalization if the add has wrap flags (Level >= AfterLegalizeDAG || (!N->getFlags().hasNoUnsignedWrap() && !N->getFlags().hasNoSignedWrap()))) { - SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), - DAG.getAllOnesConstant(DL, VT)); + SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT); return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not); } } @@ -3025,8 +3024,7 @@ SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1, // Limit this to after legalization if the add has wrap flags (Level >= AfterLegalizeDAG || (!N0->getFlags().hasNoUnsignedWrap() && !N0->getFlags().hasNoSignedWrap()))) { - SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), - DAG.getAllOnesConstant(DL, VT)); + SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT); return DAG.getNode(ISD::SUB, DL, VT, N1, Not); } -- GitLab From f18d78b477c76bc09dc580cdaedd55e121f5ebf5 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 13 Mar 2024 14:47:23 +0000 Subject: [PATCH 0130/1407] [DAG] isKnownToBeAPowerOfTwo - use sd_match to match both commutations of `x & -x` pattern`. NFC. Allows us to remove some tricky commutation matching --- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index c24303592769..b8c7d08da3e2 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -37,6 +37,7 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/RuntimeLibcalls.h" +#include "llvm/CodeGen/SDPatternMatch.h" #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h" #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/SelectionDAGTargetInfo.h" @@ -81,6 +82,7 @@ #include using namespace llvm; +using namespace llvm::SDPatternMatch; /// makeVTList - Return an instance of the SDVTList struct initialized with the /// specified members. @@ -4290,21 +4292,15 @@ bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const { return isKnownToBeAPowerOfTwo(Val.getOperand(2), Depth + 1) && isKnownToBeAPowerOfTwo(Val.getOperand(1), Depth + 1); - if (Val.getOpcode() == ISD::AND) { - // Looking for `x & -x` pattern: - // If x == 0: - // x & -x -> 0 - // If x != 0: - // x & -x -> non-zero pow2 - // so if we find the pattern return whether we know `x` is non-zero. - for (unsigned OpIdx = 0; OpIdx < 2; ++OpIdx) { - SDValue NegOp = Val.getOperand(OpIdx); - if (NegOp.getOpcode() == ISD::SUB && - NegOp.getOperand(1) == Val.getOperand(1 - OpIdx) && - isNullOrNullSplat(NegOp.getOperand(0))) - return isKnownNeverZero(Val.getOperand(1 - OpIdx), Depth); - } - } + // Looking for `x & -x` pattern: + // If x == 0: + // x & -x -> 0 + // If x != 0: + // x & -x -> non-zero pow2 + // so if we find the pattern return whether we know `x` is non-zero. + SDValue X; + if (sd_match(Val, m_And(m_Value(X), m_Sub(m_Zero(), m_Deferred(X))))) + return isKnownNeverZero(X, Depth); if (Val.getOpcode() == ISD::ZERO_EXTEND) return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1); -- GitLab From f46f5a01f4d5a7dcaf4a8fde5fc44eafdd9dbf27 Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Wed, 13 Mar 2024 14:51:09 +0000 Subject: [PATCH 0131/1407] [flang][OpenMP][OMPIRBuilder][mlir] Optionally pass reduction vars by ref (#84304) Previously reduction variables were always passed by value into and out of the initialization and combiner regions of the OpenMP reduction declare operation. This worked well for reductions of primitive types (and might perform better than passing by reference). But passing by reference will be useful for array and derived type reductions (e.g. to move allocation inside of the init region). Passing reductions by reference requires different LLVM-IR generation when lowering from MLIR because some of the loads/stores/allocations will now be moved inside of the init and combiner regions. This alternate code generation is requested using a new attribute to omp.wsloop and omp.parallel. Existing lowerings from mlir are unaffected (these will continue to use the by-value argument passing. Flang will continue to pass by-value argument passing for trivial types unless a (hidden) command line argument is supplied. Non-trivial types will always use the by-ref lowering. Array reductions are not ready yet (but are coming very soon). In the meantime, this is tested by forcing existing reductions to use by-ref. Commit series for by-ref OpenMP reductions 3/3 --------- Co-authored-by: Mats Petersson --- flang/lib/Lower/OpenMP/OpenMP.cpp | 18 +- flang/lib/Lower/OpenMP/ReductionProcessor.cpp | 137 ++++-- flang/lib/Lower/OpenMP/ReductionProcessor.h | 18 +- .../FIR/parallel-reduction-add-byref.f90 | 117 +++++ .../OpenMP/FIR/wsloop-reduction-add-byref.f90 | 392 ++++++++++++++++ .../FIR/wsloop-reduction-iand-byref.f90 | 46 ++ .../FIR/wsloop-reduction-ieor-byref.f90 | 45 ++ .../OpenMP/FIR/wsloop-reduction-ior-byref.f90 | 45 ++ .../wsloop-reduction-logical-eqv-byref.f90 | 187 ++++++++ .../wsloop-reduction-logical-neqv-byref.f90 | 189 ++++++++ .../OpenMP/FIR/wsloop-reduction-max-byref.f90 | 90 ++++ .../OpenMP/FIR/wsloop-reduction-min-byref.f90 | 91 ++++ .../Lower/OpenMP/default-clause-byref.f90 | 385 ++++++++++++++++ .../delayed-privatization-reduction-byref.f90 | 30 ++ .../OpenMP/parallel-reduction-add-byref.f90 | 125 +++++ .../Lower/OpenMP/parallel-reduction-byref.f90 | 44 ++ .../parallel-wsloop-reduction-byref.f90 | 16 + .../OpenMP/wsloop-reduction-add-byref.f90 | 433 ++++++++++++++++++ .../wsloop-reduction-add-hlfir-byref.f90 | 58 +++ .../OpenMP/wsloop-reduction-iand-byref.f90 | 64 +++ .../OpenMP/wsloop-reduction-ieor-byref.f90 | 55 +++ .../OpenMP/wsloop-reduction-ior-byref.f90 | 64 +++ .../wsloop-reduction-logical-and-byref.f90 | 206 +++++++++ .../wsloop-reduction-logical-eqv-byref.f90 | 202 ++++++++ .../wsloop-reduction-logical-neqv-byref.f90 | 207 +++++++++ .../wsloop-reduction-logical-or-byref.f90 | 204 +++++++++ .../OpenMP/wsloop-reduction-max-2-byref.f90 | 20 + .../OpenMP/wsloop-reduction-max-byref.f90 | 152 ++++++ .../wsloop-reduction-max-hlfir-byref.f90 | 62 +++ .../OpenMP/wsloop-reduction-min-byref.f90 | 154 +++++++ .../Lower/OpenMP/wsloop-reduction-min2.f90 | 41 ++ .../OpenMP/wsloop-reduction-mul-byref.f90 | 414 +++++++++++++++++ .../llvm/Frontend/OpenMP/OMPIRBuilder.h | 4 +- llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 30 +- mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 12 +- mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp | 5 +- .../OpenMP/OpenMPToLLVMIRTranslation.cpp | 99 ++-- .../Target/LLVMIR/openmp-reduction-byref.mlir | 66 +++ 38 files changed, 4451 insertions(+), 76 deletions(-) create mode 100644 flang/test/Lower/OpenMP/FIR/parallel-reduction-add-byref.f90 create mode 100644 flang/test/Lower/OpenMP/FIR/wsloop-reduction-add-byref.f90 create mode 100644 flang/test/Lower/OpenMP/FIR/wsloop-reduction-iand-byref.f90 create mode 100644 flang/test/Lower/OpenMP/FIR/wsloop-reduction-ieor-byref.f90 create mode 100644 flang/test/Lower/OpenMP/FIR/wsloop-reduction-ior-byref.f90 create mode 100644 flang/test/Lower/OpenMP/FIR/wsloop-reduction-logical-eqv-byref.f90 create mode 100644 flang/test/Lower/OpenMP/FIR/wsloop-reduction-logical-neqv-byref.f90 create mode 100644 flang/test/Lower/OpenMP/FIR/wsloop-reduction-max-byref.f90 create mode 100644 flang/test/Lower/OpenMP/FIR/wsloop-reduction-min-byref.f90 create mode 100644 flang/test/Lower/OpenMP/default-clause-byref.f90 create mode 100644 flang/test/Lower/OpenMP/delayed-privatization-reduction-byref.f90 create mode 100644 flang/test/Lower/OpenMP/parallel-reduction-add-byref.f90 create mode 100644 flang/test/Lower/OpenMP/parallel-reduction-byref.f90 create mode 100644 flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90 create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-byref.mlir diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index 4f0bb80cd7fd..1016c8389c6e 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -601,6 +601,10 @@ genParallelOp(Fortran::lower::AbstractConverter &converter, return reductionSymbols; }; + mlir::UnitAttr byrefAttr; + if (ReductionProcessor::doReductionByRef(reductionVars)) + byrefAttr = converter.getFirOpBuilder().getUnitAttr(); + OpWithBodyGenInfo genInfo = OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval) .setGenNested(genNested) @@ -620,7 +624,7 @@ genParallelOp(Fortran::lower::AbstractConverter &converter, : mlir::ArrayAttr::get(converter.getFirOpBuilder().getContext(), reductionDeclSymbols), procBindKindAttr, /*private_vars=*/llvm::SmallVector{}, - /*privatizers=*/nullptr); + /*privatizers=*/nullptr, byrefAttr); } bool privatize = !outerCombined; @@ -684,7 +688,8 @@ genParallelOp(Fortran::lower::AbstractConverter &converter, delayedPrivatizationInfo.privatizers.empty() ? nullptr : mlir::ArrayAttr::get(converter.getFirOpBuilder().getContext(), - privatizers)); + privatizers), + byrefAttr); } static mlir::omp::SectionOp @@ -1583,7 +1588,7 @@ static void createWsLoop(Fortran::lower::AbstractConverter &converter, llvm::SmallVector reductionSymbols; mlir::omp::ClauseOrderKindAttr orderClauseOperand; mlir::omp::ClauseScheduleKindAttr scheduleValClauseOperand; - mlir::UnitAttr nowaitClauseOperand, scheduleSimdClauseOperand; + mlir::UnitAttr nowaitClauseOperand, byrefOperand, scheduleSimdClauseOperand; mlir::IntegerAttr orderedClauseOperand; mlir::omp::ScheduleModifierAttr scheduleModClauseOperand; std::size_t loopVarTypeSize; @@ -1600,6 +1605,9 @@ static void createWsLoop(Fortran::lower::AbstractConverter &converter, convertLoopBounds(converter, loc, lowerBound, upperBound, step, loopVarTypeSize); + if (ReductionProcessor::doReductionByRef(reductionVars)) + byrefOperand = firOpBuilder.getUnitAttr(); + auto wsLoopOp = firOpBuilder.create( loc, lowerBound, upperBound, step, linearVars, linearStepVars, reductionVars, @@ -1609,8 +1617,8 @@ static void createWsLoop(Fortran::lower::AbstractConverter &converter, reductionDeclSymbols), scheduleValClauseOperand, scheduleChunkClauseOperand, /*schedule_modifiers=*/nullptr, - /*simd_modifier=*/nullptr, nowaitClauseOperand, orderedClauseOperand, - orderClauseOperand, + /*simd_modifier=*/nullptr, nowaitClauseOperand, byrefOperand, + orderedClauseOperand, orderClauseOperand, /*inclusive=*/firOpBuilder.getUnitAttr()); // Handle attribute based clauses. diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp index a8b98f3f5672..e6a63dd4b939 100644 --- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp +++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp @@ -14,9 +14,16 @@ #include "flang/Lower/AbstractConverter.h" #include "flang/Optimizer/Builder/Todo.h" +#include "flang/Optimizer/Dialect/FIRType.h" #include "flang/Optimizer/HLFIR/HLFIROps.h" #include "flang/Parser/tools.h" #include "mlir/Dialect/OpenMP/OpenMPDialect.h" +#include "llvm/Support/CommandLine.h" + +static llvm::cl::opt forceByrefReduction( + "force-byref-reduction", + llvm::cl::desc("Pass all reduction arguments by reference"), + llvm::cl::Hidden); namespace Fortran { namespace lower { @@ -76,16 +83,24 @@ bool ReductionProcessor::supportedIntrinsicProcReduction( } std::string ReductionProcessor::getReductionName(llvm::StringRef name, - mlir::Type ty) { + mlir::Type ty, bool isByRef) { + ty = fir::unwrapRefType(ty); + + // extra string to distinguish reduction functions for variables passed by + // reference + llvm::StringRef byrefAddition{""}; + if (isByRef) + byrefAddition = "_byref"; + return (llvm::Twine(name) + (ty.isIntOrIndex() ? llvm::Twine("_i_") : llvm::Twine("_f_")) + - llvm::Twine(ty.getIntOrFloatBitWidth())) + llvm::Twine(ty.getIntOrFloatBitWidth()) + byrefAddition) .str(); } std::string ReductionProcessor::getReductionName( Fortran::parser::DefinedOperator::IntrinsicOperator intrinsicOp, - mlir::Type ty) { + mlir::Type ty, bool isByRef) { std::string reductionName; switch (intrinsicOp) { @@ -108,13 +123,14 @@ std::string ReductionProcessor::getReductionName( break; } - return getReductionName(reductionName, ty); + return getReductionName(reductionName, ty, isByRef); } mlir::Value ReductionProcessor::getReductionInitValue(mlir::Location loc, mlir::Type type, ReductionIdentifier redId, fir::FirOpBuilder &builder) { + type = fir::unwrapRefType(type); assert((fir::isa_integer(type) || fir::isa_real(type) || type.isa()) && "only integer, logical and real types are currently supported"); @@ -188,6 +204,7 @@ mlir::Value ReductionProcessor::createScalarCombiner( fir::FirOpBuilder &builder, mlir::Location loc, ReductionIdentifier redId, mlir::Type type, mlir::Value op1, mlir::Value op2) { mlir::Value reductionOp; + type = fir::unwrapRefType(type); switch (redId) { case ReductionIdentifier::MAX: reductionOp = @@ -268,7 +285,8 @@ mlir::Value ReductionProcessor::createScalarCombiner( mlir::omp::ReductionDeclareOp ReductionProcessor::createReductionDecl( fir::FirOpBuilder &builder, llvm::StringRef reductionOpName, - const ReductionIdentifier redId, mlir::Type type, mlir::Location loc) { + const ReductionIdentifier redId, mlir::Type type, mlir::Location loc, + bool isByRef) { mlir::OpBuilder::InsertionGuard guard(builder); mlir::ModuleOp module = builder.getModule(); @@ -278,14 +296,24 @@ mlir::omp::ReductionDeclareOp ReductionProcessor::createReductionDecl( return decl; mlir::OpBuilder modBuilder(module.getBodyRegion()); + mlir::Type valTy = fir::unwrapRefType(type); + if (!isByRef) + type = valTy; decl = modBuilder.create(loc, reductionOpName, type); builder.createBlock(&decl.getInitializerRegion(), decl.getInitializerRegion().end(), {type}, {loc}); builder.setInsertionPointToEnd(&decl.getInitializerRegion().back()); + mlir::Value init = getReductionInitValue(loc, type, redId, builder); - builder.create(loc, init); + if (isByRef) { + mlir::Value alloca = builder.create(loc, valTy); + builder.createStoreWithConvert(loc, init, alloca); + builder.create(loc, alloca); + } else { + builder.create(loc, init); + } builder.createBlock(&decl.getReductionRegion(), decl.getReductionRegion().end(), {type, type}, @@ -294,14 +322,45 @@ mlir::omp::ReductionDeclareOp ReductionProcessor::createReductionDecl( builder.setInsertionPointToEnd(&decl.getReductionRegion().back()); mlir::Value op1 = decl.getReductionRegion().front().getArgument(0); mlir::Value op2 = decl.getReductionRegion().front().getArgument(1); + mlir::Value outAddr = op1; + + op1 = builder.loadIfRef(loc, op1); + op2 = builder.loadIfRef(loc, op2); mlir::Value reductionOp = createScalarCombiner(builder, loc, redId, type, op1, op2); - builder.create(loc, reductionOp); + if (isByRef) { + builder.create(loc, reductionOp, outAddr); + builder.create(loc, outAddr); + } else { + builder.create(loc, reductionOp); + } return decl; } +// TODO: By-ref vs by-val reductions are currently toggled for the whole +// operation (possibly effecting multiple reduction variables). +// This could cause a problem with openmp target reductions because +// by-ref trivial types may not be supported. +bool ReductionProcessor::doReductionByRef( + const llvm::SmallVectorImpl &reductionVars) { + if (reductionVars.empty()) + return false; + if (forceByrefReduction) + return true; + + for (mlir::Value reductionVar : reductionVars) { + if (auto declare = + mlir::dyn_cast(reductionVar.getDefiningOp())) + reductionVar = declare.getMemref(); + + if (!fir::isa_trivial(fir::unwrapRefType(reductionVar.getType()))) + return true; + } + return false; +} + void ReductionProcessor::addReductionDecl( mlir::Location currentLocation, Fortran::lower::AbstractConverter &converter, @@ -315,6 +374,37 @@ void ReductionProcessor::addReductionDecl( const auto &redOperator{ std::get(reduction.t)}; const auto &objectList{std::get(reduction.t)}; + + if (!std::holds_alternative( + redOperator.u)) { + if (const auto *reductionIntrinsic = + std::get_if(&redOperator.u)) { + if (!ReductionProcessor::supportedIntrinsicProcReduction( + *reductionIntrinsic)) { + return; + } + } else { + return; + } + } + + // initial pass to collect all recuction vars so we can figure out if this + // should happen byref + for (const Fortran::parser::OmpObject &ompObject : objectList.v) { + if (const auto *name{ + Fortran::parser::Unwrap(ompObject)}) { + if (const Fortran::semantics::Symbol * symbol{name->symbol}) { + if (reductionSymbols) + reductionSymbols->push_back(symbol); + mlir::Value symVal = converter.getSymbolAddress(*symbol); + if (auto declOp = symVal.getDefiningOp()) + symVal = declOp.getBase(); + reductionVars.push_back(symVal); + } + } + } + const bool isByRef = doReductionByRef(reductionVars); + if (const auto &redDefinedOp = std::get_if(&redOperator.u)) { const auto &intrinsicOp{ @@ -338,23 +428,20 @@ void ReductionProcessor::addReductionDecl( if (const auto *name{ Fortran::parser::Unwrap(ompObject)}) { if (const Fortran::semantics::Symbol * symbol{name->symbol}) { - if (reductionSymbols) - reductionSymbols->push_back(symbol); mlir::Value symVal = converter.getSymbolAddress(*symbol); if (auto declOp = symVal.getDefiningOp()) symVal = declOp.getBase(); - mlir::Type redType = - symVal.getType().cast().getEleTy(); - reductionVars.push_back(symVal); - if (redType.isa()) + auto redType = symVal.getType().cast(); + if (redType.getEleTy().isa()) decl = createReductionDecl( firOpBuilder, - getReductionName(intrinsicOp, firOpBuilder.getI1Type()), redId, - redType, currentLocation); - else if (redType.isIntOrIndexOrFloat()) { - decl = createReductionDecl(firOpBuilder, - getReductionName(intrinsicOp, redType), - redId, redType, currentLocation); + getReductionName(intrinsicOp, firOpBuilder.getI1Type(), + isByRef), + redId, redType, currentLocation, isByRef); + else if (redType.getEleTy().isIntOrIndexOrFloat()) { + decl = createReductionDecl( + firOpBuilder, getReductionName(intrinsicOp, redType, isByRef), + redId, redType, currentLocation, isByRef); } else { TODO(currentLocation, "Reduction of some types is not supported"); } @@ -374,21 +461,17 @@ void ReductionProcessor::addReductionDecl( if (const auto *name{ Fortran::parser::Unwrap(ompObject)}) { if (const Fortran::semantics::Symbol * symbol{name->symbol}) { - if (reductionSymbols) - reductionSymbols->push_back(symbol); mlir::Value symVal = converter.getSymbolAddress(*symbol); if (auto declOp = symVal.getDefiningOp()) symVal = declOp.getBase(); - mlir::Type redType = - symVal.getType().cast().getEleTy(); - reductionVars.push_back(symVal); - assert(redType.isIntOrIndexOrFloat() && + auto redType = symVal.getType().cast(); + assert(redType.getEleTy().isIntOrIndexOrFloat() && "Unsupported reduction type"); decl = createReductionDecl( firOpBuilder, getReductionName(getRealName(*reductionIntrinsic).ToString(), - redType), - redId, redType, currentLocation); + redType, isByRef), + redId, redType, currentLocation, isByRef); reductionDeclSymbols.push_back(mlir::SymbolRefAttr::get( firOpBuilder.getContext(), decl.getSymName())); } diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.h b/flang/lib/Lower/OpenMP/ReductionProcessor.h index 00770fe81d1e..679580f2a3ca 100644 --- a/flang/lib/Lower/OpenMP/ReductionProcessor.h +++ b/flang/lib/Lower/OpenMP/ReductionProcessor.h @@ -14,6 +14,7 @@ #define FORTRAN_LOWER_REDUCTIONPROCESSOR_H #include "flang/Optimizer/Builder/FIRBuilder.h" +#include "flang/Optimizer/Dialect/FIRType.h" #include "flang/Parser/parse-tree.h" #include "flang/Semantics/symbol.h" #include "flang/Semantics/type.h" @@ -71,11 +72,15 @@ public: static const Fortran::semantics::SourceName getRealName(const Fortran::parser::ProcedureDesignator &pd); - static std::string getReductionName(llvm::StringRef name, mlir::Type ty); + static bool + doReductionByRef(const llvm::SmallVectorImpl &reductionVars); + + static std::string getReductionName(llvm::StringRef name, mlir::Type ty, + bool isByRef); static std::string getReductionName( Fortran::parser::DefinedOperator::IntrinsicOperator intrinsicOp, - mlir::Type ty); + mlir::Type ty, bool isByRef); /// This function returns the identity value of the operator \p /// reductionOpName. For example: @@ -103,9 +108,11 @@ public: /// symbol table. The declaration has a constant initializer with the neutral /// value `initValue`, and the reduction combiner carried over from `reduce`. /// TODO: Generalize this for non-integer types, add atomic region. - static mlir::omp::ReductionDeclareOp createReductionDecl( - fir::FirOpBuilder &builder, llvm::StringRef reductionOpName, - const ReductionIdentifier redId, mlir::Type type, mlir::Location loc); + static mlir::omp::ReductionDeclareOp + createReductionDecl(fir::FirOpBuilder &builder, + llvm::StringRef reductionOpName, + const ReductionIdentifier redId, mlir::Type type, + mlir::Location loc, bool isByRef); /// Creates a reduction declaration and associates it with an OpenMP block /// directive. @@ -124,6 +131,7 @@ mlir::Value ReductionProcessor::getReductionOperation(fir::FirOpBuilder &builder, mlir::Type type, mlir::Location loc, mlir::Value op1, mlir::Value op2) { + type = fir::unwrapRefType(type); assert(type.isIntOrIndexOrFloat() && "only integer and float types are currently supported"); if (type.isIntOrIndex()) diff --git a/flang/test/Lower/OpenMP/FIR/parallel-reduction-add-byref.f90 b/flang/test/Lower/OpenMP/FIR/parallel-reduction-add-byref.f90 new file mode 100644 index 000000000000..ca432662b77c --- /dev/null +++ b/flang/test/Lower/OpenMP/FIR/parallel-reduction-add-byref.f90 @@ -0,0 +1,117 @@ +! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp -mmlir --force-byref-reduction -o - %s 2>&1 | FileCheck %s + +!CHECK-LABEL: omp.reduction.declare +!CHECK-SAME: @[[RED_F32_NAME:.*]] : !fir.ref +!CHECK-SAME: init { +!CHECK: ^bb0(%{{.*}}: !fir.ref): +!CHECK: %[[C0_1:.*]] = arith.constant 0.000000e+00 : f32 +!CHECK: %[[REF:.*]] = fir.alloca f32 +!CHECKL fir.store [[%C0_1]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) +!CHECK: } combiner { +!CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +!CHECK: %[[RES:.*]] = arith.addf %[[LD0]], %[[LD1]] {{.*}}: f32 +!CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +!CHECK: omp.yield(%[[ARG0]] : !fir.ref) +!CHECK: } + +!CHECK-LABEL: omp.reduction.declare +!CHECK-SAME: @[[RED_I32_NAME:.*]] : !fir.ref +!CHECK-SAME: init { +!CHECK: ^bb0(%{{.*}}: !fir.ref): +!CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 +!CHECK: %[[REF:.*]] = fir.alloca i32 +!CHECKL fir.store [[%C0_1]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) +!CHECK: } combiner { +!CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +!CHECK: %[[RES:.*]] = arith.addi %[[LD0]], %[[LD1]] : i32 +!CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +!CHECK: omp.yield(%[[ARG0]] : !fir.ref) +!CHECK: } + +!CHECK-LABEL: func.func @_QPsimple_int_add +!CHECK: %[[IREF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_int_addEi"} +!CHECK: %[[I_START:.*]] = arith.constant 0 : i32 +!CHECK: fir.store %[[I_START]] to %[[IREF]] : !fir.ref +!CHECK: omp.parallel byref reduction(@[[RED_I32_NAME]] %[[IREF]] -> %[[PRV:.+]] : !fir.ref) { +!CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref +!CHECK: %[[I_INCR:.+]] = arith.constant 1 : i32 +!CHECK: %[[RES:.+]] = arith.addi %[[LPRV]], %[[I_INCR]] +!CHECK: fir.store %[[RES]] to %[[PRV]] : !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: return +subroutine simple_int_add + integer :: i + i = 0 + + !$omp parallel reduction(+:i) + i = i + 1 + !$omp end parallel + + print *, i +end subroutine + +!CHECK-LABEL: func.func @_QPsimple_real_add +!CHECK: %[[RREF:.*]] = fir.alloca f32 {bindc_name = "r", uniq_name = "_QFsimple_real_addEr"} +!CHECK: %[[R_START:.*]] = arith.constant 0.000000e+00 : f32 +!CHECK: fir.store %[[R_START]] to %[[RREF]] : !fir.ref +!CHECK: omp.parallel byref reduction(@[[RED_F32_NAME]] %[[RREF]] -> %[[PRV:.+]] : !fir.ref) { +!CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref +!CHECK: %[[R_INCR:.+]] = arith.constant 1.500000e+00 : f32 +!CHECK: %[[RES]] = arith.addf %[[LPRV]], %[[R_INCR]] {{.*}} : f32 +!CHECK: fir.store %[[RES]] to %[[PRV]] : !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: return +subroutine simple_real_add + real :: r + r = 0.0 + + !$omp parallel reduction(+:r) + r = r + 1.5 + !$omp end parallel + + print *, r +end subroutine + +!CHECK-LABEL: func.func @_QPint_real_add +!CHECK: %[[IREF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFint_real_addEi"} +!CHECK: %[[RREF:.*]] = fir.alloca f32 {bindc_name = "r", uniq_name = "_QFint_real_addEr"} +!CHECK: %[[R_START:.*]] = arith.constant 0.000000e+00 : f32 +!CHECK: fir.store %[[R_START]] to %[[RREF]] : !fir.ref +!CHECK: %[[I_START:.*]] = arith.constant 0 : i32 +!CHECK: fir.store %[[I_START]] to %[[IREF]] : !fir.ref +!CHECK: omp.parallel byref reduction(@[[RED_I32_NAME]] %[[IREF]] -> %[[PRV0:.+]] : !fir.ref, @[[RED_F32_NAME]] %[[RREF]] -> %[[PRV1:.+]] : !fir.ref) { +!CHECK: %[[R_INCR:.*]] = arith.constant 1.500000e+00 : f32 +!CHECK: %[[LPRV1:.+]] = fir.load %[[PRV1]] : !fir.ref +!CHECK: %[[RES1:.+]] = arith.addf %[[R_INCR]], %[[LPRV1]] {{.*}} : f32 +!CHECK: fir.store %[[RES1]] to %[[PRV1]] +!CHECK: %[[LPRV0:.+]] = fir.load %[[PRV0]] : !fir.ref +!CHECK: %[[I_INCR:.*]] = arith.constant 3 : i32 +!CHECK: %[[RES0:.+]] = arith.addi %[[LPRV0]], %[[I_INCR]] +!CHECK: fir.store %[[RES0]] to %[[PRV0]] +!CHECK: omp.terminator +!CHECK: } +!CHECK: return +subroutine int_real_add + real :: r + integer :: i + + r = 0.0 + i = 0 + + !$omp parallel reduction(+:i,r) + r = 1.5 + r + i = i + 3 + !$omp end parallel + + print *, r + print *, i +end subroutine diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-add-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-add-byref.f90 new file mode 100644 index 000000000000..d4d3452d3e86 --- /dev/null +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-add-byref.f90 @@ -0,0 +1,392 @@ +! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + +! CHECK-LABEL: omp.reduction.declare @add_reduction_f_64_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant 0.000000e+00 : f64 +! CHECK: %[[REF:.*]] = fir.alloca f64 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.addf %[[LD0]], %[[LD1]] fastmath : f64 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: omp.reduction.declare @add_reduction_i_64_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant 0 : i64 +! CHECK: %[[REF:.*]] = fir.alloca i64 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.addi %[[LD0]], %[[LD1]] : i64 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: omp.reduction.declare @add_reduction_f_32_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: %[[REF:.*]] = fir.alloca f32 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.addf %[[LD0]], %[[LD1]] fastmath : f32 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: omp.reduction.declare @add_reduction_i_32_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 +! CHECK: %[[REF:.*]] = fir.alloca i32 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.addi %[[LD0]], %[[LD1]] : i32 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: func.func @_QPsimple_int_reduction() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_int_reductionEi"} +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFsimple_int_reductionEx"} +! CHECK: %[[VAL_2:.*]] = arith.constant 0 : i32 +! CHECK: fir.store %[[VAL_2]] to %[[VAL_1]] : !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_3:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_5:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { +! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref +! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_7]] : !fir.ref +! CHECK: %[[VAL_10:.*]] = fir.load %[[VAL_3]] : !fir.ref +! CHECK: %[[VAL_11:.*]] = arith.addi %[[VAL_9]], %[[VAL_10]] : i32 +! CHECK: fir.store %[[VAL_11]] to %[[VAL_7]] : !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + +subroutine simple_int_reduction + integer :: x + x = 0 + !$omp parallel + !$omp do reduction(+:x) + do i=1, 100 + x = x + i + end do + !$omp end do + !$omp end parallel +end subroutine + + +! CHECK-LABEL: func.func @_QPsimple_real_reduction() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_real_reductionEi"} +! CHECK: %[[VAL_1:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFsimple_real_reductionEx"} +! CHECK: %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: fir.store %[[VAL_2]] to %[[VAL_1]] : !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_3:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_5:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_f_32_byref %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { +! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref +! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_7]] : !fir.ref +! CHECK: %[[VAL_10:.*]] = fir.load %[[VAL_3]] : !fir.ref +! CHECK: %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (i32) -> f32 +! CHECK: %[[VAL_12:.*]] = arith.addf %[[VAL_9]], %[[VAL_11]] fastmath : f32 +! CHECK: fir.store %[[VAL_12]] to %[[VAL_7]] : !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + +subroutine simple_real_reduction + real :: x + x = 0.0 + !$omp parallel + !$omp do reduction(+:x) + do i=1, 100 + x = x + i + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPsimple_int_reduction_switch_order() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_int_reduction_switch_orderEi"} +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFsimple_int_reduction_switch_orderEx"} +! CHECK: %[[VAL_2:.*]] = arith.constant 0 : i32 +! CHECK: fir.store %[[VAL_2]] to %[[VAL_1]] : !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_3:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_5:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { +! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref +! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_3]] : !fir.ref +! CHECK: %[[VAL_10:.*]] = fir.load %[[VAL_7]] : !fir.ref +! CHECK: %[[VAL_11:.*]] = arith.addi %[[VAL_9]], %[[VAL_10]] : i32 +! CHECK: fir.store %[[VAL_11]] to %[[VAL_7]] : !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + +subroutine simple_int_reduction_switch_order + integer :: x + x = 0 + !$omp parallel + !$omp do reduction(+:x) + do i=1, 100 + x = i + x + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPsimple_real_reduction_switch_order() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_real_reduction_switch_orderEi"} +! CHECK: %[[VAL_1:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFsimple_real_reduction_switch_orderEx"} +! CHECK: %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: fir.store %[[VAL_2]] to %[[VAL_1]] : !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_3:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_5:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_f_32_byref %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { +! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref +! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_3]] : !fir.ref +! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> f32 +! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_7]] : !fir.ref +! CHECK: %[[VAL_12:.*]] = arith.addf %[[VAL_10]], %[[VAL_11]] fastmath : f32 +! CHECK: fir.store %[[VAL_12]] to %[[VAL_7]] : !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + +subroutine simple_real_reduction_switch_order + real :: x + x = 0.0 + !$omp parallel + !$omp do reduction(+:x) + do i=1, 100 + x = i + x + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_int_reductions_same_type() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_int_reductions_same_typeEi"} +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_int_reductions_same_typeEx"} +! CHECK: %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFmultiple_int_reductions_same_typeEy"} +! CHECK: %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFmultiple_int_reductions_same_typeEz"} +! CHECK: %[[VAL_4:.*]] = arith.constant 0 : i32 +! CHECK: fir.store %[[VAL_4]] to %[[VAL_1]] : !fir.ref +! CHECK: %[[VAL_5:.*]] = arith.constant 0 : i32 +! CHECK: fir.store %[[VAL_5]] to %[[VAL_2]] : !fir.ref +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i32 +! CHECK: fir.store %[[VAL_6]] to %[[VAL_3]] : !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_9:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_10:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_1]] -> %[[VAL_11:.*]] : !fir.ref, @add_reduction_i_32_byref %[[VAL_2]] -> %[[VAL_12:.*]] : !fir.ref, @add_reduction_i_32_byref %[[VAL_3]] -> %[[VAL_13:.*]] : !fir.ref) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { +! CHECK: fir.store %[[VAL_14]] to %[[VAL_7]] : !fir.ref +! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_11]] : !fir.ref +! CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_7]] : !fir.ref +! CHECK: %[[VAL_17:.*]] = arith.addi %[[VAL_15]], %[[VAL_16]] : i32 +! CHECK: fir.store %[[VAL_17]] to %[[VAL_11]] : !fir.ref +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_12]] : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_7]] : !fir.ref +! CHECK: %[[VAL_20:.*]] = arith.addi %[[VAL_18]], %[[VAL_19]] : i32 +! CHECK: fir.store %[[VAL_20]] to %[[VAL_12]] : !fir.ref +! CHECK: %[[VAL_21:.*]] = fir.load %[[VAL_13]] : !fir.ref +! CHECK: %[[VAL_22:.*]] = fir.load %[[VAL_7]] : !fir.ref +! CHECK: %[[VAL_23:.*]] = arith.addi %[[VAL_21]], %[[VAL_22]] : i32 +! CHECK: fir.store %[[VAL_23]] to %[[VAL_13]] : !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + +subroutine multiple_int_reductions_same_type + integer :: x,y,z + x = 0 + y = 0 + z = 0 + !$omp parallel + !$omp do reduction(+:x,y,z) + do i=1, 100 + x = x + i + y = y + i + z = z + i + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_real_reductions_same_type() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_real_reductions_same_typeEi"} +! CHECK: %[[VAL_1:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFmultiple_real_reductions_same_typeEx"} +! CHECK: %[[VAL_2:.*]] = fir.alloca f32 {bindc_name = "y", uniq_name = "_QFmultiple_real_reductions_same_typeEy"} +! CHECK: %[[VAL_3:.*]] = fir.alloca f32 {bindc_name = "z", uniq_name = "_QFmultiple_real_reductions_same_typeEz"} +! CHECK: %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: fir.store %[[VAL_4]] to %[[VAL_1]] : !fir.ref +! CHECK: %[[VAL_5:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: fir.store %[[VAL_5]] to %[[VAL_2]] : !fir.ref +! CHECK: %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: fir.store %[[VAL_6]] to %[[VAL_3]] : !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_9:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_10:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_f_32_byref %[[VAL_1]] -> %[[VAL_11:.*]] : !fir.ref, @add_reduction_f_32_byref %[[VAL_2]] -> %[[VAL_12:.*]] : !fir.ref, @add_reduction_f_32_byref %[[VAL_3]] -> %[[VAL_13:.*]] : !fir.ref) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { +! CHECK: fir.store %[[VAL_14]] to %[[VAL_7]] : !fir.ref +! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_11]] : !fir.ref +! CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_7]] : !fir.ref +! CHECK: %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i32) -> f32 +! CHECK: %[[VAL_18:.*]] = arith.addf %[[VAL_15]], %[[VAL_17]] fastmath : f32 +! CHECK: fir.store %[[VAL_18]] to %[[VAL_11]] : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_12]] : !fir.ref +! CHECK: %[[VAL_20:.*]] = fir.load %[[VAL_7]] : !fir.ref +! CHECK: %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (i32) -> f32 +! CHECK: %[[VAL_22:.*]] = arith.addf %[[VAL_19]], %[[VAL_21]] fastmath : f32 +! CHECK: fir.store %[[VAL_22]] to %[[VAL_12]] : !fir.ref +! CHECK: %[[VAL_23:.*]] = fir.load %[[VAL_13]] : !fir.ref +! CHECK: %[[VAL_24:.*]] = fir.load %[[VAL_7]] : !fir.ref +! CHECK: %[[VAL_25:.*]] = fir.convert %[[VAL_24]] : (i32) -> f32 +! CHECK: %[[VAL_26:.*]] = arith.addf %[[VAL_23]], %[[VAL_25]] fastmath : f32 +! CHECK: fir.store %[[VAL_26]] to %[[VAL_13]] : !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + +subroutine multiple_real_reductions_same_type + real :: x,y,z + x = 0.0 + y = 0.0 + z = 0.0 + !$omp parallel + !$omp do reduction(+:x,y,z) + do i=1, 100 + x = x + i + y = y + i + z = z + i + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_reductions_different_type() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_reductions_different_typeEi"} +! CHECK: %[[VAL_1:.*]] = fir.alloca f64 {bindc_name = "w", uniq_name = "_QFmultiple_reductions_different_typeEw"} +! CHECK: %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_reductions_different_typeEx"} +! CHECK: %[[VAL_3:.*]] = fir.alloca i64 {bindc_name = "y", uniq_name = "_QFmultiple_reductions_different_typeEy"} +! CHECK: %[[VAL_4:.*]] = fir.alloca f32 {bindc_name = "z", uniq_name = "_QFmultiple_reductions_different_typeEz"} +! CHECK: %[[VAL_5:.*]] = arith.constant 0 : i32 +! CHECK: fir.store %[[VAL_5]] to %[[VAL_2]] : !fir.ref +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i64 +! CHECK: fir.store %[[VAL_6]] to %[[VAL_3]] : !fir.ref +! CHECK: %[[VAL_7:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: fir.store %[[VAL_7]] to %[[VAL_4]] : !fir.ref +! CHECK: %[[VAL_8:.*]] = arith.constant 0.000000e+00 : f64 +! CHECK: fir.store %[[VAL_8]] to %[[VAL_1]] : !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_9:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_10:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_11:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_12:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_2]] -> %[[VAL_13:.*]] : !fir.ref, @add_reduction_i_64_byref %[[VAL_3]] -> %[[VAL_14:.*]] : !fir.ref, @add_reduction_f_32_byref %[[VAL_4]] -> %[[VAL_15:.*]] : !fir.ref, @add_reduction_f_64_byref %[[VAL_1]] -> %[[VAL_16:.*]] : !fir.ref) for (%[[VAL_17:.*]]) : i32 = (%[[VAL_10]]) to (%[[VAL_11]]) inclusive step (%[[VAL_12]]) { +! CHECK: fir.store %[[VAL_17]] to %[[VAL_9]] : !fir.ref +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_13]] : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_9]] : !fir.ref +! CHECK: %[[VAL_20:.*]] = arith.addi %[[VAL_18]], %[[VAL_19]] : i32 +! CHECK: fir.store %[[VAL_20]] to %[[VAL_13]] : !fir.ref +! CHECK: %[[VAL_21:.*]] = fir.load %[[VAL_14]] : !fir.ref +! CHECK: %[[VAL_22:.*]] = fir.load %[[VAL_9]] : !fir.ref +! CHECK: %[[VAL_23:.*]] = fir.convert %[[VAL_22]] : (i32) -> i64 +! CHECK: %[[VAL_24:.*]] = arith.addi %[[VAL_21]], %[[VAL_23]] : i64 +! CHECK: fir.store %[[VAL_24]] to %[[VAL_14]] : !fir.ref +! CHECK: %[[VAL_25:.*]] = fir.load %[[VAL_15]] : !fir.ref +! CHECK: %[[VAL_26:.*]] = fir.load %[[VAL_9]] : !fir.ref +! CHECK: %[[VAL_27:.*]] = fir.convert %[[VAL_26]] : (i32) -> f32 +! CHECK: %[[VAL_28:.*]] = arith.addf %[[VAL_25]], %[[VAL_27]] fastmath : f32 +! CHECK: fir.store %[[VAL_28]] to %[[VAL_15]] : !fir.ref +! CHECK: %[[VAL_29:.*]] = fir.load %[[VAL_16]] : !fir.ref +! CHECK: %[[VAL_30:.*]] = fir.load %[[VAL_9]] : !fir.ref +! CHECK: %[[VAL_31:.*]] = fir.convert %[[VAL_30]] : (i32) -> f64 +! CHECK: %[[VAL_32:.*]] = arith.addf %[[VAL_29]], %[[VAL_31]] fastmath : f64 +! CHECK: fir.store %[[VAL_32]] to %[[VAL_16]] : !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + + +subroutine multiple_reductions_different_type + integer :: x + integer(kind=8) :: y + real :: z + real(kind=8) :: w + x = 0 + y = 0 + z = 0.0 + w = 0.0 + !$omp parallel + !$omp do reduction(+:x,y,z,w) + do i=1, 100 + x = x + i + y = y + i + z = z + i + w = w + i + end do + !$omp end do + !$omp end parallel +end subroutine diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-iand-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-iand-byref.f90 new file mode 100644 index 000000000000..00dfad69248b --- /dev/null +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-iand-byref.f90 @@ -0,0 +1,46 @@ +! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + +!CHECK-LABEL: omp.reduction.declare @iand_i_32_byref : !fir.ref +!CHECK-SAME: init { +!CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +!CHECK: %[[C0_1:.*]] = arith.constant -1 : i32 +!CHECK: %[[REF:.*]] = fir.alloca i32 +!CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) + +!CHECK-LABEL: } combiner { +!CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +!CHECK: %[[RES:.*]] = arith.andi %[[LD0]], %[[LD1]] : i32 +!CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +!CHECK: omp.yield(%[[ARG0]] : !fir.ref) +!CHECK: } + + +!CHECK-LABEL: @_QPreduction_iand +!CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> +!CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_iandEx"} +!CHECK: omp.parallel +!CHECK: omp.wsloop byref reduction(@iand_i_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref +!CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] +!CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref +!CHECK: %[[RES:.+]] = arith.andi %[[LPRV]], %[[Y_I]] : i32 +!CHECK: fir.store %[[RES]] to %[[PRV]] : !fir.ref +!CHECK: omp.yield +!CHECK: omp.terminator + +subroutine reduction_iand(y) + integer :: x, y(:) + x = 0 + !$omp parallel + !$omp do reduction(iand:x) + do i=1, 100 + x = iand(x, y(i)) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ieor-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ieor-byref.f90 new file mode 100644 index 000000000000..fb35c405ca1d --- /dev/null +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ieor-byref.f90 @@ -0,0 +1,45 @@ +! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -mmlir --force-byref-reduction -fopenmp %s -o - | FileCheck %s + +! CHECK-LABEL: omp.reduction.declare @ieor_i_32_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 +! CHECK: %[[REF:.*]] = fir.alloca i32 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.xori %[[LD0]], %[[LD1]] : i32 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +!CHECK-LABEL: @_QPreduction_ieor +!CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> +!CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_ieorEx"} +!CHECK: omp.parallel +!CHECK: omp.wsloop byref reduction(@ieor_i_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref +!CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] +!CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref +!CHECK: %[[RES:.+]] = arith.xori %[[LPRV]], %[[Y_I]] : i32 +!CHECK: fir.store %[[RES]] to %[[PRV]] : !fir.ref +!CHECK: omp.yield +!CHECK: omp.terminator + +subroutine reduction_ieor(y) + integer :: x, y(:) + x = 0 + !$omp parallel + !$omp do reduction(ieor:x) + do i=1, 100 + x = ieor(x, y(i)) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ior-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ior-byref.f90 new file mode 100644 index 000000000000..0365eff3fd81 --- /dev/null +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ior-byref.f90 @@ -0,0 +1,45 @@ +! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + +! CHECK-LABEL: omp.reduction.declare @ior_i_32_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 +! CHECK: %[[REF:.*]] = fir.alloca i32 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.ori %[[LD0]], %[[LD1]] : i32 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +!CHECK-LABEL: @_QPreduction_ior +!CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> +!CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_iorEx"} +!CHECK: omp.parallel +!CHECK: omp.wsloop byref reduction(@ior_i_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref +!CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] +!CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref +!CHECK: %[[RES:.+]] = arith.ori %[[LPRV]], %[[Y_I]] : i32 +!CHECK: fir.store %[[RES]] to %[[PRV]] : !fir.ref +!CHECK: omp.yield +!CHECK: omp.terminator + +subroutine reduction_ior(y) + integer :: x, y(:) + x = 0 + !$omp parallel + !$omp do reduction(ior:x) + do i=1, 100 + x = ior(x, y(i)) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-logical-eqv-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-logical-eqv-byref.f90 new file mode 100644 index 000000000000..84219b34f964 --- /dev/null +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-logical-eqv-byref.f90 @@ -0,0 +1,187 @@ +! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + +! CHECK-LABEL: omp.reduction.declare @eqv_reduction : !fir.ref> +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref>): +! CHECK: %[[VAL_1:.*]] = arith.constant true +! CHECK: %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4> +! CHECK: %[[REF:.*]] = fir.alloca !fir.logical<4> +! CHECK: fir.store %[[VAL_2]] to %[[REF]] : !fir.ref> +! CHECK: omp.yield(%[[REF]] : !fir.ref>) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref> +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref> +! CHECK: %[[VAL_2:.*]] = fir.convert %[[LD0]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_3:.*]] = fir.convert %[[LD1]] : (!fir.logical<4>) -> i1 +! CHECK: %[[RES:.*]] = arith.cmpi eq, %[[VAL_2]], %[[VAL_3]] : i1 +! CHECK: %[[VAL_5:.*]] = fir.convert %[[RES]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_5]] to %[[ARG0]] : !fir.ref> +! CHECK: omp.yield(%[[ARG0]] : !fir.ref>) +! CHECK: } + +! CHECK-LABEL: func.func @_QPsimple_reduction( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_reductionEi"} +! CHECK: %[[VAL_2:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFsimple_reductionEx"} +! CHECK: %[[VAL_3:.*]] = arith.constant true +! CHECK: %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_4]] to %[[VAL_2]] : !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_7:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@eqv_reduction %[[VAL_2]] -> %[[VAL_9:.*]] : !fir.ref>) for (%[[VAL_10:.*]]) : i32 = (%[[VAL_6]]) to (%[[VAL_7]]) inclusive step (%[[VAL_8]]) { +! CHECK: fir.store %[[VAL_10]] to %[[VAL_5]] : !fir.ref +! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref> +! CHECK: %[[VAL_12:.*]] = fir.load %[[VAL_5]] : !fir.ref +! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i32) -> i64 +! CHECK: %[[VAL_14:.*]] = arith.constant 1 : i64 +! CHECK: %[[VAL_15:.*]] = arith.subi %[[VAL_13]], %[[VAL_14]] : i64 +! CHECK: %[[VAL_16:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_15]] : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_17:.*]] = fir.load %[[VAL_16]] : !fir.ref> +! CHECK: %[[VAL_18:.*]] = fir.convert %[[VAL_11]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_19:.*]] = fir.convert %[[VAL_17]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_20:.*]] = arith.cmpi eq, %[[VAL_18]], %[[VAL_19]] : i1 +! CHECK: %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_21]] to %[[VAL_9]] : !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_reduction(y) + logical :: x, y(100) + x = .true. + !$omp parallel + !$omp do reduction(.eqv.:x) + do i=1, 100 + x = x .eqv. y(i) + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPsimple_reduction_switch_order( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_reduction_switch_orderEi"} +! CHECK: %[[VAL_2:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFsimple_reduction_switch_orderEx"} +! CHECK: %[[VAL_3:.*]] = arith.constant true +! CHECK: %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_4]] to %[[VAL_2]] : !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_7:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@eqv_reduction %[[VAL_2]] -> %[[VAL_9:.*]] : !fir.ref>) for (%[[VAL_10:.*]]) : i32 = (%[[VAL_6]]) to (%[[VAL_7]]) inclusive step (%[[VAL_8]]) { +! CHECK: fir.store %[[VAL_10]] to %[[VAL_5]] : !fir.ref +! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_5]] : !fir.ref +! CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i32) -> i64 +! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i64 +! CHECK: %[[VAL_14:.*]] = arith.subi %[[VAL_12]], %[[VAL_13]] : i64 +! CHECK: %[[VAL_15:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_14]] : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_15]] : !fir.ref> +! CHECK: %[[VAL_17:.*]] = fir.load %[[VAL_9]] : !fir.ref> +! CHECK: %[[VAL_18:.*]] = fir.convert %[[VAL_16]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_19:.*]] = fir.convert %[[VAL_17]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_20:.*]] = arith.cmpi eq, %[[VAL_18]], %[[VAL_19]] : i1 +! CHECK: %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_21]] to %[[VAL_9]] : !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_reduction_switch_order(y) + logical :: x, y(100) + x = .true. + !$omp parallel + !$omp do reduction(.eqv.:x) + do i=1, 100 + x = y(i) .eqv. x + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_reductions( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "w"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_reductionsEi"} +! CHECK: %[[VAL_2:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"} +! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"} +! CHECK: %[[VAL_4:.*]] = fir.alloca !fir.logical<4> {bindc_name = "z", uniq_name = "_QFmultiple_reductionsEz"} +! CHECK: %[[VAL_5:.*]] = arith.constant true +! CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_6]] to %[[VAL_2]] : !fir.ref> +! CHECK: %[[VAL_7:.*]] = arith.constant true +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref> +! CHECK: %[[VAL_9:.*]] = arith.constant true +! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_10]] to %[[VAL_4]] : !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_12:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_13:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@eqv_reduction %[[VAL_2]] -> %[[VAL_15:.*]] : !fir.ref>, @eqv_reduction %[[VAL_3]] -> %[[VAL_16:.*]] : !fir.ref>, @eqv_reduction %[[VAL_4]] -> %[[VAL_17:.*]] : !fir.ref>) for (%[[VAL_18:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) { +! CHECK: fir.store %[[VAL_18]] to %[[VAL_11]] : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_15]] : !fir.ref> +! CHECK: %[[VAL_20:.*]] = fir.load %[[VAL_11]] : !fir.ref +! CHECK: %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (i32) -> i64 +! CHECK: %[[VAL_22:.*]] = arith.constant 1 : i64 +! CHECK: %[[VAL_23:.*]] = arith.subi %[[VAL_21]], %[[VAL_22]] : i64 +! CHECK: %[[VAL_24:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_23]] : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_25:.*]] = fir.load %[[VAL_24]] : !fir.ref> +! CHECK: %[[VAL_26:.*]] = fir.convert %[[VAL_19]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_27:.*]] = fir.convert %[[VAL_25]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_28:.*]] = arith.cmpi eq, %[[VAL_26]], %[[VAL_27]] : i1 +! CHECK: %[[VAL_29:.*]] = fir.convert %[[VAL_28]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_29]] to %[[VAL_15]] : !fir.ref> +! CHECK: %[[VAL_30:.*]] = fir.load %[[VAL_16]] : !fir.ref> +! CHECK: %[[VAL_31:.*]] = fir.load %[[VAL_11]] : !fir.ref +! CHECK: %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (i32) -> i64 +! CHECK: %[[VAL_33:.*]] = arith.constant 1 : i64 +! CHECK: %[[VAL_34:.*]] = arith.subi %[[VAL_32]], %[[VAL_33]] : i64 +! CHECK: %[[VAL_35:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_34]] : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_36:.*]] = fir.load %[[VAL_35]] : !fir.ref> +! CHECK: %[[VAL_37:.*]] = fir.convert %[[VAL_30]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_38:.*]] = fir.convert %[[VAL_36]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_39:.*]] = arith.cmpi eq, %[[VAL_37]], %[[VAL_38]] : i1 +! CHECK: %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_40]] to %[[VAL_16]] : !fir.ref> +! CHECK: %[[VAL_41:.*]] = fir.load %[[VAL_17]] : !fir.ref> +! CHECK: %[[VAL_42:.*]] = fir.load %[[VAL_11]] : !fir.ref +! CHECK: %[[VAL_43:.*]] = fir.convert %[[VAL_42]] : (i32) -> i64 +! CHECK: %[[VAL_44:.*]] = arith.constant 1 : i64 +! CHECK: %[[VAL_45:.*]] = arith.subi %[[VAL_43]], %[[VAL_44]] : i64 +! CHECK: %[[VAL_46:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_45]] : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref> +! CHECK: %[[VAL_48:.*]] = fir.convert %[[VAL_41]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_49:.*]] = fir.convert %[[VAL_47]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_50:.*]] = arith.cmpi eq, %[[VAL_48]], %[[VAL_49]] : i1 +! CHECK: %[[VAL_51:.*]] = fir.convert %[[VAL_50]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_51]] to %[[VAL_17]] : !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine multiple_reductions(w) + logical :: x,y,z,w(100) + x = .true. + y = .true. + z = .true. + !$omp parallel + !$omp do reduction(.eqv.:x,y,z) + do i=1, 100 + x = x .eqv. w(i) + y = y .eqv. w(i) + z = z .eqv. w(i) + end do + !$omp end do + !$omp end parallel +end subroutine diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-logical-neqv-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-logical-neqv-byref.f90 new file mode 100644 index 000000000000..ec4d8500850b --- /dev/null +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-logical-neqv-byref.f90 @@ -0,0 +1,189 @@ +! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + + +! CHECK-LABEL: omp.reduction.declare @neqv_reduction : !fir.ref> +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref>): +! CHECK: %[[VAL_1:.*]] = arith.constant false +! CHECK: %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4> +! CHECK: %[[REF:.*]] = fir.alloca !fir.logical<4> +! CHECK: fir.store %[[VAL_2]] to %[[REF]] : !fir.ref> +! CHECK: omp.yield(%[[REF]] : !fir.ref>) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref> +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref> +! CHECK: %[[VAL_2:.*]] = fir.convert %[[LD0]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_3:.*]] = fir.convert %[[LD1]] : (!fir.logical<4>) -> i1 +! CHECK: %[[RES:.*]] = arith.cmpi ne, %[[VAL_2]], %[[VAL_3]] : i1 +! CHECK: %[[VAL_5:.*]] = fir.convert %[[RES]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_5]] to %[[ARG0]] : !fir.ref> +! CHECK: omp.yield(%[[ARG0]] : !fir.ref>) +! CHECK: } + +! CHECK-LABEL: func.func @_QPsimple_reduction( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_reductionEi"} +! CHECK: %[[VAL_2:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFsimple_reductionEx"} +! CHECK: %[[VAL_3:.*]] = arith.constant true +! CHECK: %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_4]] to %[[VAL_2]] : !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_7:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@neqv_reduction %[[VAL_2]] -> %[[VAL_9:.*]] : !fir.ref>) for (%[[VAL_10:.*]]) : i32 = (%[[VAL_6]]) to (%[[VAL_7]]) inclusive step (%[[VAL_8]]) { +! CHECK: fir.store %[[VAL_10]] to %[[VAL_5]] : !fir.ref +! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref> +! CHECK: %[[VAL_12:.*]] = fir.load %[[VAL_5]] : !fir.ref +! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i32) -> i64 +! CHECK: %[[VAL_14:.*]] = arith.constant 1 : i64 +! CHECK: %[[VAL_15:.*]] = arith.subi %[[VAL_13]], %[[VAL_14]] : i64 +! CHECK: %[[VAL_16:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_15]] : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_17:.*]] = fir.load %[[VAL_16]] : !fir.ref> +! CHECK: %[[VAL_18:.*]] = fir.convert %[[VAL_11]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_19:.*]] = fir.convert %[[VAL_17]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_20:.*]] = arith.cmpi ne, %[[VAL_18]], %[[VAL_19]] : i1 +! CHECK: %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_21]] to %[[VAL_9]] : !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_reduction(y) + logical :: x, y(100) + x = .true. + !$omp parallel + !$omp do reduction(.neqv.:x) + do i=1, 100 + x = x .neqv. y(i) + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPsimple_reduction_switch_order( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_reduction_switch_orderEi"} +! CHECK: %[[VAL_2:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFsimple_reduction_switch_orderEx"} +! CHECK: %[[VAL_3:.*]] = arith.constant true +! CHECK: %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_4]] to %[[VAL_2]] : !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_7:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@neqv_reduction %[[VAL_2]] -> %[[VAL_9:.*]] : !fir.ref>) for (%[[VAL_10:.*]]) : i32 = (%[[VAL_6]]) to (%[[VAL_7]]) inclusive step (%[[VAL_8]]) { +! CHECK: fir.store %[[VAL_10]] to %[[VAL_5]] : !fir.ref +! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_5]] : !fir.ref +! CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i32) -> i64 +! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i64 +! CHECK: %[[VAL_14:.*]] = arith.subi %[[VAL_12]], %[[VAL_13]] : i64 +! CHECK: %[[VAL_15:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_14]] : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_15]] : !fir.ref> +! CHECK: %[[VAL_17:.*]] = fir.load %[[VAL_9]] : !fir.ref> +! CHECK: %[[VAL_18:.*]] = fir.convert %[[VAL_16]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_19:.*]] = fir.convert %[[VAL_17]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_20:.*]] = arith.cmpi ne, %[[VAL_18]], %[[VAL_19]] : i1 +! CHECK: %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_21]] to %[[VAL_9]] : !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_reduction_switch_order(y) + logical :: x, y(100) + x = .true. + !$omp parallel + !$omp do reduction(.neqv.:x) + do i=1, 100 + x = y(i) .neqv. x + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_reductions( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "w"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_reductionsEi"} +! CHECK: %[[VAL_2:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"} +! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"} +! CHECK: %[[VAL_4:.*]] = fir.alloca !fir.logical<4> {bindc_name = "z", uniq_name = "_QFmultiple_reductionsEz"} +! CHECK: %[[VAL_5:.*]] = arith.constant true +! CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_6]] to %[[VAL_2]] : !fir.ref> +! CHECK: %[[VAL_7:.*]] = arith.constant true +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref> +! CHECK: %[[VAL_9:.*]] = arith.constant true +! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_10]] to %[[VAL_4]] : !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_12:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_13:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@neqv_reduction %[[VAL_2]] -> %[[VAL_15:.*]] : !fir.ref>, @neqv_reduction %[[VAL_3]] -> %[[VAL_16:.*]] : !fir.ref>, @neqv_reduction %[[VAL_4]] -> %[[VAL_17:.*]] : !fir.ref>) for (%[[VAL_18:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) { +! CHECK: fir.store %[[VAL_18]] to %[[VAL_11]] : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_15]] : !fir.ref> +! CHECK: %[[VAL_20:.*]] = fir.load %[[VAL_11]] : !fir.ref +! CHECK: %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (i32) -> i64 +! CHECK: %[[VAL_22:.*]] = arith.constant 1 : i64 +! CHECK: %[[VAL_23:.*]] = arith.subi %[[VAL_21]], %[[VAL_22]] : i64 +! CHECK: %[[VAL_24:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_23]] : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_25:.*]] = fir.load %[[VAL_24]] : !fir.ref> +! CHECK: %[[VAL_26:.*]] = fir.convert %[[VAL_19]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_27:.*]] = fir.convert %[[VAL_25]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_28:.*]] = arith.cmpi ne, %[[VAL_26]], %[[VAL_27]] : i1 +! CHECK: %[[VAL_29:.*]] = fir.convert %[[VAL_28]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_29]] to %[[VAL_15]] : !fir.ref> +! CHECK: %[[VAL_30:.*]] = fir.load %[[VAL_16]] : !fir.ref> +! CHECK: %[[VAL_31:.*]] = fir.load %[[VAL_11]] : !fir.ref +! CHECK: %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (i32) -> i64 +! CHECK: %[[VAL_33:.*]] = arith.constant 1 : i64 +! CHECK: %[[VAL_34:.*]] = arith.subi %[[VAL_32]], %[[VAL_33]] : i64 +! CHECK: %[[VAL_35:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_34]] : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_36:.*]] = fir.load %[[VAL_35]] : !fir.ref> +! CHECK: %[[VAL_37:.*]] = fir.convert %[[VAL_30]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_38:.*]] = fir.convert %[[VAL_36]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_39:.*]] = arith.cmpi ne, %[[VAL_37]], %[[VAL_38]] : i1 +! CHECK: %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_40]] to %[[VAL_16]] : !fir.ref> +! CHECK: %[[VAL_41:.*]] = fir.load %[[VAL_17]] : !fir.ref> +! CHECK: %[[VAL_42:.*]] = fir.load %[[VAL_11]] : !fir.ref +! CHECK: %[[VAL_43:.*]] = fir.convert %[[VAL_42]] : (i32) -> i64 +! CHECK: %[[VAL_44:.*]] = arith.constant 1 : i64 +! CHECK: %[[VAL_45:.*]] = arith.subi %[[VAL_43]], %[[VAL_44]] : i64 +! CHECK: %[[VAL_46:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_45]] : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref> +! CHECK: %[[VAL_48:.*]] = fir.convert %[[VAL_41]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_49:.*]] = fir.convert %[[VAL_47]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_50:.*]] = arith.cmpi ne, %[[VAL_48]], %[[VAL_49]] : i1 +! CHECK: %[[VAL_51:.*]] = fir.convert %[[VAL_50]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_51]] to %[[VAL_17]] : !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + + +subroutine multiple_reductions(w) + logical :: x,y,z,w(100) + x = .true. + y = .true. + z = .true. + !$omp parallel + !$omp do reduction(.neqv.:x,y,z) + do i=1, 100 + x = x .neqv. w(i) + y = y .neqv. w(i) + z = z .neqv. w(i) + end do + !$omp end do + !$omp end parallel +end subroutine diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-max-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-max-byref.f90 new file mode 100644 index 000000000000..ddda24a17a83 --- /dev/null +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-max-byref.f90 @@ -0,0 +1,90 @@ +! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp -mmlir --force-byref-reduction -o - %s 2>&1 | FileCheck %s + +!CHECK: omp.reduction.declare @max_f_32_byref : !fir.ref +!CHECK-SAME: init { +!CHECK: %[[MINIMUM_VAL:.*]] = arith.constant -3.40282347E+38 : f32 +!CHECK: %[[REF:.*]] = fir.alloca f32 +!CHECK: fir.store %[[MINIMUM_VAL]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) +!CHECK: combiner +!CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +!CHECK: %[[RES:.*]] = arith.maximumf %[[LD0]], %[[LD1]] {{.*}}: f32 +!CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +!CHECK: omp.yield(%[[ARG0]] : !fir.ref) + +!CHECK-LABEL: omp.reduction.declare @max_i_32_byref : !fir.ref +!CHECK-SAME: init { +!CHECK: %[[MINIMUM_VAL:.*]] = arith.constant -2147483648 : i32 +!CHECK: fir.store %[[MINIMUM_VAL]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) +!CHECK: combiner +!CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +!CHECK: %[[RES:.*]] = arith.maxsi %[[LD0]], %[[LD1]] : i32 +!CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +!CHECK: omp.yield(%[[ARG0]] : !fir.ref) + +!CHECK-LABEL: @_QPreduction_max_int +!CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> +!CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_max_intEx"} +!CHECK: omp.parallel +!CHECK: omp.wsloop byref reduction(@max_i_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref +!CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] +!CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref +!CHECK: %[[RES:.+]] = arith.cmpi sgt, %[[LPRV]], %[[Y_I]] : i32 +!CHECK: %[[SEL:.+]] = arith.select %[[RES]], %[[LPRV]], %[[Y_I]] +!CHECK: fir.store %[[SEL]] to %[[PRV]] : !fir.ref +!CHECK: omp.terminator + +!CHECK-LABEL: @_QPreduction_max_real +!CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> +!CHECK: %[[X_REF:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFreduction_max_realEx"} +!CHECK: omp.parallel +!CHECK: omp.wsloop byref reduction(@max_f_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref +!CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] +!CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref +!CHECK: %[[RES:.+]] = arith.cmpf ogt, %[[Y_I]], %[[LPRV]] {{.*}} : f32 +!CHECK: omp.yield +!CHECK: omp.terminator + +subroutine reduction_max_int(y) + integer :: x, y(:) + x = 0 + !$omp parallel + !$omp do reduction(max:x) + do i=1, 100 + x = max(x, y(i)) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine + +subroutine reduction_max_real(y) + real :: x, y(:) + x = 0.0 + !$omp parallel + !$omp do reduction(max:x) + do i=1, 100 + x = max(y(i), x) + end do + !$omp end do + !$omp end parallel + print *, x + + !$omp parallel + !$omp do reduction(max:x) + do i=1, 100 + !CHECK-NOT: omp.reduction + if (y(i) .gt. x) x = y(i) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-min-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-min-byref.f90 new file mode 100644 index 000000000000..d767c54cdbc5 --- /dev/null +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-min-byref.f90 @@ -0,0 +1,91 @@ +! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -mmlir --force-byref-reduction -fopenmp -o - %s 2>&1 | FileCheck %s + +!CHECK: omp.reduction.declare @min_f_32_byref : !fir.ref +!CHECK-SAME: init { +!CHECK: %[[MAXIMUM_VAL:.*]] = arith.constant 3.40282347E+38 : f32 +!CHECK: %[[REF:.*]] = fir.alloca f32 +!CHECK: fir.store %[[MAXIMUM_VAL]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) +!CHECK: combiner +!CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +!CHECK: %[[RES:.*]] = arith.minimumf %[[LD0]], %[[LD1]] {{.*}}: f32 +!CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +!CHECK: omp.yield(%[[ARG0]] : !fir.ref) + +!CHECK-LABEL: omp.reduction.declare @min_i_32_byref : !fir.ref +!CHECK-SAME: init { +!CHECK: %[[MAXIMUM_VAL:.*]] = arith.constant 2147483647 : i32 +!CHECK: fir.store %[[MAXIMUM_VAL]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) +!CHECK: combiner +!CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +!CHECK: %[[RES:.*]] = arith.minsi %[[LD0]], %[[LD1]] : i32 +!CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +!CHECK: omp.yield(%[[ARG0]] : !fir.ref) + +!CHECK-LABEL: @_QPreduction_min_int +!CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> +!CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_min_intEx"} +!CHECK: omp.parallel +!CHECK: omp.wsloop byref reduction(@min_i_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref +!CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] +!CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref +!CHECK: %[[RES:.+]] = arith.cmpi slt, %[[LPRV]], %[[Y_I]] : i32 +!CHECK: %[[SEL:.+]] = arith.select %[[RES]], %[[LPRV]], %[[Y_I]] +!CHECK: fir.store %[[SEL]] to %[[PRV]] : !fir.ref +!CHECK: omp.yield +!CHECK: omp.terminator + +!CHECK-LABEL: @_QPreduction_min_real +!CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> +!CHECK: %[[X_REF:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFreduction_min_realEx"} +!CHECK: omp.parallel +!CHECK: omp.wsloop byref reduction(@min_f_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref +!CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] +!CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref +!CHECK: %[[RES:.+]] = arith.cmpf ogt, %[[Y_I]], %[[LPRV]] {{.*}} : f32 +!CHECK: omp.yield +!CHECK: omp.terminator + +subroutine reduction_min_int(y) + integer :: x, y(:) + x = 0 + !$omp parallel + !$omp do reduction(min:x) + do i=1, 100 + x = min(x, y(i)) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine + +subroutine reduction_min_real(y) + real :: x, y(:) + x = 0.0 + !$omp parallel + !$omp do reduction(min:x) + do i=1, 100 + x = min(y(i), x) + end do + !$omp end do + !$omp end parallel + print *, x + + !$omp parallel + !$omp do reduction(min:x) + do i=1, 100 + !CHECK-NOT: omp.reduction + if (y(i) .gt. x) x = y(i) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine diff --git a/flang/test/Lower/OpenMP/default-clause-byref.f90 b/flang/test/Lower/OpenMP/default-clause-byref.f90 new file mode 100644 index 000000000000..5d9538e53069 --- /dev/null +++ b/flang/test/Lower/OpenMP/default-clause-byref.f90 @@ -0,0 +1,385 @@ +! This test checks lowering of OpenMP parallel directive +! with `DEFAULT` clause present. + +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s +! RUN: bbc -fopenmp -emit-hlfir --force-byref-reduction %s -o - | FileCheck %s + + +!CHECK: func @_QQmain() attributes {fir.bindc_name = "default_clause_lowering"} { +!CHECK: %[[W:.*]] = fir.alloca i32 {bindc_name = "w", uniq_name = "_QFEw"} +!CHECK: %[[W_DECL:.*]]:2 = hlfir.declare %[[W]] {uniq_name = "_QFEw"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"} +!CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] {uniq_name = "_QFEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[Y:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"} +!CHECK: %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y]] {uniq_name = "_QFEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[Z:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFEz"} +!CHECK: %[[Z_DECL:.*]]:2 = hlfir.declare %[[Z]] {uniq_name = "_QFEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: omp.parallel { +!CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFEx"} +!CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[CONST:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[CONST]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFEy"} +!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRIVATE_W:.*]] = fir.alloca i32 {bindc_name = "w", pinned, uniq_name = "_QFEw"} +!CHECK: %[[PRIVATE_W_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_W]] {uniq_name = "_QFEw"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[CONST:.*]] = arith.constant 2 : i32 +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_Y_DECL]]#0 : !fir.ref +!CHECK: %[[RESULT:.*]] = arith.muli %[[CONST]], %[[TEMP]] : i32 +!CHECK: hlfir.assign %[[RESULT]] to %[[PRIVATE_X_DECL]]#0 : i32, !fir.ref +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_W_DECL]]#0 : !fir.ref +!CHECK: %[[CONST:.*]] = arith.constant 45 : i32 +!CHECK: %[[RESULT:.*]] = arith.addi %[[TEMP]], %[[CONST]] : i32 +!CHECK: hlfir.assign %[[RESULT]] to %[[Z_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } + +program default_clause_lowering + integer :: x, y, z, w + + !$omp parallel default(private) firstprivate(x) shared(z) + x = y * 2 + z = w + 45 + !$omp end parallel + +!CHECK: omp.parallel { +!CHECK: %[[TEMP:.*]] = fir.load %[[Y_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[X_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } + + !$omp parallel default(shared) + x = y + !$omp end parallel + +!CHECK: omp.parallel { +!CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFEx"} +!CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFEy"} +!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_Y_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } + + !$omp parallel default(none) private(x, y) + x = y + !$omp end parallel + +!CHECK: omp.parallel { +!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFEy"} +!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[Y_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_Y_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFEx"} +!CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_Y_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } + + !$omp parallel default(firstprivate) firstprivate(y) + x = y + !$omp end parallel + +!CHECK: omp.parallel { +!CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFEx"} +!CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFEy"} +!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[Y_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_Y_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[PRIVATE_W:.*]] = fir.alloca i32 {bindc_name = "w", pinned, uniq_name = "_QFEw"} +!CHECK: %[[PRIVATE_W_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_W]] {uniq_name = "_QFEw"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[W_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_W_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[CONST:.*]] = arith.constant 2 : i32 +!CHECK: %[[RESULT:.*]] = fir.load %[[PRIVATE_Y_DECL]]#0 : !fir.ref +!CHECK: %[[TEMP:.*]] = arith.muli %[[CONST]], %[[RESULT]] : i32 +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 : i32, !fir.ref +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_W_DECL]]#0 : !fir.ref +!CHECK: %[[CONST:.*]] = arith.constant 45 : i32 +!CHECK: %[[RESULT:.*]] = arith.addi %[[TEMP]], %[[CONST]] : i32 +!CHECK: hlfir.assign %[[RESULT]] to %[[Z_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } + + !$omp parallel default(firstprivate) private(x) shared(z) + x = y * 2 + z = w + 45 + !$omp end parallel + +!CHECK: omp.parallel { +!CHECK: omp.parallel { +!CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFEx"} +!CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFEy"} +!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_Y_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: omp.parallel { +!CHECK: %[[PRIVATE_W:.*]] = fir.alloca i32 {bindc_name = "w", pinned, uniq_name = "_QFEw"} +!CHECK: %[[PRIVATE_W_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_W]] {uniq_name = "_QFEw"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[W_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_W_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFEx"} +!CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_W_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: omp.terminator +!CHECK: } + !$omp parallel + !$omp parallel default(private) + x = y + !$omp end parallel + + !$omp parallel default(firstprivate) + w = x + !$omp end parallel + !$omp end parallel + +end program default_clause_lowering + +subroutine nested_default_clause_tests + integer :: x, y, z, w, k, a +!CHECK: %[[K:.*]] = fir.alloca i32 {bindc_name = "k", uniq_name = "_QFnested_default_clause_testsEk"} +!CHECK: %[[K_DECL:.*]]:2 = hlfir.declare %[[K]] {uniq_name = "_QFnested_default_clause_testsEk"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[W:.*]] = fir.alloca i32 {bindc_name = "w", uniq_name = "_QFnested_default_clause_testsEw"} +!CHECK: %[[W_DECL:.*]]:2 = hlfir.declare %[[W]] {uniq_name = "_QFnested_default_clause_testsEw"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFnested_default_clause_testsEx"} +!CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[Y:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFnested_default_clause_testsEy"} +!CHECK: %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[Z:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFnested_default_clause_testsEz"} +!CHECK: %[[Z_DECL:.*]]:2 = hlfir.declare %[[Z]] {uniq_name = "_QFnested_default_clause_testsEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: omp.parallel { +!CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_testsEx"} +!CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"} +!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRIVATE_Z:.*]] = fir.alloca i32 {bindc_name = "z", pinned, uniq_name = "_QFnested_default_clause_testsEz"} +!CHECK: %[[PRIVATE_Z_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Z]] {uniq_name = "_QFnested_default_clause_testsEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRIVATE_K:.*]] = fir.alloca i32 {bindc_name = "k", pinned, uniq_name = "_QFnested_default_clause_testsEk"} +!CHECK: %[[PRIVATE_K_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_K]] {uniq_name = "_QFnested_default_clause_testsEk"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: omp.parallel { +!CHECK: %[[INNER_PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"} +!CHECK: %[[INNER_PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[INNER_PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[INNER_PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_testsEx"} +!CHECK: %[[INNER_PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[INNER_PRIVATE_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[CONST:.*]] = arith.constant 20 : i32 +!CHECK: hlfir.assign %[[CONST]] to %[[INNER_PRIVATE_Y_DECL]]#0 : i32, !fir.ref +!CHECK: %[[CONST:.*]] = arith.constant 10 : i32 +!CHECK: hlfir.assign %[[CONST]] to %[[INNER_PRIVATE_X_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: omp.parallel { +!CHECK: %[[INNER_PRIVATE_W:.*]] = fir.alloca i32 {bindc_name = "w", pinned, uniq_name = "_QFnested_default_clause_testsEw"} +!CHECK: %[[INNER_PRIVATE_W_DECL:.*]]:2 = hlfir.declare %[[INNER_PRIVATE_W]] {uniq_name = "_QFnested_default_clause_testsEw"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[INNER_PRIVATE_Z:.*]] = fir.alloca i32 {bindc_name = "z", pinned, uniq_name = "_QFnested_default_clause_testsEz"} +!CHECK: %[[INNER_PRIVATE_Z_DECL:.*]]:2 = hlfir.declare %[[INNER_PRIVATE_Z]] {uniq_name = "_QFnested_default_clause_testsEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_Z_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[INNER_PRIVATE_Z_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[INNER_PRIVATE_K:.*]] = fir.alloca i32 {bindc_name = "k", pinned, uniq_name = "_QFnested_default_clause_testsEk"} +!CHECK: %[[INNER_PRIVATE_K_DECL:.*]]:2 = hlfir.declare %[[INNER_PRIVATE_K]] {uniq_name = "_QFnested_default_clause_testsEk"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_K_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[INNER_PRIVATE_K_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[CONST:.*]] = arith.constant 30 : i32 +!CHECK: hlfir.assign %[[CONST]] to %[[PRIVATE_Y_DECL]]#0 : i32, !fir.ref +!CHECK: %[[CONST:.*]] = arith.constant 40 : i32 +!CHECK: hlfir.assign %[[CONST]] to %[[INNER_PRIVATE_W_DECL]]#0 : i32, !fir.ref +!CHECK: %[[CONST:.*]] = arith.constant 50 : i32 +!CHECK: hlfir.assign %[[CONST]] to %[[INNER_PRIVATE_Z_DECL]]#0 : i32, !fir.ref +!CHECK: %[[CONST:.*]] = arith.constant 40 : i32 +!CHECK: hlfir.assign %[[CONST]] to %[[INNER_PRIVATE_K_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: omp.terminator +!CHECK: } + !$omp parallel firstprivate(x) private(y) shared(w) default(private) + !$omp parallel default(private) + y = 20 + x = 10 + !$omp end parallel + + !$omp parallel default(firstprivate) shared(y) private(w) + y = 30 + w = 40 + z = 50 + k = 40 + !$omp end parallel + !$omp end parallel + + +!CHECK: omp.parallel { +!CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"} +!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRIVATE_Z:.*]] = fir.alloca i32 {bindc_name = "z", pinned, uniq_name = "_QFnested_default_clause_testsEz"} +!CHECK: %[[PRIVATE_Z_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Z]] {uniq_name = "_QFnested_default_clause_testsEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRIVATE_W:.*]] = fir.alloca i32 {bindc_name = "w", pinned, uniq_name = "_QFnested_default_clause_testsEw"} +!CHECK: %[[PRIVATE_W_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_W]] {uniq_name = "_QFnested_default_clause_testsEw"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: omp.parallel { +!CHECK: %[[PRIVATE_INNER_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_testsEx"} +!CHECK: %[[PRIVATE_INNER_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_INNER_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_INNER_X_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[INNER_PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"} +!CHECK: %[[INNER_PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[INNER_PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_Y_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[INNER_PRIVATE_Y_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[TEMP:.*]] = fir.load %[[INNER_PRIVATE_Y_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_INNER_X_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: omp.parallel { +!CHECK: %[[PRIVATE_INNER_W:.*]] = fir.alloca i32 {bindc_name = "w", pinned, uniq_name = "_QFnested_default_clause_testsEw"} +!CHECK: %[[PRIVATE_INNER_W_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_INNER_W]] {uniq_name = "_QFnested_default_clause_testsEw"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRIVATE_INNER_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_testsEx"} +!CHECK: %[[PRIVATE_INNER_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_INNER_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP_1:.*]] = fir.load %[[PRIVATE_INNER_X_DECL]]#0 : !fir.ref +!CHECK: %[[TEMP_2:.*]] = fir.load %[[PRIVATE_Z_DECL]]#0 : !fir.ref +!CHECK: %[[RESULT:.*]] = arith.addi %{{.*}}, %{{.*}} : i32 +!CHECK: hlfir.assign %[[RESULT]] to %[[PRIVATE_INNER_W_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } + !$omp parallel default(private) + !$omp parallel default(firstprivate) + x = y + !$omp end parallel + + !$omp parallel default(private) shared(z) + w = x + z + !$omp end parallel + !$omp end parallel + +!CHECK: omp.parallel { +!CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_testsEx"} +!CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"} +!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRIVATE_W:.*]] = fir.alloca i32 {bindc_name = "w", pinned, uniq_name = "_QFnested_default_clause_testsEw"} +!CHECK: %[[PRIVATE_W_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_W]] {uniq_name = "_QFnested_default_clause_testsEw"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRIVATE_Z:.*]] = fir.alloca i32 {bindc_name = "z", pinned, uniq_name = "_QFnested_default_clause_testsEz"} +!CHECK: %[[PRIVATE_Z_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Z]] {uniq_name = "_QFnested_default_clause_testsEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: omp.parallel { +!CHECK: %[[INNER_PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_testsEx"} +!CHECK: %[[INNER_PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[INNER_PRIVATE_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[INNER_PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[INNER_PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"} +!CHECK: %[[INNER_PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[INNER_PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_Y_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[INNER_PRIVATE_Y_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[TEMP:.*]] = fir.load %[[INNER_PRIVATE_Y_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[INNER_PRIVATE_X_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: omp.parallel { +!CHECK: %[[TEMP_1:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref +!CHECK: %[[TEMP_2:.*]] = fir.load %[[PRIVATE_Z_DECL]]#0 : !fir.ref +!CHECK: %[[TEMP_3:.*]] = arith.addi %[[TEMP_1]], %[[TEMP_2]] : i32 +!CHECK: hlfir.assign %[[TEMP_3]] to %[[PRIVATE_W_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: } + !$omp parallel default(private) + !$omp parallel default(firstprivate) + x = y + !$omp end parallel + + !$omp parallel default(shared) + w = x + z + !$omp end parallel + !$omp end parallel + +!CHECK: omp.parallel { +!CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_testsEx"} +!CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"} +!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[TEMP:.*]] = fir.load %[[Y_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_Y_DECL]]#0 temporary_lhs : i32, !fir.ref +!CHECK: omp.single { +!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_Y_DECL]]#0 : !fir.ref +!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: omp.terminator +!CHECK: } +!CHECK: return +!CHECK: } + !$omp parallel default(firstprivate) + !$omp single + x = y + !$omp end single + !$omp end parallel +end subroutine + +!CHECK: func.func @_QPskipped_default_clause_checks() { +!CHECK: %[[TYPE_ADDR:.*]] = fir.address_of(@_QFskipped_default_clause_checksE.n.i1) : !fir.ref> +!CHECK: %[[VAL_CONST_2:.*]] = arith.constant 2 : index +!CHECK: %[[VAL_I1_DECLARE:.*]]:2 = hlfir.declare %[[TYPE_ADDR]] typeparams %[[VAL_CONST_2]] {{.*}} +!CHECK: %[[TYPE_ADDR_IT:.*]] = fir.address_of(@_QFskipped_default_clause_checksE.n.it) : !fir.ref> +!CHECK: %[[VAL_CONST_2_0:.*]] = arith.constant 2 : index +!CHECK: %[[VAL_IT_DECLARE:.*]]:2 = hlfir.declare %[[TYPE_ADDR_IT]] typeparams %[[VAL_CONST_2_0]] {{.*}} +!CHECK: %[[VAL_I_ALLOCA:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFskipped_default_clause_checksEi"} +!CHECK: %[[VAL_I_DECLARE:.*]]:2 = hlfir.declare %[[VAL_I_ALLOCA]] {{.*}} +!CHECK: %[[VAL_III_ALLOCA:.*]] = fir.alloca !fir.type<_QFskipped_default_clause_checksTit{i1:i32}> {bindc_name = "iii", uniq_name = "_QFskipped_default_clause_checksEiii"} +!CHECK: %[[VAL_III_DECLARE:.*]]:2 = hlfir.declare %[[VAL_III_ALLOCA]] {{.*}} +!CHECK: %[[VAL_X_ALLOCA:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFskipped_default_clause_checksEx"} +!CHECK: %[[VAL_X_DECLARE:.*]]:2 = hlfir.declare %[[VAL_X_ALLOCA]] {{.*}} +!CHECK: %[[VAL_Y_ALLOCA:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFskipped_default_clause_checksEy"} +!CHECK: %[[VAL_Y_DECLARE:.*]]:2 = hlfir.declare %[[VAL_Y_ALLOCA]] {{.*}} +!CHECK: %[[VAL_Z_ALLOCA:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFskipped_default_clause_checksEz"} +!CHECK: %[[VAL_Z_DECLARE:.*]]:2 = hlfir.declare %[[VAL_Z_ALLOCA]] {{.*}} +subroutine skipped_default_clause_checks() + integer :: x,y,z + type it + integer::i1 + end type + type(it)::iii + +!CHECK: omp.parallel { +!CHECK: omp.wsloop byref reduction(@min_i_32_byref %[[VAL_Z_DECLARE]]#0 -> %[[PRV:.+]] : !fir.ref) for (%[[ARG:.*]]) {{.*}} { +!CHECK: omp.yield +!CHECK: } +!CHECK: omp.terminator +!CHECK: } + !$omp parallel do default(private) REDUCTION(MIN:z) + do i = 1, 10 + x = x + MIN(y,x) + enddo + !$omp end parallel do + +!CHECK: omp.parallel { +!CHECK: omp.terminator +!CHECK: } + namelist /nam/i + !$omp parallel default(private) + write(1,nam ) + !$omp endparallel + +!CHECK: omp.parallel { +!CHECK: %[[PRIVATE_III_ALLOCA:.*]] = fir.alloca !fir.type<_QFskipped_default_clause_checksTit{i1:i32}> {{.*}} +!CHECK: %[[PRIVATE_III_DECLARE:.*]]:2 = hlfir.declare %[[PRIVATE_III_ALLOCA]] {{.*}} +!CHECK: %[[PRIVATE_ADDR:.*]] = fir.address_of(@_QQro._QFskipped_default_clause_checksTit.0) : !fir.ref> +!CHECK: %[[PRIVATE_PARAM:.*]]:2 = hlfir.declare %[[PRIVATE_ADDR]] {{.*}} +!CHECK: hlfir.assign %[[PRIVATE_PARAM]]#0 to %[[PRIVATE_III_DECLARE]]#0 {{.*}} +!CHECK: omp.terminator +!CHECK: } + !$omp parallel default(private) + iii=it(11) + !$omp end parallel +end subroutine diff --git a/flang/test/Lower/OpenMP/delayed-privatization-reduction-byref.f90 b/flang/test/Lower/OpenMP/delayed-privatization-reduction-byref.f90 new file mode 100644 index 000000000000..067a71340b8d --- /dev/null +++ b/flang/test/Lower/OpenMP/delayed-privatization-reduction-byref.f90 @@ -0,0 +1,30 @@ +! Test that reductions and delayed privatization work properly togehter. Since +! both types of clauses add block arguments to the OpenMP region, we make sure +! that the block arguments are added in the proper order (reductions first and +! then delayed privatization. + +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction --openmp-enable-delayed-privatization -o - %s 2>&1 | FileCheck %s + +subroutine red_and_delayed_private + integer :: red + integer :: prv + + red = 0 + prv = 10 + + !$omp parallel reduction(+:red) private(prv) + red = red + 1 + prv = 20 + !$omp end parallel +end subroutine + +! CHECK-LABEL: omp.private {type = private} +! CHECK-SAME: @[[PRIVATIZER_SYM:.*]] : !fir.ref alloc { + +! CHECK-LABEL: omp.reduction.declare +! CHECK-SAME: @[[REDUCTION_SYM:.*]] : !fir.ref init + +! CHECK-LABEL: _QPred_and_delayed_private +! CHECK: omp.parallel +! CHECK-SAME: reduction(@[[REDUCTION_SYM]] %{{.*}} -> %arg0 : !fir.ref) +! CHECK-SAME: private(@[[PRIVATIZER_SYM]] %{{.*}} -> %arg1 : !fir.ref) { diff --git a/flang/test/Lower/OpenMP/parallel-reduction-add-byref.f90 b/flang/test/Lower/OpenMP/parallel-reduction-add-byref.f90 new file mode 100644 index 000000000000..c4a4695b8d9f --- /dev/null +++ b/flang/test/Lower/OpenMP/parallel-reduction-add-byref.f90 @@ -0,0 +1,125 @@ +! RUN: bbc -emit-hlfir --force-byref-reduction -fopenmp -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction -o - %s 2>&1 | FileCheck %s + +!CHECK-LABEL: omp.reduction.declare +!CHECK-SAME: @[[RED_F32_NAME:.*]] : !fir.ref +!CHECK-SAME: init { +!CHECK: ^bb0(%{{.*}}: !fir.ref): +!CHECK: %[[C0_1:.*]] = arith.constant 0.000000e+00 : f32 +!CHECK: %[[REF:.*]] = fir.alloca f32 +!CHECKL fir.store [[%C0_1]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) +!CHECK: } combiner { +!CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +!CHECK: %[[RES:.*]] = arith.addf %[[LD0]], %[[LD1]] {{.*}}: f32 +!CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +!CHECK: omp.yield(%[[ARG0]] : !fir.ref) +!CHECK: } + +!CHECK-LABEL: omp.reduction.declare +!CHECK-SAME: @[[RED_I32_NAME:.*]] : !fir.ref +!CHECK-SAME: init { +!CHECK: ^bb0(%{{.*}}: !fir.ref): +!CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 +!CHECK: %[[REF:.*]] = fir.alloca i32 +!CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) +!CHECK: } combiner { +!CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +!CHECK: %[[RES:.*]] = arith.addi %[[LD0]], %[[LD1]] : i32 +!CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +!CHECK: omp.yield(%[[ARG0]] : !fir.ref) +!CHECK: } + +!CHECK-LABEL: func.func @_QPsimple_int_add +!CHECK: %[[IREF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_int_addEi"} +!CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[IREF]] {uniq_name = "_QFsimple_int_addEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[I_START:.*]] = arith.constant 0 : i32 +!CHECK: hlfir.assign %[[I_START]] to %[[I_DECL]]#0 : i32, !fir.ref +!CHECK: omp.parallel byref reduction(@[[RED_I32_NAME]] %[[I_DECL]]#0 -> %[[PRV:.+]] : !fir.ref) { +!CHECK: %[[P_DECL:.+]]:2 = hlfir.declare %[[PRV]] {{.*}} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[LPRV:.+]] = fir.load %[[P_DECL]]#0 : !fir.ref +!CHECK: %[[I_INCR:.*]] = arith.constant 1 : i32 +!CHECK: %[[RES:.+]] = arith.addi %[[LPRV]], %[[I_INCR]] : i32 +!CHECK: hlfir.assign %[[RES]] to %[[P_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: return +subroutine simple_int_add + integer :: i + i = 0 + + !$omp parallel reduction(+:i) + i = i + 1 + !$omp end parallel + + print *, i +end subroutine + +!CHECK-LABEL: func.func @_QPsimple_real_add +!CHECK: %[[RREF:.*]] = fir.alloca f32 {bindc_name = "r", uniq_name = "_QFsimple_real_addEr"} +!CHECK: %[[R_DECL:.*]]:2 = hlfir.declare %[[RREF]] {uniq_name = "_QFsimple_real_addEr"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[R_START:.*]] = arith.constant 0.000000e+00 : f32 +!CHECK: hlfir.assign %[[R_START]] to %[[R_DECL]]#0 : f32, !fir.ref +!CHECK: omp.parallel byref reduction(@[[RED_F32_NAME]] %[[R_DECL]]#0 -> %[[PRV:.+]] : !fir.ref) { +!CHECK: %[[P_DECL:.+]]:2 = hlfir.declare %[[PRV]] {{.*}} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[LPRV:.+]] = fir.load %[[P_DECL]]#0 : !fir.ref +!CHECK: %[[R_INCR:.*]] = arith.constant 1.500000e+00 : f32 +!CHECK: %[[RES:.+]] = arith.addf %[[LPRV]], %[[R_INCR]] {{.*}} : f32 +!CHECK: hlfir.assign %[[RES]] to %[[P_DECL]]#0 : f32, !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: return +subroutine simple_real_add + real :: r + r = 0.0 + + !$omp parallel reduction(+:r) + r = r + 1.5 + !$omp end parallel + + print *, r +end subroutine + +!CHECK-LABEL: func.func @_QPint_real_add +!CHECK: %[[IREF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFint_real_addEi"} +!CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[IREF]] {uniq_name = "_QFint_real_addEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[RREF:.*]] = fir.alloca f32 {bindc_name = "r", uniq_name = "_QFint_real_addEr"} +!CHECK: %[[R_DECL:.*]]:2 = hlfir.declare %[[RREF]] {uniq_name = "_QFint_real_addEr"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[R_START:.*]] = arith.constant 0.000000e+00 : f32 +!CHECK: hlfir.assign %[[R_START]] to %[[R_DECL]]#0 : f32, !fir.ref +!CHECK: %[[I_START:.*]] = arith.constant 0 : i32 +!CHECK: hlfir.assign %[[I_START]] to %[[I_DECL]]#0 : i32, !fir.ref +!CHECK: omp.parallel byref reduction(@[[RED_I32_NAME]] %[[I_DECL]]#0 -> %[[IPRV:.+]] : !fir.ref, @[[RED_F32_NAME]] %[[R_DECL]]#0 -> %[[RPRV:.+]] : !fir.ref) { +!CHECK: %[[IP_DECL:.+]]:2 = hlfir.declare %[[IPRV]] {{.*}} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[RP_DECL:.+]]:2 = hlfir.declare %[[RPRV]] {{.*}} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[R_INCR:.*]] = arith.constant 1.500000e+00 : f32 +!CHECK: %[[R_LPRV:.+]] = fir.load %[[RP_DECL]]#0 : !fir.ref +!CHECK: %[[RES1:.+]] = arith.addf %[[R_INCR]], %[[R_LPRV]] {{.*}} : f32 +!CHECK: hlfir.assign %[[RES1]] to %[[RP_DECL]]#0 : f32, !fir.ref +!CHECK: %[[I_LPRV:.+]] = fir.load %[[IP_DECL]]#0 : !fir.ref +!CHECK: %[[I_INCR:.*]] = arith.constant 3 : i32 +!CHECK: %[[RES0:.+]] = arith.addi %[[I_LPRV]], %[[I_INCR]] : i32 +!CHECK: hlfir.assign %[[RES0]] to %[[IP_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: return +subroutine int_real_add + real :: r + integer :: i + + r = 0.0 + i = 0 + + !$omp parallel reduction(+:i,r) + r = 1.5 + r + i = i + 3 + !$omp end parallel + + print *, r + print *, i +end subroutine diff --git a/flang/test/Lower/OpenMP/parallel-reduction-byref.f90 b/flang/test/Lower/OpenMP/parallel-reduction-byref.f90 new file mode 100644 index 000000000000..a7c77c52674e --- /dev/null +++ b/flang/test/Lower/OpenMP/parallel-reduction-byref.f90 @@ -0,0 +1,44 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction -o - %s 2>&1 | FileCheck %s + +!CHECK: omp.reduction.declare @[[REDUCTION_DECLARE:[_a-z0-9]+]] : !fir.ref +!CHECK-SAME: init { +!CHECK: ^bb0(%{{.*}}: !fir.ref): +!CHECK: %[[I0:[_a-z0-9]+]] = arith.constant 0 : i32 +!CHECK: %[[REF:.*]] = fir.alloca i32 +!CHECKL fir.store [[%I0]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) +!CHECK: } combiner { +!CHECK: ^bb0(%[[C0:[_a-z0-9]+]]: !fir.ref, %[[C1:[_a-z0-9]+]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[C0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[C1]] : !fir.ref +!CHECK: %[[CR:[_a-z0-9]+]] = arith.addi %[[LD0]], %[[LD1]] : i32 +!CHECK: fir.store %[[CR]] to %[[C0]] : !fir.ref +!CHECK: omp.yield(%[[C0]] : !fir.ref) +!CHECK: } +!CHECK: func.func @_QQmain() attributes {fir.bindc_name = "mn"} { +!CHECK: %[[RED_ACCUM_REF:[_a-z0-9]+]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"} +!CHECK: %[[RED_ACCUM_DECL:[_a-z0-9]+]]:2 = hlfir.declare %[[RED_ACCUM_REF]] {uniq_name = "_QFEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[C0:[_a-z0-9]+]] = arith.constant 0 : i32 +!CHECK: hlfir.assign %[[C0]] to %[[RED_ACCUM_DECL]]#0 : i32, !fir.ref +!CHECK: omp.parallel byref reduction(@[[REDUCTION_DECLARE]] %[[RED_ACCUM_DECL]]#0 -> %[[PRIVATE_RED:[a-z0-9]+]] : !fir.ref) { +!CHECK: %[[PRIVATE_DECL:[_a-z0-9]+]]:2 = hlfir.declare %[[PRIVATE_RED]] {uniq_name = "_QFEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[C1:[_a-z0-9]+]] = arith.constant 1 : i32 +!CHECK: hlfir.assign %[[C1]] to %[[PRIVATE_DECL]]#0 : i32, !fir.ref +!CHECK: omp.terminator +!CHECK: } +!CHECK: %[[RED_ACCUM_VAL:[_a-z0-9]+]] = fir.load %[[RED_ACCUM_DECL]]#0 : !fir.ref +!CHECK: {{.*}} = fir.call @_FortranAioOutputInteger32(%{{.*}}, %[[RED_ACCUM_VAL]]) fastmath : (!fir.ref, i32) -> i1 +!CHECK: return +!CHECK: } + +program mn + integer :: i + i = 0 + + !$omp parallel reduction(+:i) + i = 1 + !$omp end parallel + + print *, i +end program diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90 new file mode 100644 index 000000000000..8492a69fed58 --- /dev/null +++ b/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90 @@ -0,0 +1,16 @@ +! Check that for parallel do, reduction is only processed for the loop + +! RUN: bbc -fopenmp --force-byref-reduction -emit-hlfir %s -o - | FileCheck %s +! RUN: flang-new -fc1 -fopenmp -mmlir --force-byref-reduction -emit-hlfir %s -o - | FileCheck %s + +! CHECK: omp.parallel { +! CHECK: omp.wsloop byref reduction(@add_reduction_i_32 +subroutine sb + integer :: x + x = 0 + !$omp parallel do reduction(+:x) + do i=1,100 + x = x + 1 + end do + !$omp end parallel do +end subroutine diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90 new file mode 100644 index 000000000000..e8a04c23f4ea --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90 @@ -0,0 +1,433 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + +! CHECK-LABEL: omp.reduction.declare @add_reduction_f_64_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant 0.000000e+00 : f64 +! CHECK: %[[REF:.*]] = fir.alloca f64 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.addf %[[LD0]], %[[LD1]] fastmath : f64 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: omp.reduction.declare @add_reduction_i_64_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant 0 : i64 +! CHECK: %[[REF:.*]] = fir.alloca i64 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.addi %[[LD0]], %[[LD1]] : i64 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: omp.reduction.declare @add_reduction_f_32_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: %[[REF:.*]] = fir.alloca f32 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.addf %[[LD0]], %[[LD1]] fastmath : f32 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: omp.reduction.declare @add_reduction_i_32_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 +! CHECK: %[[REF:.*]] = fir.alloca i32 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.addi %[[LD0]], %[[LD1]] : i32 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: func.func @_QPsimple_int_reduction() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_int_reductionEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFsimple_int_reductionEx"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = arith.constant 0 : i32 +! CHECK: hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref +! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_14:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref +! CHECK: %[[VAL_15:.*]] = arith.addi %[[VAL_13]], %[[VAL_14]] : i32 +! CHECK: hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : i32, !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + +subroutine simple_int_reduction + integer :: x + x = 0 + !$omp parallel + !$omp do reduction(+:x) + do i=1, 100 + x = x + i + end do + !$omp end do + !$omp end parallel +end subroutine + + +! CHECK-LABEL: func.func @_QPsimple_real_reduction() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_real_reductionEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFsimple_real_reductionEx"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_f_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref +! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_14:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref +! CHECK: %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (i32) -> f32 +! CHECK: %[[VAL_16:.*]] = arith.addf %[[VAL_13]], %[[VAL_15]] fastmath : f32 +! CHECK: hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : f32, !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + +subroutine simple_real_reduction + real :: x + x = 0.0 + !$omp parallel + !$omp do reduction(+:x) + do i=1, 100 + x = x + i + end do + !$omp end do + !$omp end parallel +end subroutine + + +! CHECK-LABEL: func.func @_QPsimple_int_reduction_switch_order() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_int_reduction_switch_orderEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFsimple_int_reduction_switch_orderEx"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = arith.constant 0 : i32 +! CHECK: hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref +! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref +! CHECK: %[[VAL_14:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_15:.*]] = arith.addi %[[VAL_13]], %[[VAL_14]] : i32 +! CHECK: hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : i32, !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + +subroutine simple_int_reduction_switch_order + integer :: x + x = 0 + !$omp parallel + !$omp do reduction(+:x) + do i=1, 100 + x = i + x + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPsimple_real_reduction_switch_order() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_real_reduction_switch_orderEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFsimple_real_reduction_switch_orderEx"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_f_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref +! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref +! CHECK: %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (i32) -> f32 +! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_16:.*]] = arith.addf %[[VAL_14]], %[[VAL_15]] fastmath : f32 +! CHECK: hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : f32, !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + +subroutine simple_real_reduction_switch_order + real :: x + x = 0.0 + !$omp parallel + !$omp do reduction(+:x) + do i=1, 100 + x = i + x + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_int_reductions_same_type() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_int_reductions_same_typeEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_int_reductions_same_typeEx"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFmultiple_int_reductions_same_typeEy"} +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFmultiple_int_reductions_same_typeEz"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_int_reductions_same_typeEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_8:.*]] = arith.constant 0 : i32 +! CHECK: hlfir.assign %[[VAL_8]] to %[[VAL_3]]#0 : i32, !fir.ref +! CHECK: %[[VAL_9:.*]] = arith.constant 0 : i32 +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_5]]#0 : i32, !fir.ref +! CHECK: %[[VAL_10:.*]] = arith.constant 0 : i32 +! CHECK: hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : i32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @add_reduction_i_32_byref %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @add_reduction_i_32_byref %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { +! CHECK: fir.store %[[VAL_19]] to %[[VAL_12]]#1 : !fir.ref +! CHECK: %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_int_reductions_same_typeEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_23:.*]] = fir.load %[[VAL_20]]#0 : !fir.ref +! CHECK: %[[VAL_24:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_25:.*]] = arith.addi %[[VAL_23]], %[[VAL_24]] : i32 +! CHECK: hlfir.assign %[[VAL_25]] to %[[VAL_20]]#0 : i32, !fir.ref +! CHECK: %[[VAL_26:.*]] = fir.load %[[VAL_21]]#0 : !fir.ref +! CHECK: %[[VAL_27:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_28:.*]] = arith.addi %[[VAL_26]], %[[VAL_27]] : i32 +! CHECK: hlfir.assign %[[VAL_28]] to %[[VAL_21]]#0 : i32, !fir.ref +! CHECK: %[[VAL_29:.*]] = fir.load %[[VAL_22]]#0 : !fir.ref +! CHECK: %[[VAL_30:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_31:.*]] = arith.addi %[[VAL_29]], %[[VAL_30]] : i32 +! CHECK: hlfir.assign %[[VAL_31]] to %[[VAL_22]]#0 : i32, !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + +subroutine multiple_int_reductions_same_type + integer :: x,y,z + x = 0 + y = 0 + z = 0 + !$omp parallel + !$omp do reduction(+:x,y,z) + do i=1, 100 + x = x + i + y = y + i + z = z + i + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_real_reductions_same_type() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_real_reductions_same_typeEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFmultiple_real_reductions_same_typeEx"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = fir.alloca f32 {bindc_name = "y", uniq_name = "_QFmultiple_real_reductions_same_typeEy"} +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_6:.*]] = fir.alloca f32 {bindc_name = "z", uniq_name = "_QFmultiple_real_reductions_same_typeEz"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_real_reductions_same_typeEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_8:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_8]] to %[[VAL_3]]#0 : f32, !fir.ref +! CHECK: %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_5]]#0 : f32, !fir.ref +! CHECK: %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : f32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_f_32_byref %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @add_reduction_f_32_byref %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @add_reduction_f_32_byref %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { +! CHECK: fir.store %[[VAL_19]] to %[[VAL_12]]#1 : !fir.ref +! CHECK: %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_real_reductions_same_typeEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_23:.*]] = fir.load %[[VAL_20]]#0 : !fir.ref +! CHECK: %[[VAL_24:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_25:.*]] = fir.convert %[[VAL_24]] : (i32) -> f32 +! CHECK: %[[VAL_26:.*]] = arith.addf %[[VAL_23]], %[[VAL_25]] fastmath : f32 +! CHECK: hlfir.assign %[[VAL_26]] to %[[VAL_20]]#0 : f32, !fir.ref +! CHECK: %[[VAL_27:.*]] = fir.load %[[VAL_21]]#0 : !fir.ref +! CHECK: %[[VAL_28:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_29:.*]] = fir.convert %[[VAL_28]] : (i32) -> f32 +! CHECK: %[[VAL_30:.*]] = arith.addf %[[VAL_27]], %[[VAL_29]] fastmath : f32 +! CHECK: hlfir.assign %[[VAL_30]] to %[[VAL_21]]#0 : f32, !fir.ref +! CHECK: %[[VAL_31:.*]] = fir.load %[[VAL_22]]#0 : !fir.ref +! CHECK: %[[VAL_32:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (i32) -> f32 +! CHECK: %[[VAL_34:.*]] = arith.addf %[[VAL_31]], %[[VAL_33]] fastmath : f32 +! CHECK: hlfir.assign %[[VAL_34]] to %[[VAL_22]]#0 : f32, !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + +subroutine multiple_real_reductions_same_type + real :: x,y,z + x = 0.0 + y = 0.0 + z = 0.0 + !$omp parallel + !$omp do reduction(+:x,y,z) + do i=1, 100 + x = x + i + y = y + i + z = z + i + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_reductions_different_type() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_reductions_different_typeEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca f64 {bindc_name = "w", uniq_name = "_QFmultiple_reductions_different_typeEw"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFmultiple_reductions_different_typeEw"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_reductions_different_typeEx"} +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "y", uniq_name = "_QFmultiple_reductions_different_typeEy"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_8:.*]] = fir.alloca f32 {bindc_name = "z", uniq_name = "_QFmultiple_reductions_different_typeEz"} +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFmultiple_reductions_different_typeEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_10:.*]] = arith.constant 0 : i32 +! CHECK: hlfir.assign %[[VAL_10]] to %[[VAL_5]]#0 : i32, !fir.ref +! CHECK: %[[VAL_11:.*]] = arith.constant 0 : i64 +! CHECK: hlfir.assign %[[VAL_11]] to %[[VAL_7]]#0 : i64, !fir.ref +! CHECK: %[[VAL_12:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_12]] to %[[VAL_9]]#0 : f32, !fir.ref +! CHECK: %[[VAL_13:.*]] = arith.constant 0.000000e+00 : f64 +! CHECK: hlfir.assign %[[VAL_13]] to %[[VAL_3]]#0 : f64, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_14:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_16:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_17:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_18:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref, @add_reduction_i_64_byref %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref, @add_reduction_f_32_byref %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref, @add_reduction_f_64_byref %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref) for (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) { +! CHECK: fir.store %[[VAL_23]] to %[[VAL_15]]#1 : !fir.ref +! CHECK: %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_26:.*]]:2 = hlfir.declare %[[VAL_21]] {uniq_name = "_QFmultiple_reductions_different_typeEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_22]] {uniq_name = "_QFmultiple_reductions_different_typeEw"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_28:.*]] = fir.load %[[VAL_24]]#0 : !fir.ref +! CHECK: %[[VAL_29:.*]] = fir.load %[[VAL_15]]#0 : !fir.ref +! CHECK: %[[VAL_30:.*]] = arith.addi %[[VAL_28]], %[[VAL_29]] : i32 +! CHECK: hlfir.assign %[[VAL_30]] to %[[VAL_24]]#0 : i32, !fir.ref +! CHECK: %[[VAL_31:.*]] = fir.load %[[VAL_25]]#0 : !fir.ref +! CHECK: %[[VAL_32:.*]] = fir.load %[[VAL_15]]#0 : !fir.ref +! CHECK: %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (i32) -> i64 +! CHECK: %[[VAL_34:.*]] = arith.addi %[[VAL_31]], %[[VAL_33]] : i64 +! CHECK: hlfir.assign %[[VAL_34]] to %[[VAL_25]]#0 : i64, !fir.ref +! CHECK: %[[VAL_35:.*]] = fir.load %[[VAL_26]]#0 : !fir.ref +! CHECK: %[[VAL_36:.*]] = fir.load %[[VAL_15]]#0 : !fir.ref +! CHECK: %[[VAL_37:.*]] = fir.convert %[[VAL_36]] : (i32) -> f32 +! CHECK: %[[VAL_38:.*]] = arith.addf %[[VAL_35]], %[[VAL_37]] fastmath : f32 +! CHECK: hlfir.assign %[[VAL_38]] to %[[VAL_26]]#0 : f32, !fir.ref +! CHECK: %[[VAL_39:.*]] = fir.load %[[VAL_27]]#0 : !fir.ref +! CHECK: %[[VAL_40:.*]] = fir.load %[[VAL_15]]#0 : !fir.ref +! CHECK: %[[VAL_41:.*]] = fir.convert %[[VAL_40]] : (i32) -> f64 +! CHECK: %[[VAL_42:.*]] = arith.addf %[[VAL_39]], %[[VAL_41]] fastmath : f64 +! CHECK: hlfir.assign %[[VAL_42]] to %[[VAL_27]]#0 : f64, !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: return +! CHECK: } + +subroutine multiple_reductions_different_type + integer :: x + integer(kind=8) :: y + real :: z + real(kind=8) :: w + x = 0 + y = 0 + z = 0.0 + w = 0.0 + !$omp parallel + !$omp do reduction(+:x,y,z,w) + do i=1, 100 + x = x + i + y = y + i + z = z + i + w = w + i + end do + !$omp end do + !$omp end parallel +end subroutine diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90 new file mode 100644 index 000000000000..3739b3ae36ea --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90 @@ -0,0 +1,58 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + +! CHECK-LABEL: omp.reduction.declare @add_reduction_i_32_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 +! CHECK: %[[REF:.*]] = fir.alloca i32 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.addi %[[LD0]], %[[LD1]] : i32 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: func.func @_QPsimple_int_reduction() +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_int_reductionEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFsimple_int_reductionEx"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = arith.constant 0 : i32 +! CHECK: hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) +! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref +! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_14:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref +! CHECK: %[[VAL_15:.*]] = arith.addi %[[VAL_13]], %[[VAL_14]] : i32 +! CHECK: hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : i32, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + + +subroutine simple_int_reduction + integer :: x + x = 0 + !$omp parallel + !$omp do reduction(+:x) + do i=1, 100 + x = x + i + end do + !$omp end do + !$omp end parallel +end subroutine diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90 new file mode 100644 index 000000000000..15a6dde046fe --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90 @@ -0,0 +1,64 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + +! CHECK-LABEL: omp.reduction.declare @iand_i_32_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant -1 : i32 +! CHECK: %[[REF:.*]] = fir.alloca i32 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.andi %[[LD0]], %[[LD1]] : i32 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: func.func @_QPreduction_iand( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFreduction_iandEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_iandEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_iandEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_iandEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_iandEy"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i32 +! CHECK: hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iandEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@iand_i_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref +! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_iandEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref +! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64 +! CHECK: %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]]) : (!fir.box>, i64) -> !fir.ref +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_17]] : !fir.ref +! CHECK: %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i32 +! CHECK: hlfir.assign %[[VAL_20]] to %[[VAL_14]]#0 : i32, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator + + + +subroutine reduction_iand(y) + integer :: x, y(:) + x = 0 + !$omp parallel + !$omp do reduction(iand:x) + do i=1, 100 + x = iand(x, y(i)) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90 new file mode 100644 index 000000000000..4e0957219fa5 --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90 @@ -0,0 +1,55 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + +! CHECK-LABEL: omp.reduction.declare @ieor_i_32_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 +! CHECK: %[[REF:.*]] = fir.alloca i32 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.xori %[[LD0]], %[[LD1]] : i32 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +!CHECK-LABEL: @_QPreduction_ieor +!CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> +!CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_ieorEx"} +!CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {uniq_name = "_QFreduction_ieorEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y_BOX]] {uniq_name = "_QFreduction_ieorEy"} : (!fir.box>) -> (!fir.box>, !fir.box>) + + +!CHECK: omp.parallel +!CHECK: %[[I_REF:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +!CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I_REF]] {uniq_name = "_QFreduction_ieorEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: omp.wsloop byref reduction(@ieor_i_32_byref %[[X_DECL]]#0 -> %[[PRV:.+]] : !fir.ref) for +!CHECK: fir.store %{{.*}} to %[[I_DECL]]#1 : !fir.ref +!CHECK: %[[PRV_DECL:.+]]:2 = hlfir.declare %[[PRV]] {{.*}} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[I_32:.*]] = fir.load %[[I_DECL]]#0 : !fir.ref +!CHECK: %[[I_64:.*]] = fir.convert %[[I_32]] : (i32) -> i64 +!CHECK: %[[Y_I_REF:.*]] = hlfir.designate %[[Y_DECL]]#0 (%[[I_64]]) : (!fir.box>, i64) -> !fir.ref +!CHECK: %[[LPRV:.+]] = fir.load %[[PRV_DECL]]#0 : !fir.ref +!CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref +!CHECK: %[[RES:.+]] = arith.xori %[[LPRV]], %[[Y_I]] : i32 +!CHECK: hlfir.assign %[[RES]] to %[[PRV_DECL]]#0 : i32, !fir.ref +!CHECK: omp.yield +!CHECK: omp.terminator + +subroutine reduction_ieor(y) + integer :: x, y(:) + x = 0 + !$omp parallel + !$omp do reduction(ieor:x) + do i=1, 100 + x = ieor(x, y(i)) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90 new file mode 100644 index 000000000000..712edaf95575 --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90 @@ -0,0 +1,64 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + +! CHECK-LABEL: omp.reduction.declare @ior_i_32_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 +! CHECK: %[[REF:.*]] = fir.alloca i32 +! CHECK: fir.store %[[C0_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.ori %[[LD0]], %[[LD1]] : i32 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: func.func @_QPreduction_ior( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFreduction_iorEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_iorEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_iorEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_iorEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_iorEy"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i32 +! CHECK: hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref +! CHECK: omp.parallel +! CHECK: %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iorEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@ior_i_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) +! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref +! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_iorEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref +! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64 +! CHECK: %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]]) : (!fir.box>, i64) -> !fir.ref +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_17]] : !fir.ref +! CHECK: %[[VAL_20:.*]] = arith.ori %[[VAL_18]], %[[VAL_19]] : i32 +! CHECK: hlfir.assign %[[VAL_20]] to %[[VAL_14]]#0 : i32, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator + + + +subroutine reduction_ior(y) + integer :: x, y(:) + x = 0 + !$omp parallel + !$omp do reduction(ior:x) + do i=1, 100 + x = ior(x, y(i)) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 new file mode 100644 index 000000000000..4162626f926a --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 @@ -0,0 +1,206 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + +! CHECK-LABEL: omp.reduction.declare @and_reduction : !fir.ref> +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref>): +! CHECK: %[[VAL_1:.*]] = arith.constant true +! CHECK: %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4> +! CHECK: %[[REF:.*]] = fir.alloca !fir.logical<4> +! CHECK: fir.store %[[VAL_2]] to %[[REF]] : !fir.ref> +! CHECK: omp.yield(%[[REF]] : !fir.ref>) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref> +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref> +! CHECK: %[[VAL_2:.*]] = fir.convert %[[LD0]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_3:.*]] = fir.convert %[[LD1]] : (!fir.logical<4>) -> i1 +! CHECK: %[[RES:.*]] = arith.andi %[[VAL_2]], %[[VAL_3]] : i1 +! CHECK: %[[VAL_5:.*]] = fir.convert %[[RES]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_5]] to %[[ARG0]] : !fir.ref> +! CHECK: omp.yield(%[[ARG0]] : !fir.ref>) +! CHECK: } + +! CHECK-LABEL: func.func @_QPsimple_reduction( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_reductionEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFsimple_reductionEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_5:.*]] = arith.constant 100 : index +! CHECK: %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1> +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref>>, !fir.shape<1>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_8:.*]] = arith.constant true +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_12:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_13:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref>) for (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) { +! CHECK: fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref +! CHECK: %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref> +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref +! CHECK: %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (i32) -> i64 +! CHECK: %[[VAL_21:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_20]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref> +! CHECK: %[[VAL_23:.*]] = fir.convert %[[VAL_18]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_24:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_25:.*]] = arith.andi %[[VAL_23]], %[[VAL_24]] : i1 +! CHECK: %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_reduction(y) + logical :: x, y(100) + x = .true. + !$omp parallel + !$omp do reduction(.and.:x) + do i=1, 100 + x = x .and. y(i) + end do + !$omp end do + !$omp end parallel +end subroutine simple_reduction + + +! CHECK-LABEL: func.func @_QPsimple_reduction_switch_order( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_reduction_switch_orderEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFsimple_reduction_switch_orderEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_5:.*]] = arith.constant 100 : index +! CHECK: %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1> +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref>>, !fir.shape<1>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_8:.*]] = arith.constant true +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_12:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_13:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref>) for (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) { +! CHECK: fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref +! CHECK: %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (i32) -> i64 +! CHECK: %[[VAL_20:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_19]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref> +! CHECK: %[[VAL_22:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref> +! CHECK: %[[VAL_23:.*]] = fir.convert %[[VAL_21]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_24:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_25:.*]] = arith.andi %[[VAL_23]], %[[VAL_24]] : i1 +! CHECK: %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_reduction_switch_order(y) + logical :: x, y(100) + x = .true. + !$omp parallel + !$omp do reduction(.and.:x) + do i=1, 100 + x = y(i) .and. x + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_reductions( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "w"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_reductionsEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = arith.constant 100 : index +! CHECK: %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1> +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref>>, !fir.shape<1>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_6:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_8:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"} +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_10:.*]] = fir.alloca !fir.logical<4> {bindc_name = "z", uniq_name = "_QFmultiple_reductionsEz"} +! CHECK: %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_12:.*]] = arith.constant true +! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_13]] to %[[VAL_7]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_14:.*]] = arith.constant true +! CHECK: %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_15]] to %[[VAL_9]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_16:.*]] = arith.constant true +! CHECK: %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_18:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_20:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_21:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_22:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]] : !fir.ref>, @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]] : !fir.ref>, @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref>) for (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) { +! CHECK: fir.store %[[VAL_26]] to %[[VAL_19]]#1 : !fir.ref +! CHECK: %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_25]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_30:.*]] = fir.load %[[VAL_27]]#0 : !fir.ref> +! CHECK: %[[VAL_31:.*]] = fir.load %[[VAL_19]]#0 : !fir.ref +! CHECK: %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (i32) -> i64 +! CHECK: %[[VAL_33:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_32]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref> +! CHECK: %[[VAL_35:.*]] = fir.convert %[[VAL_30]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_36:.*]] = fir.convert %[[VAL_34]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_37:.*]] = arith.andi %[[VAL_35]], %[[VAL_36]] : i1 +! CHECK: %[[VAL_38:.*]] = fir.convert %[[VAL_37]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_38]] to %[[VAL_27]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_39:.*]] = fir.load %[[VAL_28]]#0 : !fir.ref> +! CHECK: %[[VAL_40:.*]] = fir.load %[[VAL_19]]#0 : !fir.ref +! CHECK: %[[VAL_41:.*]] = fir.convert %[[VAL_40]] : (i32) -> i64 +! CHECK: %[[VAL_42:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_41]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_43:.*]] = fir.load %[[VAL_42]] : !fir.ref> +! CHECK: %[[VAL_44:.*]] = fir.convert %[[VAL_39]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_45:.*]] = fir.convert %[[VAL_43]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_46:.*]] = arith.andi %[[VAL_44]], %[[VAL_45]] : i1 +! CHECK: %[[VAL_47:.*]] = fir.convert %[[VAL_46]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_47]] to %[[VAL_28]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_48:.*]] = fir.load %[[VAL_29]]#0 : !fir.ref> +! CHECK: %[[VAL_49:.*]] = fir.load %[[VAL_19]]#0 : !fir.ref +! CHECK: %[[VAL_50:.*]] = fir.convert %[[VAL_49]] : (i32) -> i64 +! CHECK: %[[VAL_51:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_50]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_52:.*]] = fir.load %[[VAL_51]] : !fir.ref> +! CHECK: %[[VAL_53:.*]] = fir.convert %[[VAL_48]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_54:.*]] = fir.convert %[[VAL_52]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_55:.*]] = arith.andi %[[VAL_53]], %[[VAL_54]] : i1 +! CHECK: %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_56]] to %[[VAL_29]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + + + +subroutine multiple_reductions(w) + logical :: x,y,z,w(100) + x = .true. + y = .true. + z = .true. + !$omp parallel + !$omp do reduction(.and.:x,y,z) + do i=1, 100 + x = x .and. w(i) + y = y .and. w(i) + z = z .and. w(i) + end do + !$omp end do + !$omp end parallel +end subroutine + diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 new file mode 100644 index 000000000000..4c159f3a69f9 --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 @@ -0,0 +1,202 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + +! CHECK-LABEL: omp.reduction.declare @eqv_reduction : !fir.ref> +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref>): +! CHECK: %[[VAL_1:.*]] = arith.constant true +! CHECK: %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4> +! CHECK: %[[REF:.*]] = fir.alloca !fir.logical<4> +! CHECK: fir.store %[[VAL_2]] to %[[REF]] : !fir.ref> +! CHECK: omp.yield(%[[REF]] : !fir.ref>) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref> +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref> +! CHECK: %[[VAL_2:.*]] = fir.convert %[[LD0]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_3:.*]] = fir.convert %[[LD1]] : (!fir.logical<4>) -> i1 +! CHECK: %[[RES:.*]] = arith.cmpi eq, %[[VAL_2]], %[[VAL_3]] : i1 +! CHECK: %[[VAL_5:.*]] = fir.convert %[[RES]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_5]] to %[[ARG0]] : !fir.ref> +! CHECK: omp.yield(%[[ARG0]] : !fir.ref>) +! CHECK: } + +! CHECK-LABEL: func.func @_QPsimple_reduction( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_reductionEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFsimple_reductionEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_5:.*]] = arith.constant 100 : index +! CHECK: %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1> +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref>>, !fir.shape<1>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_8:.*]] = arith.constant true +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_12:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_13:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref>) for (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) { +! CHECK: fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref +! CHECK: %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref> +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref +! CHECK: %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (i32) -> i64 +! CHECK: %[[VAL_21:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_20]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref> +! CHECK: %[[VAL_23:.*]] = fir.convert %[[VAL_18]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_24:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_25:.*]] = arith.cmpi eq, %[[VAL_23]], %[[VAL_24]] : i1 +! CHECK: %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_reduction(y) + logical :: x, y(100) + x = .true. + !$omp parallel + !$omp do reduction(.eqv.:x) + do i=1, 100 + x = x .eqv. y(i) + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPsimple_reduction_switch_order( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_reduction_switch_orderEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFsimple_reduction_switch_orderEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_5:.*]] = arith.constant 100 : index +! CHECK: %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1> +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref>>, !fir.shape<1>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_8:.*]] = arith.constant true +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_12:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_13:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref>) for (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) { +! CHECK: fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref +! CHECK: %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (i32) -> i64 +! CHECK: %[[VAL_20:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_19]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref> +! CHECK: %[[VAL_22:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref> +! CHECK: %[[VAL_23:.*]] = fir.convert %[[VAL_21]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_24:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_25:.*]] = arith.cmpi eq, %[[VAL_23]], %[[VAL_24]] : i1 +! CHECK: %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_reduction_switch_order(y) + logical :: x, y(100) + x = .true. + !$omp parallel + !$omp do reduction(.eqv.:x) + do i=1, 100 + x = y(i) .eqv. x + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_reductions( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "w"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_reductionsEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = arith.constant 100 : index +! CHECK: %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1> +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref>>, !fir.shape<1>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_6:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_8:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"} +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_10:.*]] = fir.alloca !fir.logical<4> {bindc_name = "z", uniq_name = "_QFmultiple_reductionsEz"} +! CHECK: %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_12:.*]] = arith.constant true +! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_13]] to %[[VAL_7]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_14:.*]] = arith.constant true +! CHECK: %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_15]] to %[[VAL_9]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_16:.*]] = arith.constant true +! CHECK: %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_18:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_20:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_21:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_22:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]] : !fir.ref>, @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]] : !fir.ref>, @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref>) for (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) { +! CHECK: fir.store %[[VAL_26]] to %[[VAL_19]]#1 : !fir.ref +! CHECK: %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_25]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_30:.*]] = fir.load %[[VAL_27]]#0 : !fir.ref> +! CHECK: %[[VAL_31:.*]] = fir.load %[[VAL_19]]#0 : !fir.ref +! CHECK: %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (i32) -> i64 +! CHECK: %[[VAL_33:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_32]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref> +! CHECK: %[[VAL_35:.*]] = fir.convert %[[VAL_30]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_36:.*]] = fir.convert %[[VAL_34]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_37:.*]] = arith.cmpi eq, %[[VAL_35]], %[[VAL_36]] : i1 +! CHECK: %[[VAL_38:.*]] = fir.convert %[[VAL_37]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_38]] to %[[VAL_27]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_39:.*]] = fir.load %[[VAL_28]]#0 : !fir.ref> +! CHECK: %[[VAL_40:.*]] = fir.load %[[VAL_19]]#0 : !fir.ref +! CHECK: %[[VAL_41:.*]] = fir.convert %[[VAL_40]] : (i32) -> i64 +! CHECK: %[[VAL_42:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_41]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_43:.*]] = fir.load %[[VAL_42]] : !fir.ref> +! CHECK: %[[VAL_44:.*]] = fir.convert %[[VAL_39]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_45:.*]] = fir.convert %[[VAL_43]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_46:.*]] = arith.cmpi eq, %[[VAL_44]], %[[VAL_45]] : i1 +! CHECK: %[[VAL_47:.*]] = fir.convert %[[VAL_46]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_47]] to %[[VAL_28]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_48:.*]] = fir.load %[[VAL_29]]#0 : !fir.ref> +! CHECK: %[[VAL_49:.*]] = fir.load %[[VAL_19]]#0 : !fir.ref +! CHECK: %[[VAL_50:.*]] = fir.convert %[[VAL_49]] : (i32) -> i64 +! CHECK: %[[VAL_51:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_50]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_52:.*]] = fir.load %[[VAL_51]] : !fir.ref> +! CHECK: %[[VAL_53:.*]] = fir.convert %[[VAL_48]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_54:.*]] = fir.convert %[[VAL_52]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_55:.*]] = arith.cmpi eq, %[[VAL_53]], %[[VAL_54]] : i1 +! CHECK: %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_56]] to %[[VAL_29]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine multiple_reductions(w) + logical :: x,y,z,w(100) + x = .true. + y = .true. + z = .true. + !$omp parallel + !$omp do reduction(.eqv.:x,y,z) + do i=1, 100 + x = x .eqv. w(i) + y = y .eqv. w(i) + z = z .eqv. w(i) + end do + !$omp end do + !$omp end parallel +end subroutine diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 new file mode 100644 index 000000000000..cfa8e47d3ca7 --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 @@ -0,0 +1,207 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + +! CHECK-LABEL: omp.reduction.declare @neqv_reduction : !fir.ref> +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref>): +! CHECK: %[[VAL_1:.*]] = arith.constant false +! CHECK: %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4> +! CHECK: %[[REF:.*]] = fir.alloca !fir.logical<4> +! CHECK: fir.store %[[VAL_2]] to %[[REF]] : !fir.ref> +! CHECK: omp.yield(%[[REF]] : !fir.ref>) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref> +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref> +! CHECK: %[[VAL_2:.*]] = fir.convert %[[LD0]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_3:.*]] = fir.convert %[[LD1]] : (!fir.logical<4>) -> i1 +! CHECK: %[[RES:.*]] = arith.cmpi ne, %[[VAL_2]], %[[VAL_3]] : i1 +! CHECK: %[[VAL_5:.*]] = fir.convert %[[RES]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_5]] to %[[ARG0]] : !fir.ref> +! CHECK: omp.yield(%[[ARG0]] : !fir.ref>) +! CHECK: } + +! CHECK-LABEL: func.func @_QPsimple_reduction( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_reductionEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFsimple_reductionEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_5:.*]] = arith.constant 100 : index +! CHECK: %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1> +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref>>, !fir.shape<1>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_8:.*]] = arith.constant true +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_12:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_13:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref>) for (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) { +! CHECK: fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref +! CHECK: %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref> +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref +! CHECK: %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (i32) -> i64 +! CHECK: %[[VAL_21:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_20]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref> +! CHECK: %[[VAL_23:.*]] = fir.convert %[[VAL_18]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_24:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_25:.*]] = arith.cmpi ne, %[[VAL_23]], %[[VAL_24]] : i1 +! CHECK: %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_reduction(y) + logical :: x, y(100) + x = .true. + !$omp parallel + !$omp do reduction(.neqv.:x) + do i=1, 100 + x = x .neqv. y(i) + end do + !$omp end do + !$omp end parallel +end subroutine + + +! CHECK-LABEL: func.func @_QPsimple_reduction_switch_order( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_reduction_switch_orderEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFsimple_reduction_switch_orderEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_5:.*]] = arith.constant 100 : index +! CHECK: %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1> +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref>>, !fir.shape<1>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_8:.*]] = arith.constant true +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_12:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_13:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref>) for (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) { +! CHECK: fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref +! CHECK: %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (i32) -> i64 +! CHECK: %[[VAL_20:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_19]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref> +! CHECK: %[[VAL_22:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref> +! CHECK: %[[VAL_23:.*]] = fir.convert %[[VAL_21]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_24:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_25:.*]] = arith.cmpi ne, %[[VAL_23]], %[[VAL_24]] : i1 +! CHECK: %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + + +subroutine simple_reduction_switch_order(y) + logical :: x, y(100) + x = .true. + !$omp parallel + !$omp do reduction(.neqv.:x) + do i=1, 100 + x = y(i) .neqv. x + end do + !$omp end do + !$omp end parallel +end subroutine + + +! CHECK-LABEL: func.func @_QPmultiple_reductions( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "w"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_reductionsEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = arith.constant 100 : index +! CHECK: %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1> +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref>>, !fir.shape<1>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_6:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_8:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"} +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_10:.*]] = fir.alloca !fir.logical<4> {bindc_name = "z", uniq_name = "_QFmultiple_reductionsEz"} +! CHECK: %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_12:.*]] = arith.constant true +! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_13]] to %[[VAL_7]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_14:.*]] = arith.constant true +! CHECK: %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_15]] to %[[VAL_9]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_16:.*]] = arith.constant true +! CHECK: %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_18:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_20:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_21:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_22:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]] : !fir.ref>, @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]] : !fir.ref>, @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref>) for (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) { +! CHECK: fir.store %[[VAL_26]] to %[[VAL_19]]#1 : !fir.ref +! CHECK: %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_25]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_30:.*]] = fir.load %[[VAL_27]]#0 : !fir.ref> +! CHECK: %[[VAL_31:.*]] = fir.load %[[VAL_19]]#0 : !fir.ref +! CHECK: %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (i32) -> i64 +! CHECK: %[[VAL_33:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_32]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref> +! CHECK: %[[VAL_35:.*]] = fir.convert %[[VAL_30]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_36:.*]] = fir.convert %[[VAL_34]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_37:.*]] = arith.cmpi ne, %[[VAL_35]], %[[VAL_36]] : i1 +! CHECK: %[[VAL_38:.*]] = fir.convert %[[VAL_37]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_38]] to %[[VAL_27]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_39:.*]] = fir.load %[[VAL_28]]#0 : !fir.ref> +! CHECK: %[[VAL_40:.*]] = fir.load %[[VAL_19]]#0 : !fir.ref +! CHECK: %[[VAL_41:.*]] = fir.convert %[[VAL_40]] : (i32) -> i64 +! CHECK: %[[VAL_42:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_41]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_43:.*]] = fir.load %[[VAL_42]] : !fir.ref> +! CHECK: %[[VAL_44:.*]] = fir.convert %[[VAL_39]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_45:.*]] = fir.convert %[[VAL_43]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_46:.*]] = arith.cmpi ne, %[[VAL_44]], %[[VAL_45]] : i1 +! CHECK: %[[VAL_47:.*]] = fir.convert %[[VAL_46]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_47]] to %[[VAL_28]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_48:.*]] = fir.load %[[VAL_29]]#0 : !fir.ref> +! CHECK: %[[VAL_49:.*]] = fir.load %[[VAL_19]]#0 : !fir.ref +! CHECK: %[[VAL_50:.*]] = fir.convert %[[VAL_49]] : (i32) -> i64 +! CHECK: %[[VAL_51:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_50]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_52:.*]] = fir.load %[[VAL_51]] : !fir.ref> +! CHECK: %[[VAL_53:.*]] = fir.convert %[[VAL_48]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_54:.*]] = fir.convert %[[VAL_52]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_55:.*]] = arith.cmpi ne, %[[VAL_53]], %[[VAL_54]] : i1 +! CHECK: %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_56]] to %[[VAL_29]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return +! CHECK: } + + +subroutine multiple_reductions(w) + logical :: x,y,z,w(100) + x = .true. + y = .true. + z = .true. + !$omp parallel + !$omp do reduction(.neqv.:x,y,z) + do i=1, 100 + x = x .neqv. w(i) + y = y .neqv. w(i) + z = z .neqv. w(i) + end do + !$omp end do + !$omp end parallel +end subroutine diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 new file mode 100644 index 000000000000..c71ea02f9373 --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 @@ -0,0 +1,204 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + +! CHECK-LABEL: omp.reduction.declare @or_reduction : !fir.ref> +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref>): +! CHECK: %[[VAL_1:.*]] = arith.constant false +! CHECK: %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4> +! CHECK: %[[REF:.*]] = fir.alloca !fir.logical<4> +! CHECK: fir.store %[[VAL_2]] to %[[REF]] : !fir.ref> +! CHECK: omp.yield(%[[REF]] : !fir.ref>) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref> +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref> +! CHECK: %[[VAL_2:.*]] = fir.convert %[[LD0]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_3:.*]] = fir.convert %[[LD1]] : (!fir.logical<4>) -> i1 +! CHECK: %[[RES:.*]] = arith.ori %[[VAL_2]], %[[VAL_3]] : i1 +! CHECK: %[[VAL_5:.*]] = fir.convert %[[RES]] : (i1) -> !fir.logical<4> +! CHECK: fir.store %[[VAL_5]] to %[[ARG0]] : !fir.ref> +! CHECK: omp.yield(%[[ARG0]] : !fir.ref>) + +! CHECK-LABEL: func.func @_QPsimple_reduction( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_reductionEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFsimple_reductionEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_5:.*]] = arith.constant 100 : index +! CHECK: %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1> +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref>>, !fir.shape<1>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_8:.*]] = arith.constant true +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_12:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_13:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref>) for (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) { +! CHECK: fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref +! CHECK: %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref> +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref +! CHECK: %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (i32) -> i64 +! CHECK: %[[VAL_21:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_20]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref> +! CHECK: %[[VAL_23:.*]] = fir.convert %[[VAL_18]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_24:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_25:.*]] = arith.ori %[[VAL_23]], %[[VAL_24]] : i1 +! CHECK: %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_reduction(y) + logical :: x, y(100) + x = .true. + !$omp parallel + !$omp do reduction(.or.:x) + do i=1, 100 + x = x .or. y(i) + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPsimple_reduction_switch_order( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_reduction_switch_orderEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFsimple_reduction_switch_orderEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_5:.*]] = arith.constant 100 : index +! CHECK: %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1> +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref>>, !fir.shape<1>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_8:.*]] = arith.constant true +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_12:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_13:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref>) for (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) { +! CHECK: fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref +! CHECK: %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (i32) -> i64 +! CHECK: %[[VAL_20:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_19]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref> +! CHECK: %[[VAL_22:.*]] = fir.load %[[VAL_17]]#0 : !fir.ref> +! CHECK: %[[VAL_23:.*]] = fir.convert %[[VAL_21]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_24:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_25:.*]] = arith.ori %[[VAL_23]], %[[VAL_24]] : i1 +! CHECK: %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_reduction_switch_order(y) + logical :: x, y(100) + x = .true. + !$omp parallel + !$omp do reduction(.or.:x) + do i=1, 100 + x = y(i) .or. x + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_reductions( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>> {fir.bindc_name = "w"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_reductionsEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = arith.constant 100 : index +! CHECK: %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1> +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref>>, !fir.shape<1>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_6:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_8:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"} +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_10:.*]] = fir.alloca !fir.logical<4> {bindc_name = "z", uniq_name = "_QFmultiple_reductionsEz"} +! CHECK: %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_12:.*]] = arith.constant true +! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_13]] to %[[VAL_7]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_14:.*]] = arith.constant true +! CHECK: %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_15]] to %[[VAL_9]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_16:.*]] = arith.constant true +! CHECK: %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.parallel { +! CHECK: %[[VAL_18:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_20:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_21:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_22:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]] : !fir.ref>, @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]] : !fir.ref>, @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref>) for (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) { +! CHECK: fir.store %[[VAL_26]] to %[[VAL_19]]#1 : !fir.ref +! CHECK: %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_25]] {uniq_name = "_QFmultiple_reductionsEz"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_30:.*]] = fir.load %[[VAL_27]]#0 : !fir.ref> +! CHECK: %[[VAL_31:.*]] = fir.load %[[VAL_19]]#0 : !fir.ref +! CHECK: %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (i32) -> i64 +! CHECK: %[[VAL_33:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_32]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref> +! CHECK: %[[VAL_35:.*]] = fir.convert %[[VAL_30]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_36:.*]] = fir.convert %[[VAL_34]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_37:.*]] = arith.ori %[[VAL_35]], %[[VAL_36]] : i1 +! CHECK: %[[VAL_38:.*]] = fir.convert %[[VAL_37]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_38]] to %[[VAL_27]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_39:.*]] = fir.load %[[VAL_28]]#0 : !fir.ref> +! CHECK: %[[VAL_40:.*]] = fir.load %[[VAL_19]]#0 : !fir.ref +! CHECK: %[[VAL_41:.*]] = fir.convert %[[VAL_40]] : (i32) -> i64 +! CHECK: %[[VAL_42:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_41]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_43:.*]] = fir.load %[[VAL_42]] : !fir.ref> +! CHECK: %[[VAL_44:.*]] = fir.convert %[[VAL_39]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_45:.*]] = fir.convert %[[VAL_43]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_46:.*]] = arith.ori %[[VAL_44]], %[[VAL_45]] : i1 +! CHECK: %[[VAL_47:.*]] = fir.convert %[[VAL_46]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_47]] to %[[VAL_28]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: %[[VAL_48:.*]] = fir.load %[[VAL_29]]#0 : !fir.ref> +! CHECK: %[[VAL_49:.*]] = fir.load %[[VAL_19]]#0 : !fir.ref +! CHECK: %[[VAL_50:.*]] = fir.convert %[[VAL_49]] : (i32) -> i64 +! CHECK: %[[VAL_51:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_50]]) : (!fir.ref>>, i64) -> !fir.ref> +! CHECK: %[[VAL_52:.*]] = fir.load %[[VAL_51]] : !fir.ref> +! CHECK: %[[VAL_53:.*]] = fir.convert %[[VAL_48]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_54:.*]] = fir.convert %[[VAL_52]] : (!fir.logical<4>) -> i1 +! CHECK: %[[VAL_55:.*]] = arith.ori %[[VAL_53]], %[[VAL_54]] : i1 +! CHECK: %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (i1) -> !fir.logical<4> +! CHECK: hlfir.assign %[[VAL_56]] to %[[VAL_29]]#0 : !fir.logical<4>, !fir.ref> +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + + + +subroutine multiple_reductions(w) + logical :: x,y,z,w(100) + x = .true. + y = .true. + z = .true. + !$omp parallel + !$omp do reduction(.or.:x,y,z) + do i=1, 100 + x = x .or. w(i) + y = y .or. w(i) + z = z .or. w(i) + end do + !$omp end do + !$omp end parallel +end subroutine + diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90 new file mode 100644 index 000000000000..360cd34df2d1 --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90 @@ -0,0 +1,20 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction -o - %s 2>&1 | FileCheck %s + +! CHECK: omp.wsloop byref reduction(@max_i_32 +! CHECK: arith.cmpi sgt +! CHECK: arith.select + +module m1 + intrinsic max +end module m1 +program main + use m1, ren=>max + n=0 + !$omp parallel do reduction(ren:n) + do i=1,100 + n=max(n,i) + end do + if (n/=100) print *,101 + print *,'pass' +end program main diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 new file mode 100644 index 000000000000..f9bffc269bb8 --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 @@ -0,0 +1,152 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction -o - %s 2>&1 | FileCheck %s + +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + +!CHECK: omp.reduction.declare @max_f_32_byref : !fir.ref +!CHECK-SAME: init { +!CHECK: %[[MINIMUM_VAL:.*]] = arith.constant -3.40282347E+38 : f32 +!CHECK: %[[REF:.*]] = fir.alloca f32 +!CHECK: fir.store %[[MINIMUM_VAL]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) +!CHECK: combiner +!CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +!CHECK: %[[RES:.*]] = arith.maximumf %[[LD0]], %[[LD1]] {{.*}}: f32 +!CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +!CHECK: omp.yield(%[[ARG0]] : !fir.ref) + +!CHECK-LABEL: omp.reduction.declare @max_i_32_byref : !fir.ref +!CHECK-SAME: init { +!CHECK: %[[MINIMUM_VAL:.*]] = arith.constant -2147483648 : i32 +!CHECK: %[[REF:.*]] = fir.alloca i32 +!CHECK: fir.store %[[MINIMUM_VAL]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) +!CHECK: combiner +!CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +!CHECK: %[[RES:.*]] = arith.maxsi %[[LD0]], %[[LD1]] : i32 +!CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +!CHECK: omp.yield(%[[ARG0]] : !fir.ref) + +! CHECK-LABEL: func.func @_QPreduction_max_int( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFreduction_max_intEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_max_intEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_max_intEy"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i32 +! CHECK: hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@max_i_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref +! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref +! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64 +! CHECK: %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]]) : (!fir.box>, i64) -> !fir.ref +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_17]] : !fir.ref +! CHECK: %[[VAL_20:.*]] = arith.cmpi sgt, %[[VAL_18]], %[[VAL_19]] : i32 +! CHECK: %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : i32 +! CHECK: hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : i32, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator + +! CHECK-LABEL: func.func @_QPreduction_max_real( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFreduction_max_realEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFreduction_max_realEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_max_realEy"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@max_f_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref +! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref +! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64 +! CHECK: %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]]) : (!fir.box>, i64) -> !fir.ref +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_17]] : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref +! CHECK: %[[VAL_20:.*]] = arith.cmpf ogt, %[[VAL_18]], %[[VAL_19]] fastmath : f32 +! CHECK: %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : f32 +! CHECK: hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : f32, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: omp.parallel { +! CHECK: %[[VAL_30:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_32:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_33:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_34:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@max_f_32_byref %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref) for (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) { +! CHECK: fir.store %[[VAL_36]] to %[[VAL_31]]#1 : !fir.ref +! CHECK: %[[VAL_37:.*]]:2 = hlfir.declare %[[VAL_35]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_38:.*]] = fir.load %[[VAL_31]]#0 : !fir.ref +! CHECK: %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (i32) -> i64 +! CHECK: %[[VAL_40:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_39]]) : (!fir.box>, i64) -> !fir.ref +! CHECK: %[[VAL_41:.*]] = fir.load %[[VAL_40]] : !fir.ref +! CHECK: %[[VAL_42:.*]] = fir.load %[[VAL_37]]#0 : !fir.ref +! CHECK: %[[VAL_43:.*]] = arith.cmpf ogt, %[[VAL_41]], %[[VAL_42]] fastmath : f32 +! CHECK: fir.if %[[VAL_43]] { +! CHECK: %[[VAL_44:.*]] = fir.load %[[VAL_31]]#0 : !fir.ref +! CHECK: %[[VAL_45:.*]] = fir.convert %[[VAL_44]] : (i32) -> i64 +! CHECK: %[[VAL_46:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_45]]) : (!fir.box>, i64) -> !fir.ref +! CHECK: %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref +! CHECK: hlfir.assign %[[VAL_47]] to %[[VAL_37]]#0 : f32, !fir.ref +! CHECK: } else { +! CHECK: } +! CHECK: omp.yield +! CHECK: omp.terminator + + + +subroutine reduction_max_int(y) + integer :: x, y(:) + x = 0 + !$omp parallel + !$omp do reduction(max:x) + do i=1, 100 + x = max(x, y(i)) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine + +subroutine reduction_max_real(y) + real :: x, y(:) + x = 0.0 + !$omp parallel + !$omp do reduction(max:x) + do i=1, 100 + x = max(y(i), x) + end do + !$omp end do + !$omp end parallel + print *, x + + !$omp parallel + !$omp do reduction(max:x) + do i=1, 100 + if (y(i) .gt. x) x = y(i) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90 new file mode 100644 index 000000000000..a296ce47f20f --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90 @@ -0,0 +1,62 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction -o - %s 2>&1 | FileCheck %s + +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + +! CHECK-LABEL: omp.reduction.declare @max_i_32_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[MINIMUM_VAL:.*]] = arith.constant -2147483648 : i32 +! CHECK: %[[REF:.*]] = fir.alloca i32 +! CHECK: fir.store %[[MINIMUM_VAL]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) +! CHECK: combiner +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.maxsi %[[LD0]], %[[LD1]] : i32 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) + +! CHECK-LABEL: func.func @_QPreduction_max_int( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFreduction_max_intEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_max_intEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_max_intEy"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i32 +! CHECK: hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@max_i_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref +! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref +! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64 +! CHECK: %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]]) : (!fir.box>, i64) -> !fir.ref +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_17]] : !fir.ref +! CHECK: %[[VAL_20:.*]] = arith.cmpi sgt, %[[VAL_18]], %[[VAL_19]] : i32 +! CHECK: %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : i32 +! CHECK: hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : i32, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator + + +subroutine reduction_max_int(y) + integer :: x, y(:) + x = 0 + !$omp parallel + !$omp do reduction(max:x) + do i=1, 100 + x = max(x, y(i)) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 new file mode 100644 index 000000000000..da9e686362b6 --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 @@ -0,0 +1,154 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction -o - %s 2>&1 | FileCheck %s + +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + +!CHECK: omp.reduction.declare @min_f_32_byref : !fir.ref +!CHECK-SAME: init { +!CHECK: %[[MAXIMUM_VAL:.*]] = arith.constant 3.40282347E+38 : f32 +!CHECK: %[[REF:.*]] = fir.alloca f32 +!CHECK: fir.store %[[MAXIMUM_VAL]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) +!CHECK: combiner +!CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +!CHECK: %[[RES:.*]] = arith.minimumf %[[LD0]], %[[LD1]] {{.*}}: f32 +!CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +!CHECK: omp.yield(%[[ARG0]] : !fir.ref) + +!CHECK-LABEL: omp.reduction.declare @min_i_32_byref : !fir.ref +!CHECK-SAME: init { +!CHECK: %[[MAXIMUM_VAL:.*]] = arith.constant 2147483647 : i32 +!CHECK: %[[REF:.*]] = fir.alloca i32 +!CHECK: fir.store %[[MAXIMUM_VAL]] to %[[REF]] : !fir.ref +!CHECK: omp.yield(%[[REF]] : !fir.ref) +!CHECK: combiner +!CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +!CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +!CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +!CHECK: %[[RES:.*]] = arith.minsi %[[LD0]], %[[LD1]] : i32 +!CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +!CHECK: omp.yield(%[[ARG0]] : !fir.ref) + +! CHECK-LABEL: func.func @_QPreduction_min_int( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFreduction_min_intEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_min_intEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_min_intEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_min_intEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_min_intEy"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i32 +! CHECK: hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_intEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@min_i_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref +! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_intEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref +! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64 +! CHECK: %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]]) : (!fir.box>, i64) -> !fir.ref +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_17]] : !fir.ref +! CHECK: %[[VAL_20:.*]] = arith.cmpi slt, %[[VAL_18]], %[[VAL_19]] : i32 +! CHECK: %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : i32 +! CHECK: hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : i32, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator + +! CHECK-LABEL: func.func @_QPreduction_min_real( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "y"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFreduction_min_realEi"} +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFreduction_min_realEx"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_min_realEy"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@min_f_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref +! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref +! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64 +! CHECK: %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]]) : (!fir.box>, i64) -> !fir.ref +! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_17]] : !fir.ref +! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref +! CHECK: %[[VAL_20:.*]] = arith.cmpf olt, %[[VAL_18]], %[[VAL_19]] fastmath : f32 +! CHECK: %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : f32 +! CHECK: hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : f32, !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } +! CHECK: omp.parallel { +! CHECK: %[[VAL_30:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_32:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_33:.*]] = arith.constant 100 : i32 +! CHECK: %[[VAL_34:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@min_f_32_byref %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref) for (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) { +! CHECK: fir.store %[[VAL_36]] to %[[VAL_31]]#1 : !fir.ref +! CHECK: %[[VAL_37:.*]]:2 = hlfir.declare %[[VAL_35]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_38:.*]] = fir.load %[[VAL_31]]#0 : !fir.ref +! CHECK: %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (i32) -> i64 +! CHECK: %[[VAL_40:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_39]]) : (!fir.box>, i64) -> !fir.ref +! CHECK: %[[VAL_41:.*]] = fir.load %[[VAL_40]] : !fir.ref +! CHECK: %[[VAL_42:.*]] = fir.load %[[VAL_37]]#0 : !fir.ref +! CHECK: %[[VAL_43:.*]] = arith.cmpf ogt, %[[VAL_41]], %[[VAL_42]] fastmath : f32 +! CHECK: fir.if %[[VAL_43]] { +! CHECK: %[[VAL_44:.*]] = fir.load %[[VAL_31]]#0 : !fir.ref +! CHECK: %[[VAL_45:.*]] = fir.convert %[[VAL_44]] : (i32) -> i64 +! CHECK: %[[VAL_46:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_45]]) : (!fir.box>, i64) -> !fir.ref +! CHECK: %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref +! CHECK: hlfir.assign %[[VAL_47]] to %[[VAL_37]]#0 : f32, !fir.ref +! CHECK: } else { +! CHECK: } +! CHECK: omp.yield +! CHECK: omp.terminator + + + +subroutine reduction_min_int(y) + integer :: x, y(:) + x = 0 + !$omp parallel + !$omp do reduction(min:x) + do i=1, 100 + x = min(x, y(i)) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine + +subroutine reduction_min_real(y) + real :: x, y(:) + x = 0.0 + !$omp parallel + !$omp do reduction(min:x) + do i=1, 100 + x = min(y(i), x) + end do + !$omp end do + !$omp end parallel + print *, x + + !$omp parallel + !$omp do reduction(min:x) + do i=1, 100 + if (y(i) .gt. x) x = y(i) + end do + !$omp end do + !$omp end parallel + print *, x +end subroutine diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 new file mode 100644 index 000000000000..9289973bae20 --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 @@ -0,0 +1,41 @@ +! RUN: bbc -emit-hlfir -fopenmp -o - %s | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s | FileCheck %s + +! regression test for crash + +program reduce +integer :: i = 0 +integer :: r = 0 + +!$omp parallel do reduction(min:r) +do i=0,10 + r = i +enddo +!$omp end parallel do + +print *,r + +end program + +! TODO: the reduction is not curently lowered correctly. This test is checking +! that we do not crash and we still produce the same broken IR as before. + +! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "reduce"} { +! CHECK: %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFEr"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: omp.parallel { +! CHECK: %[[VAL_4:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i32 +! CHECK: %[[VAL_7:.*]] = arith.constant 10 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop for (%[[VAL_9:.*]]) : i32 = (%[[VAL_6]]) to (%[[VAL_7]]) inclusive step (%[[VAL_8]]) { +! CHECK: fir.store %[[VAL_9]] to %[[VAL_5]]#1 : !fir.ref +! CHECK: %[[VAL_10:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref +! CHECK: hlfir.assign %[[VAL_10]] to %[[VAL_3]]#0 : i32, !fir.ref +! CHECK: omp.yield +! CHECK: } +! CHECK: omp.terminator +! CHECK: } diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90 new file mode 100644 index 000000000000..00854281b87e --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90 @@ -0,0 +1,414 @@ +! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s + + +! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py + +! CHECK-LABEL: omp.reduction.declare @multiply_reduction_f_64_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[VAL_1:.*]] = arith.constant 1.000000e+00 : f64 +! CHECK: %[[REF:.*]] = fir.alloca f64 +! CHECK: fir.store %[[VAL_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.mulf %[[LD0]], %[[LD1]] fastmath : f64 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: omp.reduction.declare @multiply_reduction_i_64_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[VAL_1:.*]] = arith.constant 1 : i64 +! CHECK: %[[REF:.*]] = fir.alloca i64 +! CHECK: fir.store %[[VAL_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.muli %[[LD0]], %[[LD1]] : i64 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: omp.reduction.declare @multiply_reduction_f_32_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[VAL_1:.*]] = arith.constant 1.000000e+00 : f32 +! CHECK: %[[REF:.*]] = fir.alloca f32 +! CHECK: fir.store %[[VAL_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.mulf %[[LD0]], %[[LD1]] fastmath : f32 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: omp.reduction.declare @multiply_reduction_i_32_byref : !fir.ref +! CHECK-SAME: init { +! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): +! CHECK: %[[VAL_1:.*]] = arith.constant 1 : i32 +! CHECK: %[[REF:.*]] = fir.alloca i32 +! CHECK: fir.store %[[VAL_1]] to %[[REF]] : !fir.ref +! CHECK: omp.yield(%[[REF]] : !fir.ref) + +! CHECK-LABEL: } combiner { +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): +! CHECK: %[[LD0:.*]] = fir.load %[[ARG0]] : !fir.ref +! CHECK: %[[LD1:.*]] = fir.load %[[ARG1]] : !fir.ref +! CHECK: %[[RES:.*]] = arith.muli %[[LD0]], %[[LD1]] : i32 +! CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref +! CHECK: omp.yield(%[[ARG0]] : !fir.ref) +! CHECK: } + +! CHECK-LABEL: func.func @_QPsimple_int_reduction() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_int_reductionEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFsimple_int_reductionEx"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 +! CHECK: hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 10 : i32 +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@multiply_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref +! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_14:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref +! CHECK: %[[VAL_15:.*]] = arith.muli %[[VAL_13]], %[[VAL_14]] : i32 +! CHECK: hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : i32, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_int_reduction + integer :: x + x = 1 + !$omp parallel + !$omp do reduction(*:x) + do i=1, 10 + x = x * i + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPsimple_real_reduction() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_real_reductionEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFsimple_real_reductionEx"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 10 : i32 +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@multiply_reduction_f_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref +! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_14:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref +! CHECK: %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (i32) -> f32 +! CHECK: %[[VAL_16:.*]] = arith.mulf %[[VAL_13]], %[[VAL_15]] fastmath : f32 +! CHECK: hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : f32, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_real_reduction + real :: x + x = 1.0 + !$omp parallel + !$omp do reduction(*:x) + do i=1, 10 + x = x * i + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPsimple_int_reduction_switch_order() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_int_reduction_switch_orderEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFsimple_int_reduction_switch_orderEx"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 +! CHECK: hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 10 : i32 +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@multiply_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref +! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref +! CHECK: %[[VAL_14:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_15:.*]] = arith.muli %[[VAL_13]], %[[VAL_14]] : i32 +! CHECK: hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : i32, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_int_reduction_switch_order + integer :: x + x = 1 + !$omp parallel + !$omp do reduction(*:x) + do i=1, 10 + x = i * x + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPsimple_real_reduction_switch_order() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_real_reduction_switch_orderEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFsimple_real_reduction_switch_orderEx"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_8:.*]] = arith.constant 10 : i32 +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@multiply_reduction_f_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref +! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref +! CHECK: %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (i32) -> f32 +! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_16:.*]] = arith.mulf %[[VAL_14]], %[[VAL_15]] fastmath : f32 +! CHECK: hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : f32, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine simple_real_reduction_switch_order + real :: x + x = 1.0 + !$omp parallel + !$omp do reduction(*:x) + do i=1, 10 + x = i * x + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_int_reductions_same_type() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_int_reductions_same_typeEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_int_reductions_same_typeEx"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFmultiple_int_reductions_same_typeEy"} +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFmultiple_int_reductions_same_typeEz"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_int_reductions_same_typeEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 +! CHECK: hlfir.assign %[[VAL_8]] to %[[VAL_3]]#0 : i32, !fir.ref +! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_5]]#0 : i32, !fir.ref +! CHECK: %[[VAL_10:.*]] = arith.constant 1 : i32 +! CHECK: hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : i32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 10 : i32 +! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@multiply_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @multiply_reduction_i_32_byref %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @multiply_reduction_i_32_byref %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { +! CHECK: fir.store %[[VAL_19]] to %[[VAL_12]]#1 : !fir.ref +! CHECK: %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_int_reductions_same_typeEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_23:.*]] = fir.load %[[VAL_20]]#0 : !fir.ref +! CHECK: %[[VAL_24:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_25:.*]] = arith.muli %[[VAL_23]], %[[VAL_24]] : i32 +! CHECK: hlfir.assign %[[VAL_25]] to %[[VAL_20]]#0 : i32, !fir.ref +! CHECK: %[[VAL_26:.*]] = fir.load %[[VAL_21]]#0 : !fir.ref +! CHECK: %[[VAL_27:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_28:.*]] = arith.muli %[[VAL_26]], %[[VAL_27]] : i32 +! CHECK: hlfir.assign %[[VAL_28]] to %[[VAL_21]]#0 : i32, !fir.ref +! CHECK: %[[VAL_29:.*]] = fir.load %[[VAL_22]]#0 : !fir.ref +! CHECK: %[[VAL_30:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_31:.*]] = arith.muli %[[VAL_29]], %[[VAL_30]] : i32 +! CHECK: hlfir.assign %[[VAL_31]] to %[[VAL_22]]#0 : i32, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine multiple_int_reductions_same_type + integer :: x,y,z + x = 1 + y = 1 + z = 1 + !$omp parallel + !$omp do reduction(*:x,y,z) + do i=1, 10 + x = x * i + y = y * i + z = z * i + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_real_reductions_same_type() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_real_reductions_same_typeEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFmultiple_real_reductions_same_typeEx"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = fir.alloca f32 {bindc_name = "y", uniq_name = "_QFmultiple_real_reductions_same_typeEy"} +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_6:.*]] = fir.alloca f32 {bindc_name = "z", uniq_name = "_QFmultiple_real_reductions_same_typeEz"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_real_reductions_same_typeEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_8:.*]] = arith.constant 1.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_8]] to %[[VAL_3]]#0 : f32, !fir.ref +! CHECK: %[[VAL_9:.*]] = arith.constant 1.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_5]]#0 : f32, !fir.ref +! CHECK: %[[VAL_10:.*]] = arith.constant 1.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : f32, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_14:.*]] = arith.constant 10 : i32 +! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@multiply_reduction_f_32_byref %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @multiply_reduction_f_32_byref %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @multiply_reduction_f_32_byref %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { +! CHECK: fir.store %[[VAL_19]] to %[[VAL_12]]#1 : !fir.ref +! CHECK: %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_real_reductions_same_typeEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_23:.*]] = fir.load %[[VAL_20]]#0 : !fir.ref +! CHECK: %[[VAL_24:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_25:.*]] = fir.convert %[[VAL_24]] : (i32) -> f32 +! CHECK: %[[VAL_26:.*]] = arith.mulf %[[VAL_23]], %[[VAL_25]] fastmath : f32 +! CHECK: hlfir.assign %[[VAL_26]] to %[[VAL_20]]#0 : f32, !fir.ref +! CHECK: %[[VAL_27:.*]] = fir.load %[[VAL_21]]#0 : !fir.ref +! CHECK: %[[VAL_28:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_29:.*]] = fir.convert %[[VAL_28]] : (i32) -> f32 +! CHECK: %[[VAL_30:.*]] = arith.mulf %[[VAL_27]], %[[VAL_29]] fastmath : f32 +! CHECK: hlfir.assign %[[VAL_30]] to %[[VAL_21]]#0 : f32, !fir.ref +! CHECK: %[[VAL_31:.*]] = fir.load %[[VAL_22]]#0 : !fir.ref +! CHECK: %[[VAL_32:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref +! CHECK: %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (i32) -> f32 +! CHECK: %[[VAL_34:.*]] = arith.mulf %[[VAL_31]], %[[VAL_33]] fastmath : f32 +! CHECK: hlfir.assign %[[VAL_34]] to %[[VAL_22]]#0 : f32, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + +subroutine multiple_real_reductions_same_type + real :: x,y,z + x = 1 + y = 1 + z = 1 + !$omp parallel + !$omp do reduction(*:x,y,z) + do i=1, 10 + x = x * i + y = y * i + z = z * i + end do + !$omp end do + !$omp end parallel +end subroutine + +! CHECK-LABEL: func.func @_QPmultiple_reductions_different_type() { +! CHECK: %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_reductions_different_typeEi"} +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_2:.*]] = fir.alloca f64 {bindc_name = "w", uniq_name = "_QFmultiple_reductions_different_typeEw"} +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFmultiple_reductions_different_typeEw"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_reductions_different_typeEx"} +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "y", uniq_name = "_QFmultiple_reductions_different_typeEy"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_8:.*]] = fir.alloca f32 {bindc_name = "z", uniq_name = "_QFmultiple_reductions_different_typeEz"} +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFmultiple_reductions_different_typeEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_10:.*]] = arith.constant 1 : i32 +! CHECK: hlfir.assign %[[VAL_10]] to %[[VAL_5]]#0 : i32, !fir.ref +! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i64 +! CHECK: hlfir.assign %[[VAL_11]] to %[[VAL_7]]#0 : i64, !fir.ref +! CHECK: %[[VAL_12:.*]] = arith.constant 1.000000e+00 : f32 +! CHECK: hlfir.assign %[[VAL_12]] to %[[VAL_9]]#0 : f32, !fir.ref +! CHECK: %[[VAL_13:.*]] = arith.constant 1.000000e+00 : f64 +! CHECK: hlfir.assign %[[VAL_13]] to %[[VAL_3]]#0 : f64, !fir.ref +! CHECK: omp.parallel { +! CHECK: %[[VAL_14:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} +! CHECK: %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_16:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_17:.*]] = arith.constant 10 : i32 +! CHECK: %[[VAL_18:.*]] = arith.constant 1 : i32 +! CHECK: omp.wsloop byref reduction(@multiply_reduction_i_32_byref %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref, @multiply_reduction_i_64_byref %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref, @multiply_reduction_f_32_byref %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref, @multiply_reduction_f_64_byref %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref) for (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) { +! CHECK: fir.store %[[VAL_23]] to %[[VAL_15]]#1 : !fir.ref +! CHECK: %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_26:.*]]:2 = hlfir.declare %[[VAL_21]] {uniq_name = "_QFmultiple_reductions_different_typeEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_22]] {uniq_name = "_QFmultiple_reductions_different_typeEw"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_28:.*]] = fir.load %[[VAL_24]]#0 : !fir.ref +! CHECK: %[[VAL_29:.*]] = fir.load %[[VAL_15]]#0 : !fir.ref +! CHECK: %[[VAL_30:.*]] = arith.muli %[[VAL_28]], %[[VAL_29]] : i32 +! CHECK: hlfir.assign %[[VAL_30]] to %[[VAL_24]]#0 : i32, !fir.ref +! CHECK: %[[VAL_31:.*]] = fir.load %[[VAL_25]]#0 : !fir.ref +! CHECK: %[[VAL_32:.*]] = fir.load %[[VAL_15]]#0 : !fir.ref +! CHECK: %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (i32) -> i64 +! CHECK: %[[VAL_34:.*]] = arith.muli %[[VAL_31]], %[[VAL_33]] : i64 +! CHECK: hlfir.assign %[[VAL_34]] to %[[VAL_25]]#0 : i64, !fir.ref +! CHECK: %[[VAL_35:.*]] = fir.load %[[VAL_26]]#0 : !fir.ref +! CHECK: %[[VAL_36:.*]] = fir.load %[[VAL_15]]#0 : !fir.ref +! CHECK: %[[VAL_37:.*]] = fir.convert %[[VAL_36]] : (i32) -> f32 +! CHECK: %[[VAL_38:.*]] = arith.mulf %[[VAL_35]], %[[VAL_37]] fastmath : f32 +! CHECK: hlfir.assign %[[VAL_38]] to %[[VAL_26]]#0 : f32, !fir.ref +! CHECK: %[[VAL_39:.*]] = fir.load %[[VAL_27]]#0 : !fir.ref +! CHECK: %[[VAL_40:.*]] = fir.load %[[VAL_15]]#0 : !fir.ref +! CHECK: %[[VAL_41:.*]] = fir.convert %[[VAL_40]] : (i32) -> f64 +! CHECK: %[[VAL_42:.*]] = arith.mulf %[[VAL_39]], %[[VAL_41]] fastmath : f64 +! CHECK: hlfir.assign %[[VAL_42]] to %[[VAL_27]]#0 : f64, !fir.ref +! CHECK: omp.yield +! CHECK: omp.terminator +! CHECK: return + + +subroutine multiple_reductions_different_type + integer :: x + integer(kind=8) :: y + real :: z + real(kind=8) :: w + x = 1 + y = 1 + z = 1 + w = 1 + !$omp parallel + !$omp do reduction(*:x,y,z,w) + do i=1, 10 + x = x * i + y = y * i + z = z * i + w = w * i + end do + !$omp end do + !$omp end parallel +end subroutine diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h index 5bbaa8c208b8..c9ee0c25194c 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -1339,10 +1339,12 @@ public: /// in reductions. /// \param ReductionInfos A list of info on each reduction variable. /// \param IsNoWait A flag set if the reduction is marked as nowait. + /// \param IsByRef A flag set if the reduction is using reference + /// or direct value. InsertPointTy createReductions(const LocationDescription &Loc, InsertPointTy AllocaIP, ArrayRef ReductionInfos, - bool IsNoWait = false); + bool IsNoWait = false, bool IsByRef = false); ///} diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index d65ed8c11d86..c74c898e1e88 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -2110,7 +2110,7 @@ Function *getFreshReductionFunc(Module &M) { OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions( const LocationDescription &Loc, InsertPointTy AllocaIP, - ArrayRef ReductionInfos, bool IsNoWait) { + ArrayRef ReductionInfos, bool IsNoWait, bool IsByRef) { for (const ReductionInfo &RI : ReductionInfos) { (void)RI; assert(RI.Variable && "expected non-null variable"); @@ -2197,17 +2197,29 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions( for (auto En : enumerate(ReductionInfos)) { const ReductionInfo &RI = En.value(); Type *ValueType = RI.ElementType; - Value *RedValue = Builder.CreateLoad(ValueType, RI.Variable, - "red.value." + Twine(En.index())); + // We have one less load for by-ref case because that load is now inside of + // the reduction region + Value *RedValue = nullptr; + if (!IsByRef) { + RedValue = Builder.CreateLoad(ValueType, RI.Variable, + "red.value." + Twine(En.index())); + } Value *PrivateRedValue = Builder.CreateLoad(ValueType, RI.PrivateVariable, "red.private.value." + Twine(En.index())); Value *Reduced; - Builder.restoreIP( - RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced)); + if (IsByRef) { + Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), RI.Variable, + PrivateRedValue, Reduced)); + } else { + Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), RedValue, + PrivateRedValue, Reduced)); + } if (!Builder.GetInsertBlock()) return InsertPointTy(); - Builder.CreateStore(Reduced, RI.Variable); + // for by-ref case, the load is inside of the reduction region + if (!IsByRef) + Builder.CreateStore(Reduced, RI.Variable); } Function *EndReduceFunc = getOrCreateRuntimeFunctionPtr( IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait @@ -2219,7 +2231,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions( // function. There are no loads/stores here because they will be happening // inside the atomic elementwise reduction. Builder.SetInsertPoint(AtomicRedBlock); - if (CanGenerateAtomic) { + if (CanGenerateAtomic && !IsByRef) { for (const ReductionInfo &RI : ReductionInfos) { Builder.restoreIP(RI.AtomicReductionGen(Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable)); @@ -2257,7 +2269,9 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions( Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced)); if (!Builder.GetInsertBlock()) return InsertPointTy(); - Builder.CreateStore(Reduced, LHSPtr); + // store is inside of the reduction region when using by-ref + if (!IsByRef) + Builder.CreateStore(Reduced, LHSPtr); } Builder.CreateRetVoid(); diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td index 39ff49f0ccf5..0bd402d626dc 100644 --- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td +++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td @@ -268,6 +268,9 @@ def ParallelOp : OpenMP_Op<"parallel", [ The optional $proc_bind_val attribute controls the thread affinity for the execution of the parallel region. + + The optional byref attribute controls whether reduction arguments are passed by + reference or by value. }]; let arguments = (ins Optional:$if_expr_var, @@ -278,7 +281,8 @@ def ParallelOp : OpenMP_Op<"parallel", [ OptionalAttr:$reductions, OptionalAttr:$proc_bind_val, Variadic:$private_vars, - OptionalAttr:$privatizers); + OptionalAttr:$privatizers, + UnitAttr:$byref); let regions = (region AnyRegion:$region); @@ -299,6 +303,7 @@ def ParallelOp : OpenMP_Op<"parallel", [ $allocators_vars, type($allocators_vars) ) `)` | `proc_bind` `(` custom($proc_bind_val) `)` + | `byref` $byref ) custom($region, $reduction_vars, type($reduction_vars), $reductions, $private_vars, type($private_vars), $privatizers) attr-dict @@ -570,6 +575,9 @@ def WsLoopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments, The optional `order` attribute specifies which order the iterations of the associate loops are executed in. Currently the only option for this attribute is "concurrent". + + The optional `byref` attribute indicates that reduction arguments should be + passed by reference. }]; let arguments = (ins Variadic:$lowerBound, @@ -584,6 +592,7 @@ def WsLoopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments, OptionalAttr:$schedule_modifier, UnitAttr:$simd_modifier, UnitAttr:$nowait, + UnitAttr:$byref, ConfinedAttr, [IntMinValue<0>]>:$ordered_val, OptionalAttr:$order_val, UnitAttr:$inclusive); @@ -613,6 +622,7 @@ def WsLoopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments, $schedule_val, $schedule_modifier, $simd_modifier, $schedule_chunk_var, type($schedule_chunk_var)) `)` |`nowait` $nowait + |`byref` $byref |`ordered` `(` $ordered_val `)` |`order` `(` custom($order_val) `)` ) custom($region, $lowerBound, $upperBound, $step, diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp index 8a6980e2c6a2..e7b899aec4af 100644 --- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp +++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp @@ -1209,7 +1209,7 @@ void ParallelOp::build(OpBuilder &builder, OperationState &state, /*allocate_vars=*/ValueRange(), /*allocators_vars=*/ValueRange(), /*reduction_vars=*/ValueRange(), /*reductions=*/nullptr, /*proc_bind_val=*/nullptr, /*private_vars=*/ValueRange(), - /*privatizers=*/nullptr); + /*privatizers=*/nullptr, /*byref=*/false); state.addAttributes(attributes); } @@ -1674,7 +1674,8 @@ void WsLoopOp::build(OpBuilder &builder, OperationState &state, /*linear_step_vars=*/ValueRange(), /*reduction_vars=*/ValueRange(), /*reductions=*/nullptr, /*schedule_val=*/nullptr, /*schedule_chunk_var=*/nullptr, /*schedule_modifier=*/nullptr, - /*simd_modifier=*/false, /*nowait=*/false, /*ordered_val=*/nullptr, + /*simd_modifier=*/false, /*nowait=*/false, /*byref=*/false, + /*ordered_val=*/nullptr, /*order_val=*/nullptr, /*inclusive=*/false); state.addAttributes(attributes); } diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp index bef227f2c583..5027f2afe921 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp @@ -805,12 +805,12 @@ convertOmpTaskgroupOp(omp::TaskGroupOp tgOp, llvm::IRBuilderBase &builder, /// Allocate space for privatized reduction variables. template static void -allocReductionVars(T loop, llvm::IRBuilderBase &builder, - LLVM::ModuleTranslation &moduleTranslation, - llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, - SmallVector &reductionDecls, - SmallVector &privateReductionVariables, - DenseMap &reductionVariableMap) { +allocByValReductionVars(T loop, llvm::IRBuilderBase &builder, + LLVM::ModuleTranslation &moduleTranslation, + llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, + SmallVector &reductionDecls, + SmallVector &privateReductionVariables, + DenseMap &reductionVariableMap) { llvm::IRBuilderBase::InsertPointGuard guard(builder); builder.restoreIP(allocaIP); auto args = @@ -863,6 +863,7 @@ static LogicalResult convertOmpWsLoop(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { auto loop = cast(opInst); + const bool isByRef = loop.getByref(); // TODO: this should be in the op verifier instead. if (loop.getLowerBound().empty()) return failure(); @@ -888,18 +889,17 @@ convertOmpWsLoop(Operation &opInst, llvm::IRBuilderBase &builder, SmallVector privateReductionVariables; DenseMap reductionVariableMap; - allocReductionVars(loop, builder, moduleTranslation, allocaIP, reductionDecls, - privateReductionVariables, reductionVariableMap); - - // Store the mapping between reduction variables and their private copies on - // ModuleTranslation stack. It can be then recovered when translating - // omp.reduce operations in a separate call. - LLVM::ModuleTranslation::SaveStack mappingGuard( - moduleTranslation, reductionVariableMap); + if (!isByRef) { + allocByValReductionVars(loop, builder, moduleTranslation, allocaIP, + reductionDecls, privateReductionVariables, + reductionVariableMap); + } // Before the loop, store the initial values of reductions into reduction // variables. Although this could be done after allocas, we don't want to mess // up with the alloca insertion point. + MutableArrayRef reductionArgs = + loop.getRegion().getArguments().take_back(loop.getNumReductionVars()); for (unsigned i = 0; i < loop.getNumReductionVars(); ++i) { SmallVector phis; if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(), @@ -908,9 +908,31 @@ convertOmpWsLoop(Operation &opInst, llvm::IRBuilderBase &builder, return failure(); assert(phis.size() == 1 && "expected one value to be yielded from the " "reduction neutral element declaration region"); - builder.CreateStore(phis[0], privateReductionVariables[i]); + if (isByRef) { + // Allocate reduction variable (which is a pointer to the real reduction + // variable allocated in the inlined region) + llvm::Value *var = builder.CreateAlloca( + moduleTranslation.convertType(reductionDecls[i].getType())); + // Store the result of the inlined region to the allocated reduction var + // ptr + builder.CreateStore(phis[0], var); + + privateReductionVariables.push_back(var); + moduleTranslation.mapValue(reductionArgs[i], phis[0]); + reductionVariableMap.try_emplace(loop.getReductionVars()[i], phis[0]); + } else { + // for by-ref case the store is inside of the reduction region + builder.CreateStore(phis[0], privateReductionVariables[i]); + // the rest was handled in allocByValReductionVars + } } + // Store the mapping between reduction variables and their private copies on + // ModuleTranslation stack. It can be then recovered when translating + // omp.reduce operations in a separate call. + LLVM::ModuleTranslation::SaveStack mappingGuard( + moduleTranslation, reductionVariableMap); + // Set up the source location value for OpenMP runtime. llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); @@ -1014,7 +1036,7 @@ convertOmpWsLoop(Operation &opInst, llvm::IRBuilderBase &builder, builder.SetInsertPoint(tempTerminator); llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint = ompBuilder->createReductions(builder.saveIP(), allocaIP, reductionInfos, - loop.getNowait()); + loop.getNowait(), isByRef); if (!contInsertPoint.getBlock()) return loop->emitOpError() << "failed to convert reductions"; auto nextInsertionPoint = @@ -1068,6 +1090,7 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; OmpParallelOpConversionManager raii(opInst); + const bool isByRef = opInst.getByref(); // TODO: support error propagation in OpenMPIRBuilder and use it instead of // relying on captured variables. @@ -1082,18 +1105,17 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, // Allocate reduction vars SmallVector privateReductionVariables; DenseMap reductionVariableMap; - allocReductionVars(opInst, builder, moduleTranslation, allocaIP, - reductionDecls, privateReductionVariables, - reductionVariableMap); - - // Store the mapping between reduction variables and their private copies on - // ModuleTranslation stack. It can be then recovered when translating - // omp.reduce operations in a separate call. - LLVM::ModuleTranslation::SaveStack mappingGuard( - moduleTranslation, reductionVariableMap); + if (!isByRef) { + allocByValReductionVars(opInst, builder, moduleTranslation, allocaIP, + reductionDecls, privateReductionVariables, + reductionVariableMap); + } // Initialize reduction vars builder.restoreIP(allocaIP); + MutableArrayRef reductionArgs = + opInst.getRegion().getArguments().take_back( + opInst.getNumReductionVars()); for (unsigned i = 0; i < opInst.getNumReductionVars(); ++i) { SmallVector phis; if (failed(inlineConvertOmpRegions( @@ -1104,9 +1126,32 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, "expected one value to be yielded from the " "reduction neutral element declaration region"); builder.restoreIP(allocaIP); - builder.CreateStore(phis[0], privateReductionVariables[i]); + + if (isByRef) { + // Allocate reduction variable (which is a pointer to the real reduciton + // variable allocated in the inlined region) + llvm::Value *var = builder.CreateAlloca( + moduleTranslation.convertType(reductionDecls[i].getType())); + // Store the result of the inlined region to the allocated reduction var + // ptr + builder.CreateStore(phis[0], var); + + privateReductionVariables.push_back(var); + moduleTranslation.mapValue(reductionArgs[i], phis[0]); + reductionVariableMap.try_emplace(opInst.getReductionVars()[i], phis[0]); + } else { + // for by-ref case the store is inside of the reduction init region + builder.CreateStore(phis[0], privateReductionVariables[i]); + // the rest is done in allocByValReductionVars + } } + // Store the mapping between reduction variables and their private copies on + // ModuleTranslation stack. It can be then recovered when translating + // omp.reduce operations in a separate call. + LLVM::ModuleTranslation::SaveStack mappingGuard( + moduleTranslation, reductionVariableMap); + // Save the alloca insertion point on ModuleTranslation stack for use in // nested regions. LLVM::ModuleTranslation::SaveStack frame( @@ -1137,7 +1182,7 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint = ompBuilder->createReductions(builder.saveIP(), allocaIP, - reductionInfos, false); + reductionInfos, false, isByRef); if (!contInsertPoint.getBlock()) { bodyGenStatus = opInst->emitOpError() << "failed to convert reductions"; return; diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-byref.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-byref.mlir new file mode 100644 index 000000000000..4ac1ebd43e1e --- /dev/null +++ b/mlir/test/Target/LLVMIR/openmp-reduction-byref.mlir @@ -0,0 +1,66 @@ +// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s + + omp.reduction.declare @add_reduction_i_32 : !llvm.ptr init { + ^bb0(%arg0: !llvm.ptr): + %0 = llvm.mlir.constant(0 : i32) : i32 + %1 = llvm.mlir.constant(1 : i64) : i64 + %2 = llvm.alloca %1 x i32 : (i64) -> !llvm.ptr + llvm.store %0, %2 : i32, !llvm.ptr + omp.yield(%2 : !llvm.ptr) + } combiner { + ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr): + %0 = llvm.load %arg0 : !llvm.ptr -> i32 + %1 = llvm.load %arg1 : !llvm.ptr -> i32 + %2 = llvm.add %0, %1 : i32 + llvm.store %2, %arg0 : i32, !llvm.ptr + omp.yield(%arg0 : !llvm.ptr) + } + + // CHECK-LABEL: @main + llvm.func @main() { + %0 = llvm.mlir.constant(-1 : i32) : i32 + %1 = llvm.mlir.addressof @i : !llvm.ptr + omp.parallel byref reduction(@add_reduction_i_32 %1 -> %arg0 : !llvm.ptr) { + llvm.store %0, %arg0 : i32, !llvm.ptr + omp.terminator + } + llvm.return + } + llvm.mlir.global internal @i() {addr_space = 0 : i32} : i32 { + %0 = llvm.mlir.constant(0 : i32) : i32 + llvm.return %0 : i32 + } + +// CHECK: %{{.+}} = +// Call to the outlined function. +// CHECK: call void {{.*}} @__kmpc_fork_call +// CHECK-SAME: @[[OUTLINED:[A-Za-z_.][A-Za-z0-9_.]*]] + +// Outlined function. +// CHECK: define internal void @[[OUTLINED]] + +// Private reduction variable and its initialization. +// CHECK: %tid.addr.local = alloca i32 +// CHECK: %[[PRIVATE:.+]] = alloca i32 +// CHECK: store i32 0, ptr %[[PRIVATE]] +// CHECK: store ptr %[[PRIVATE]], ptr %[[PRIV_PTR:.+]], + +// Call to the reduction function. +// CHECK: call i32 @__kmpc_reduce +// CHECK-SAME: @[[REDFUNC:[A-Za-z_.][A-Za-z0-9_.]*]] + + +// Non-atomic reduction: +// CHECK: %[[PRIV_VAL_PTR:.+]] = load ptr, ptr %[[PRIV_PTR]] +// CHECK: %[[LOAD:.+]] = load i32, ptr @i +// CHECK: %[[PRIV_VAL:.+]] = load i32, ptr %[[PRIV_VAL_PTR]] +// CHECK: %[[SUM:.+]] = add i32 %[[LOAD]], %[[PRIV_VAL]] +// CHECK: store i32 %[[SUM]], ptr @i +// CHECK: call void @__kmpc_end_reduce +// CHECK: br label %[[FINALIZE:.+]] + +// CHECK: [[FINALIZE]]: + +// Reduction function. +// CHECK: define internal void @[[REDFUNC]] +// CHECK: add i32 -- GitLab From 3c227a31dd1046db02323868b9690a6152cfb3b8 Mon Sep 17 00:00:00 2001 From: Jon Roelofs Date: Wed, 13 Mar 2024 08:02:22 -0700 Subject: [PATCH 0132/1407] [cmake] Silence a duplicate libraries warning from Apple's linker (#85012) ld: warning: ignoring duplicate libraries: This triggers quite frequently in llvm's build because CMake's library depends mechanism doesn't de-duplicate libraries on the link line. Duplication is necessary for ELF platforms, but means something subtly different on Darwin platforms, hence the warning. Since we don't have much control over that from CMake, just disable the warning wholesale whenever the linker is detected to support it. --- llvm/cmake/modules/AddLLVM.cmake | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake index 374f5e085d91..eb9e6101bdce 100644 --- a/llvm/cmake/modules/AddLLVM.cmake +++ b/llvm/cmake/modules/AddLLVM.cmake @@ -257,6 +257,16 @@ if (NOT DEFINED LLVM_LINKER_DETECTED AND NOT WIN32) message(STATUS "Linker detection: unknown") endif() endif() + + # Apple's linker complains about duplicate libraries, which CMake likes to do + # to support ELF platforms. To silence that warning, we can use + # -no_warn_duplicate_libraries, but only in versions of the linker that + # support that flag. + if(NOT LLVM_USE_LINKER AND ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + check_linker_flag(C "-Wl,-no_warn_duplicate_libraries" LLVM_LINKER_SUPPORTS_NO_WARN_DUPLICATE_LIBRARIES) + else() + set(LLVM_LINKER_SUPPORTS_NO_WARN_DUPLICATE_LIBRARIES OFF CACHE INTERNAL "") + endif() endif() function(add_link_opts target_name) @@ -310,6 +320,11 @@ function(add_link_opts target_name) endif() endif() + if(LLVM_LINKER_SUPPORTS_NO_WARN_DUPLICATE_LIBRARIES) + set_property(TARGET ${target_name} APPEND_STRING PROPERTY + LINK_FLAGS " -Wl,-no_warn_duplicate_libraries") + endif() + if(ARG_SUPPORT_PLUGINS AND ${CMAKE_SYSTEM_NAME} MATCHES "AIX") set_property(TARGET ${target_name} APPEND_STRING PROPERTY LINK_FLAGS " -Wl,-brtl") -- GitLab From 628a79dad30befed82ee1c115b00fa9aca5305ed Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Wed, 13 Mar 2024 16:09:25 +0100 Subject: [PATCH 0133/1407] [InstCombine] Don't generate crash dialog for fixpoint verification failure (NFC) Fixpoint verification failures outside our tests are usually not indicative of a bug -- don't be pushy about having people report them. --- llvm/lib/Transforms/InstCombine/InstructionCombining.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 1688005de210..c9bbe437b368 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -5202,7 +5202,8 @@ static bool combineInstructionsOverFunction( if (Iteration > Opts.MaxIterations) { report_fatal_error( "Instruction Combining did not reach a fixpoint after " + - Twine(Opts.MaxIterations) + " iterations"); + Twine(Opts.MaxIterations) + " iterations", + /*GenCrashDiag=*/false); } } -- GitLab From 59ff907fc14aa2d02e57b4af4140949d4f8caca1 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 13 Mar 2024 07:45:14 -0700 Subject: [PATCH 0134/1407] [SLP]Fix PR85082: PHI node has multiple entries. Need to record casted extractelement for the externally used scalar, not original extract instruction. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 10 +-- .../X86/same-scalar-in-same-phi-extract.ll | 75 +++++++++++++++++++ 2 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index b8b67609d755..5e0f5b7efadc 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -12592,6 +12592,11 @@ Value *BoUpSLP::vectorizeTree( } else { Ex = Builder.CreateExtractElement(Vec, Lane); } + // If necessary, sign-extend or zero-extend ScalarRoot + // to the larger type. + if (Scalar->getType() != Ex->getType()) + Ex = Builder.CreateIntCast(Ex, Scalar->getType(), + MinBWs.find(E)->second.second); if (auto *I = dyn_cast(Ex)) ScalarToEEs[Scalar].try_emplace(Builder.GetInsertBlock(), I); } @@ -12601,11 +12606,6 @@ Value *BoUpSLP::vectorizeTree( GatherShuffleExtractSeq.insert(ExI); CSEBlocks.insert(ExI->getParent()); } - // If necessary, sign-extend or zero-extend ScalarRoot - // to the larger type. - if (Scalar->getType() != Ex->getType()) - return Builder.CreateIntCast(Ex, Scalar->getType(), - MinBWs.find(E)->second.second); return Ex; } assert(isa(Scalar->getType()) && diff --git a/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll b/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll new file mode 100644 index 000000000000..35f2f9e052e7 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll @@ -0,0 +1,75 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S --passes=slp-vectorizer -slp-threshold=-99999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s + +define void @test(i32 %arg) { +; CHECK-LABEL: define void @test( +; CHECK-SAME: i32 [[ARG:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> , i32 [[ARG]], i32 0 +; CHECK-NEXT: br label [[BB2:%.*]] +; CHECK: bb2: +; CHECK-NEXT: switch i32 0, label [[BB10:%.*]] [ +; CHECK-NEXT: i32 0, label [[BB9:%.*]] +; CHECK-NEXT: i32 11, label [[BB9]] +; CHECK-NEXT: i32 1, label [[BB4:%.*]] +; CHECK-NEXT: ] +; CHECK: bb3: +; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0 +; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 +; CHECK-NEXT: switch i32 0, label [[BB10]] [ +; CHECK-NEXT: i32 18, label [[BB7:%.*]] +; CHECK-NEXT: i32 1, label [[BB7]] +; CHECK-NEXT: i32 0, label [[BB10]] +; CHECK-NEXT: ] +; CHECK: bb4: +; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x i32> [ [[TMP0]], [[BB2]] ] +; CHECK-NEXT: [[TMP4:%.*]] = zext <2 x i32> [[TMP3]] to <2 x i64> +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0 +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr null, i64 [[TMP5]] +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1 +; CHECK-NEXT: [[GETELEMENTPTR6:%.*]] = getelementptr i32, ptr null, i64 [[TMP6]] +; CHECK-NEXT: ret void +; CHECK: bb7: +; CHECK-NEXT: [[PHI8:%.*]] = phi i64 [ [[TMP2]], [[BB3:%.*]] ], [ [[TMP2]], [[BB3]] ] +; CHECK-NEXT: br label [[BB9]] +; CHECK: bb9: +; CHECK-NEXT: ret void +; CHECK: bb10: +; CHECK-NEXT: ret void +; +bb: + %zext = zext i32 %arg to i64 + %zext1 = zext i32 0 to i64 + br label %bb2 + +bb2: + switch i32 0, label %bb10 [ + i32 0, label %bb9 + i32 11, label %bb9 + i32 1, label %bb4 + ] + +bb3: + switch i32 0, label %bb10 [ + i32 18, label %bb7 + i32 1, label %bb7 + i32 0, label %bb10 + ] + +bb4: + %phi = phi i64 [ %zext, %bb2 ] + %phi5 = phi i64 [ %zext1, %bb2 ] + %getelementptr = getelementptr i32, ptr null, i64 %phi + %getelementptr6 = getelementptr i32, ptr null, i64 %phi5 + ret void + +bb7: + %phi8 = phi i64 [ %zext, %bb3 ], [ %zext, %bb3 ] + br label %bb9 + +bb9: + ret void + +bb10: + ret void +} -- GitLab From 096ee4e16fd62cd578d20ec4e8ad4756f4e369ee Mon Sep 17 00:00:00 2001 From: agozillon Date: Wed, 13 Mar 2024 16:18:21 +0100 Subject: [PATCH 0135/1407] [Flang][OpenMP] Implement "promotion" of use_device_ptr non-cptr arguments to use_device_addr (#82834) This effectively implements some now deprecated OpenMP functionality that some applications (most notably at the moment GenASiS) unfortunately depend on (deprecated in specification version 5.2): "If a list item in a use_device_ptr clause is not of type C_PTR, the behavior is as if the list item appeared in a use_device_addr clause. Support for such list items in a use_device_ptr clause is deprecated." This PR downgrades the hard-error to a deprecated warning and "promotes" the above cases by simply moving the offending operands from the use_device_ptr value list to the back of the use_device_addr list (and moves the related symbols, locs and types that form the BlockArgs correspondingly) and then the generation of the target data construct proceeds as normal. --- flang/lib/Lower/OpenMP/OpenMP.cpp | 66 +++++++++++++++++ flang/lib/Semantics/check-omp-structure.cpp | 2 +- .../use-device-ptr-to-use-device-addr.f90 | 72 +++++++++++++++++++ .../test/Semantics/OpenMP/use_device_ptr1.f90 | 2 +- 4 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 flang/test/Lower/OpenMP/use-device-ptr-to-use-device-addr.f90 diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index 1016c8389c6e..25bb4d9cff5d 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -798,6 +798,58 @@ genTaskGroupOp(Fortran::lower::AbstractConverter &converter, /*task_reductions=*/nullptr, allocateOperands, allocatorOperands); } +// This helper function implements the functionality of "promoting" +// non-CPTR arguments of use_device_ptr to use_device_addr +// arguments (automagic conversion of use_device_ptr -> +// use_device_addr in these cases). The way we do so currently is +// through the shuffling of operands from the devicePtrOperands to +// deviceAddrOperands where neccesary and re-organizing the types, +// locations and symbols to maintain the correct ordering of ptr/addr +// input -> BlockArg. +// +// This effectively implements some deprecated OpenMP functionality +// that some legacy applications unfortunately depend on +// (deprecated in specification version 5.2): +// +// "If a list item in a use_device_ptr clause is not of type C_PTR, +// the behavior is as if the list item appeared in a use_device_addr +// clause. Support for such list items in a use_device_ptr clause +// is deprecated." +static void promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr( + llvm::SmallVector &devicePtrOperands, + llvm::SmallVector &deviceAddrOperands, + llvm::SmallVector &useDeviceTypes, + llvm::SmallVector &useDeviceLocs, + llvm::SmallVector &useDeviceSymbols) { + auto moveElementToBack = [](size_t idx, auto &vector) { + auto *iter = std::next(vector.begin(), idx); + vector.push_back(*iter); + vector.erase(iter); + }; + + // Iterate over our use_device_ptr list and shift all non-cptr arguments into + // use_device_addr. + for (auto *it = devicePtrOperands.begin(); it != devicePtrOperands.end();) { + if (!fir::isa_builtin_cptr_type(fir::unwrapRefType(it->getType()))) { + deviceAddrOperands.push_back(*it); + // We have to shuffle the symbols around as well, to maintain + // the correct Input -> BlockArg for use_device_ptr/use_device_addr. + // NOTE: However, as map's do not seem to be included currently + // this isn't as pertinent, but we must try to maintain for + // future alterations. I believe the reason they are not currently + // is that the BlockArg assign/lowering needs to be extended + // to a greater set of types. + auto idx = std::distance(devicePtrOperands.begin(), it); + moveElementToBack(idx, useDeviceTypes); + moveElementToBack(idx, useDeviceLocs); + moveElementToBack(idx, useDeviceSymbols); + it = devicePtrOperands.erase(it); + continue; + } + ++it; + } +} + static mlir::omp::DataOp genDataOp(Fortran::lower::AbstractConverter &converter, Fortran::semantics::SemanticsContext &semaCtx, @@ -820,6 +872,20 @@ genDataOp(Fortran::lower::AbstractConverter &converter, useDeviceSymbols); cp.processUseDeviceAddr(deviceAddrOperands, useDeviceTypes, useDeviceLocs, useDeviceSymbols); + // This function implements the deprecated functionality of use_device_ptr + // that allows users to provide non-CPTR arguments to it with the caveat + // that the compiler will treat them as use_device_addr. A lot of legacy + // code may still depend on this functionality, so we should support it + // in some manner. We do so currently by simply shifting non-cptr operands + // from the use_device_ptr list into the front of the use_device_addr list + // whilst maintaining the ordering of useDeviceLocs, useDeviceSymbols and + // useDeviceTypes to use_device_ptr/use_device_addr input for BlockArg + // ordering. + // TODO: Perhaps create a user provideable compiler option that will + // re-introduce a hard-error rather than a warning in these cases. + promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr( + devicePtrOperands, deviceAddrOperands, useDeviceTypes, useDeviceLocs, + useDeviceSymbols); cp.processMap(currentLocation, llvm::omp::Directive::OMPD_target_data, stmtCtx, mapOperands); diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp index 54101ab8a42b..bf4debee1df3 100644 --- a/flang/lib/Semantics/check-omp-structure.cpp +++ b/flang/lib/Semantics/check-omp-structure.cpp @@ -2948,7 +2948,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::UseDevicePtr &x) { if (name->symbol) { if (!(IsBuiltinCPtr(*(name->symbol)))) { context_.Say(itr->second->source, - "'%s' in USE_DEVICE_PTR clause must be of type C_PTR"_err_en_US, + "Use of non-C_PTR type '%s' in USE_DEVICE_PTR is deprecated, use USE_DEVICE_ADDR instead"_warn_en_US, name->ToString()); } else { useDevicePtrNameList.push_back(*name); diff --git a/flang/test/Lower/OpenMP/use-device-ptr-to-use-device-addr.f90 b/flang/test/Lower/OpenMP/use-device-ptr-to-use-device-addr.f90 new file mode 100644 index 000000000000..33b597165601 --- /dev/null +++ b/flang/test/Lower/OpenMP/use-device-ptr-to-use-device-addr.f90 @@ -0,0 +1,72 @@ +!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s +!RUN: bbc -emit-hlfir -fopenmp %s -o - | FileCheck %s + +! This tests primary goal is to check the promotion of +! non-CPTR arguments from use_device_ptr to +! use_device_addr works, without breaking any +! functionality + +!CHECK: func.func @{{.*}}only_use_device_ptr() +!CHECK: omp.target_data use_device_ptr(%{{.*}} : !fir.ref>) use_device_addr(%{{.*}}, %{{.*}} : !fir.ref>>>, !fir.ref>>>) { +!CHECK: ^bb0(%{{.*}}: !fir.ref>, %{{.*}}: !fir.ref>>>, %{{.*}}: !fir.ref>>>): +subroutine only_use_device_ptr + use iso_c_binding + integer, pointer, dimension(:) :: array + real, pointer :: pa(:) + type(c_ptr) :: cptr + + !$omp target data use_device_ptr(pa, cptr, array) + !$omp end target data +end subroutine + +!CHECK: func.func @{{.*}}mix_use_device_ptr_and_addr() +!CHECK: omp.target_data use_device_ptr({{.*}} : !fir.ref>) use_device_addr(%{{.*}}, %{{.*}} : !fir.ref>>>, !fir.ref>>>) { +!CHECK: ^bb0(%{{.*}}: !fir.ref>, %{{.*}}: !fir.ref>>>, %{{.*}}: !fir.ref>>>): +subroutine mix_use_device_ptr_and_addr + use iso_c_binding + integer, pointer, dimension(:) :: array + real, pointer :: pa(:) + type(c_ptr) :: cptr + + !$omp target data use_device_ptr(pa, cptr) use_device_addr(array) + !$omp end target data +end subroutine + +!CHECK: func.func @{{.*}}only_use_device_addr() +!CHECK: omp.target_data use_device_addr(%{{.*}}, %{{.*}}, %{{.*}} : !fir.ref>>>, !fir.ref>, !fir.ref>>>) { +!CHECK: ^bb0(%{{.*}}: !fir.ref>>>, %{{.*}}: !fir.ref>, %{{.*}}: !fir.ref>>>): +subroutine only_use_device_addr + use iso_c_binding + integer, pointer, dimension(:) :: array + real, pointer :: pa(:) + type(c_ptr) :: cptr + + !$omp target data use_device_addr(pa, cptr, array) + !$omp end target data +end subroutine + +!CHECK: func.func @{{.*}}mix_use_device_ptr_and_addr_and_map() +!CHECK: omp.target_data map_entries(%{{.*}}, %{{.*}} : !fir.ref, !fir.ref) use_device_ptr(%{{.*}} : !fir.ref>) use_device_addr(%{{.*}}, %{{.*}} : !fir.ref>>>, !fir.ref>>>) { +!CHECK: ^bb0(%{{.*}}: !fir.ref>, %{{.*}}: !fir.ref>>>, %{{.*}}: !fir.ref>>>): +subroutine mix_use_device_ptr_and_addr_and_map + use iso_c_binding + integer :: i, j + integer, pointer, dimension(:) :: array + real, pointer :: pa(:) + type(c_ptr) :: cptr + + !$omp target data use_device_ptr(pa, cptr) use_device_addr(array) map(tofrom: i, j) + !$omp end target data +end subroutine + +!CHECK: func.func @{{.*}}only_use_map() +!CHECK: omp.target_data map_entries(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !fir.llvm_ptr>>, !fir.ref>>>, !fir.ref>, !fir.llvm_ptr>>, !fir.ref>>>) { +subroutine only_use_map + use iso_c_binding + integer, pointer, dimension(:) :: array + real, pointer :: pa(:) + type(c_ptr) :: cptr + + !$omp target data map(pa, cptr, array) + !$omp end target data +end subroutine diff --git a/flang/test/Semantics/OpenMP/use_device_ptr1.f90 b/flang/test/Semantics/OpenMP/use_device_ptr1.f90 index af89698a5c5a..176fb5f35a84 100644 --- a/flang/test/Semantics/OpenMP/use_device_ptr1.f90 +++ b/flang/test/Semantics/OpenMP/use_device_ptr1.f90 @@ -27,7 +27,7 @@ subroutine omp_target_data a = arrayB !$omp end target data - !ERROR: 'a' in USE_DEVICE_PTR clause must be of type C_PTR + !WARNING: Use of non-C_PTR type 'a' in USE_DEVICE_PTR is deprecated, use USE_DEVICE_ADDR instead !$omp target data map(tofrom: a) use_device_ptr(a) a = 2 !$omp end target data -- GitLab From 732f5368cdc297e83f8720fb13a8c848ff116ccf Mon Sep 17 00:00:00 2001 From: Slava Zakharin Date: Wed, 13 Mar 2024 08:23:10 -0700 Subject: [PATCH 0136/1407] [RFC][mlir] Add profitability callback to the Inliner. (#84258) Discussion at https://discourse.llvm.org/t/inliner-cost-model/2992 This change adds a callback that reports whether inlining of the particular call site (communicated via ResolvedCall argument) is profitable or not. The default MLIR inliner pass behavior is unchanged, i.e. the callback always returns true. This callback may be used to customize the inliner behavior based on the target specifics (like target instructions costs), profitability of the inlining for further optimizations (e.g. if inlining may enable loop optimizations or scalar optimizations due to object shape propagation), optimization levels (e.g. -Os inlining may be quite different from -Ofast inlining), etc. One of the questions is whether the ResolvedCall entity represents enough of the context for the custom inlining models to come up with the profitability decision. I think we can start with this and extend it as necessary. --------- Co-authored-by: Mehdi Amini --- mlir/include/mlir/Transforms/Inliner.h | 43 +++++++++++-------- mlir/include/mlir/Transforms/Passes.td | 7 +++ mlir/lib/Transforms/InlinerPass.cpp | 38 +++++++++++++++- mlir/lib/Transforms/Utils/Inliner.cpp | 3 ++ .../inlining-dump-default-pipeline.mlir | 2 +- mlir/test/Transforms/inlining-threshold.mlir | 18 ++++++++ 6 files changed, 92 insertions(+), 19 deletions(-) create mode 100644 mlir/test/Transforms/inlining-threshold.mlir diff --git a/mlir/include/mlir/Transforms/Inliner.h b/mlir/include/mlir/Transforms/Inliner.h index 1fe61fb4bbe7..073b83f6f844 100644 --- a/mlir/include/mlir/Transforms/Inliner.h +++ b/mlir/include/mlir/Transforms/Inliner.h @@ -69,19 +69,6 @@ private: /// of inlining decisions from the leafs to the roots of the callgraph. class Inliner { public: - using RunPipelineHelperTy = std::function; - - Inliner(Operation *op, CallGraph &cg, Pass &pass, AnalysisManager am, - RunPipelineHelperTy runPipelineHelper, const InlinerConfig &config) - : op(op), cg(cg), pass(pass), am(am), - runPipelineHelper(std::move(runPipelineHelper)), config(config) {} - Inliner(Inliner &) = delete; - void operator=(const Inliner &) = delete; - - /// Perform inlining on a OpTrait::SymbolTable operation. - LogicalResult doInlining(); - /// This struct represents a resolved call to a given callgraph node. Given /// that the call does not actually contain a direct reference to the /// Region(CallGraphNode) that it is dispatching to, we need to resolve them @@ -94,7 +81,29 @@ public: CallGraphNode *sourceNode, *targetNode; }; -protected: + using RunPipelineHelperTy = std::function; + + /// Type of the callback answering if it is profitable + /// to inline a callable operation at a call site. + /// It might be the case that the ResolvedCall does not provide + /// enough context to make the profitability decision, so + /// this hook's interface might need to be extended in future. + using ProfitabilityCallbackTy = std::function; + + Inliner(Operation *op, CallGraph &cg, Pass &pass, AnalysisManager am, + RunPipelineHelperTy runPipelineHelper, const InlinerConfig &config, + ProfitabilityCallbackTy isProfitableToInline) + : op(op), cg(cg), pass(pass), am(am), + runPipelineHelper(std::move(runPipelineHelper)), config(config), + isProfitableToInline(std::move(isProfitableToInline)) {} + Inliner(Inliner &) = delete; + void operator=(const Inliner &) = delete; + + /// Perform inlining on a OpTrait::SymbolTable operation. + LogicalResult doInlining(); + +private: /// An OpTrait::SymbolTable operation to run the inlining on. Operation *op; /// A CallGraph analysis for the given operation. @@ -108,12 +117,12 @@ protected: const RunPipelineHelperTy runPipelineHelper; /// The inliner configuration parameters. const InlinerConfig &config; + /// Returns true, if it is profitable to inline the callable operation + /// at the call site. + ProfitabilityCallbackTy isProfitableToInline; -private: /// Forward declaration of the class providing the actual implementation. class Impl; - -public: }; } // namespace mlir diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td index b8fdf7a58047..51b2a27da639 100644 --- a/mlir/include/mlir/Transforms/Passes.td +++ b/mlir/include/mlir/Transforms/Passes.td @@ -278,6 +278,13 @@ def Inliner : Pass<"inline"> { Option<"maxInliningIterations", "max-iterations", "unsigned", /*default=*/"4", "Maximum number of iterations when inlining within an SCC">, + Option<"inliningThreshold", "inlining-threshold", "unsigned", + /*default=*/"-1U", + "If the ratio between the number of the operations " + "in the callee and the number of the operations " + "in the caller exceeds this value (in percentage), " + "then the callee is not inlined even if it is legal " + "to inline it">, ]; } diff --git a/mlir/lib/Transforms/InlinerPass.cpp b/mlir/lib/Transforms/InlinerPass.cpp index c058e8050cd1..08d8dbf73a6a 100644 --- a/mlir/lib/Transforms/InlinerPass.cpp +++ b/mlir/lib/Transforms/InlinerPass.cpp @@ -24,6 +24,8 @@ namespace mlir { #include "mlir/Transforms/Passes.h.inc" } // namespace mlir +#define DEBUG_TYPE "inliner-pass" + using namespace mlir; /// This function implements the inliner optimization pipeline. @@ -88,6 +90,35 @@ InlinerPass::InlinerPass(std::function defaultPipeline, config.setOpPipelines(std::move(opPipelines)); } +// Return true if the inlining ratio does not exceed the threshold. +static bool isProfitableToInline(const Inliner::ResolvedCall &resolvedCall, + unsigned inliningThreshold) { + Region *callerRegion = resolvedCall.sourceNode->getCallableRegion(); + Region *calleeRegion = resolvedCall.targetNode->getCallableRegion(); + + // We should not get external nodes here, but just return true + // for now to preserve the original behavior of the inliner pass. + if (!calleeRegion || !calleeRegion) + return true; + + auto countOps = [](Region *region) { + unsigned count = 0; + region->walk([&](Operation *) { ++count; }); + return count; + }; + + unsigned callerOps = countOps(callerRegion); + + // Always inline empty callees (if it is possible at all). + if (callerOps == 0) + return true; + + unsigned ratio = countOps(calleeRegion) * 100 / callerOps; + LLVM_DEBUG(llvm::dbgs() << "Callee / caller operation ratio (max: " + << inliningThreshold << "%): " << ratio << "%\n"); + return ratio <= inliningThreshold; +} + void InlinerPass::runOnOperation() { CallGraph &cg = getAnalysis(); @@ -100,9 +131,14 @@ void InlinerPass::runOnOperation() { return signalPassFailure(); } + // By default, assume that any inlining is profitable. + auto profitabilityCb = [=](const Inliner::ResolvedCall &call) { + return isProfitableToInline(call, inliningThreshold); + }; + // Get an instance of the inliner. Inliner inliner(op, cg, *this, getAnalysisManager(), runPipelineHelper, - config); + config, profitabilityCb); // Run the inlining. if (failed(inliner.doInlining())) diff --git a/mlir/lib/Transforms/Utils/Inliner.cpp b/mlir/lib/Transforms/Utils/Inliner.cpp index f227cedb269d..8acfc96d2b61 100644 --- a/mlir/lib/Transforms/Utils/Inliner.cpp +++ b/mlir/lib/Transforms/Utils/Inliner.cpp @@ -741,6 +741,9 @@ bool Inliner::Impl::shouldInline(ResolvedCall &resolvedCall) { if (calleeHasMultipleBlocks && !callerRegionSupportsMultipleBlocks()) return false; + if (!inliner.isProfitableToInline(resolvedCall)) + return false; + // Otherwise, inline. return true; } diff --git a/mlir/test/Transforms/inlining-dump-default-pipeline.mlir b/mlir/test/Transforms/inlining-dump-default-pipeline.mlir index e2c31867a8e0..4f8638054206 100644 --- a/mlir/test/Transforms/inlining-dump-default-pipeline.mlir +++ b/mlir/test/Transforms/inlining-dump-default-pipeline.mlir @@ -1,2 +1,2 @@ // RUN: mlir-opt %s -pass-pipeline="builtin.module(inline)" -dump-pass-pipeline 2>&1 | FileCheck %s -// CHECK: builtin.module(inline{default-pipeline=canonicalize max-iterations=4 }) +// CHECK: builtin.module(inline{default-pipeline=canonicalize inlining-threshold=4294967295 max-iterations=4 }) diff --git a/mlir/test/Transforms/inlining-threshold.mlir b/mlir/test/Transforms/inlining-threshold.mlir new file mode 100644 index 000000000000..b94115d8f264 --- /dev/null +++ b/mlir/test/Transforms/inlining-threshold.mlir @@ -0,0 +1,18 @@ +// RUN: mlir-opt %s --mlir-disable-threading -inline='default-pipeline='' inlining-threshold=100' -debug-only=inliner-pass 2>&1 | FileCheck %s + +// Check that inlining does not happen when the threshold is exceeded. +func.func @callee1(%arg : i32) -> i32 { + %v1 = arith.addi %arg, %arg : i32 + %v2 = arith.addi %v1, %arg : i32 + %v3 = arith.addi %v2, %arg : i32 + return %v3 : i32 +} + +// CHECK-LABEL: func @caller1 +func.func @caller1(%arg0 : i32) -> i32 { + // CHECK-NEXT: call @callee1 + // CHECK-NEXT: return + + %0 = call @callee1(%arg0) : (i32) -> i32 + return %0 : i32 +} -- GitLab From e0738cc65865c31975b5bdbbf89c5a4dbbe06dc5 Mon Sep 17 00:00:00 2001 From: Slava Zakharin Date: Wed, 13 Mar 2024 08:26:33 -0700 Subject: [PATCH 0137/1407] [flang] Moved REAL(16) RANDOM_NUMBER to Float128Math library. (#85002) --- .../Optimizer/Builder/Runtime/Intrinsics.cpp | 29 ++++++- flang/runtime/Float128Math/CMakeLists.txt | 1 + flang/runtime/Float128Math/random.cpp | 23 +++++ flang/runtime/random-templates.h | 87 +++++++++++++++++++ flang/runtime/random.cpp | 81 ++--------------- .../Lower/Intrinsics/random_number_real16.f90 | 16 ++++ 6 files changed, 160 insertions(+), 77 deletions(-) create mode 100644 flang/runtime/Float128Math/random.cpp create mode 100644 flang/runtime/random-templates.h create mode 100644 flang/test/Lower/Intrinsics/random_number_real16.f90 diff --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp index 638bfd60a246..57c47da0f3f8 100644 --- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp +++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp @@ -27,6 +27,24 @@ using namespace Fortran::runtime; +namespace { +/// Placeholder for real*16 version of RandomNumber Intrinsic +struct ForcedRandomNumberReal16 { + static constexpr const char *name = ExpandAndQuoteKey(RTNAME(RandomNumber16)); + static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { + return [](mlir::MLIRContext *ctx) { + auto boxTy = + fir::runtime::getModel()(ctx); + auto strTy = fir::runtime::getModel()(ctx); + auto intTy = fir::runtime::getModel()(ctx); + ; + return mlir::FunctionType::get(ctx, {boxTy, strTy, intTy}, + mlir::NoneType::get(ctx)); + }; + } +}; +} // namespace + mlir::Value fir::runtime::genAssociated(fir::FirOpBuilder &builder, mlir::Location loc, mlir::Value pointer, mlir::Value target) { @@ -100,8 +118,15 @@ void fir::runtime::genRandomInit(fir::FirOpBuilder &builder, mlir::Location loc, void fir::runtime::genRandomNumber(fir::FirOpBuilder &builder, mlir::Location loc, mlir::Value harvest) { - mlir::func::FuncOp func = - fir::runtime::getRuntimeFunc(loc, builder); + mlir::func::FuncOp func; + auto boxEleTy = fir::dyn_cast_ptrOrBoxEleTy(harvest.getType()); + auto eleTy = fir::unwrapSequenceType(boxEleTy); + if (eleTy.isF128()) { + func = fir::runtime::getRuntimeFunc(loc, builder); + } else { + func = fir::runtime::getRuntimeFunc(loc, builder); + } + mlir::FunctionType funcTy = func.getFunctionType(); mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc); mlir::Value sourceLine = diff --git a/flang/runtime/Float128Math/CMakeLists.txt b/flang/runtime/Float128Math/CMakeLists.txt index 980356131b68..33f73a9c5445 100644 --- a/flang/runtime/Float128Math/CMakeLists.txt +++ b/flang/runtime/Float128Math/CMakeLists.txt @@ -48,6 +48,7 @@ set(sources nearest.cpp norm2.cpp pow.cpp + random.cpp round.cpp rrspacing.cpp scale.cpp diff --git a/flang/runtime/Float128Math/random.cpp b/flang/runtime/Float128Math/random.cpp new file mode 100644 index 000000000000..cda962b41614 --- /dev/null +++ b/flang/runtime/Float128Math/random.cpp @@ -0,0 +1,23 @@ +//===-- runtime/Float128Math/random.cpp -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "math-entries.h" +#include "numeric-template-specs.h" +#include "random-templates.h" + +using namespace Fortran::runtime::random; +extern "C" { + +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 +void RTDEF(RandomNumber16)( + const Descriptor &harvest, const char *source, int line) { + return Generate, 113>(harvest); +} +#endif + +} // extern "C" diff --git a/flang/runtime/random-templates.h b/flang/runtime/random-templates.h new file mode 100644 index 000000000000..ce64a94901a2 --- /dev/null +++ b/flang/runtime/random-templates.h @@ -0,0 +1,87 @@ +//===-- runtime/random-templates.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef FORTRAN_RUNTIME_RANDOM_TEMPLATES_H_ +#define FORTRAN_RUNTIME_RANDOM_TEMPLATES_H_ + +#include "lock.h" +#include "numeric-templates.h" +#include "flang/Runtime/descriptor.h" +#include +#include + +namespace Fortran::runtime::random { + +// Newer "Minimum standard", recommended by Park, Miller, and Stockmeyer in +// 1993. Same as C++17 std::minstd_rand, but explicitly instantiated for +// permanence. +using Generator = + std::linear_congruential_engine; + +using GeneratedWord = typename Generator::result_type; +static constexpr std::uint64_t range{ + static_cast(Generator::max() - Generator::min() + 1)}; +static constexpr bool rangeIsPowerOfTwo{(range & (range - 1)) == 0}; +static constexpr int rangeBits{ + 64 - common::LeadingZeroBitCount(range) - !rangeIsPowerOfTwo}; + +extern Lock lock; +extern Generator generator; +extern std::optional nextValue; + +// Call only with lock held +static GeneratedWord GetNextValue() { + GeneratedWord result; + if (nextValue.has_value()) { + result = *nextValue; + nextValue.reset(); + } else { + result = generator(); + } + return result; +} + +template +inline void Generate(const Descriptor &harvest) { + static constexpr std::size_t minBits{ + std::max(PREC, 8 * sizeof(GeneratedWord))}; + using Int = common::HostUnsignedIntType; + static constexpr std::size_t words{ + static_cast(PREC + rangeBits - 1) / rangeBits}; + std::size_t elements{harvest.Elements()}; + SubscriptValue at[maxRank]; + harvest.GetLowerBounds(at); + { + CriticalSection critical{lock}; + for (std::size_t j{0}; j < elements; ++j) { + while (true) { + Int fraction{GetNextValue()}; + if constexpr (words > 1) { + for (std::size_t k{1}; k < words; ++k) { + static constexpr auto rangeMask{ + (GeneratedWord{1} << rangeBits) - 1}; + GeneratedWord word{(GetNextValue() - generator.min()) & rangeMask}; + fraction = (fraction << rangeBits) | word; + } + } + fraction >>= words * rangeBits - PREC; + REAL next{ + LDEXPTy::compute(static_cast(fraction), -(PREC + 1))}; + if (next >= 0.0 && next < 1.0) { + *harvest.Element(at) = next; + break; + } + } + harvest.IncrementSubscripts(at); + } + } +} + +} // namespace Fortran::runtime::random + +#endif // FORTRAN_RUNTIME_RANDOM_TEMPLATES_H_ diff --git a/flang/runtime/random.cpp b/flang/runtime/random.cpp index 642091a06aff..13bed1f0abe1 100644 --- a/flang/runtime/random.cpp +++ b/flang/runtime/random.cpp @@ -11,85 +11,24 @@ #include "flang/Runtime/random.h" #include "lock.h" +#include "random-templates.h" #include "terminator.h" #include "flang/Common/float128.h" #include "flang/Common/leading-zero-bit-count.h" #include "flang/Common/uint128.h" #include "flang/Runtime/cpp-type.h" #include "flang/Runtime/descriptor.h" -#include #include #include #include #include -#include #include -namespace Fortran::runtime { +namespace Fortran::runtime::random { -// Newer "Minimum standard", recommended by Park, Miller, and Stockmeyer in -// 1993. Same as C++17 std::minstd_rand, but explicitly instantiated for -// permanence. -using Generator = - std::linear_congruential_engine; - -using GeneratedWord = typename Generator::result_type; -static constexpr std::uint64_t range{ - static_cast(Generator::max() - Generator::min() + 1)}; -static constexpr bool rangeIsPowerOfTwo{(range & (range - 1)) == 0}; -static constexpr int rangeBits{ - 64 - common::LeadingZeroBitCount(range) - !rangeIsPowerOfTwo}; - -static Lock lock; -static Generator generator; -static std::optional nextValue; - -// Call only with lock held -static GeneratedWord GetNextValue() { - GeneratedWord result; - if (nextValue.has_value()) { - result = *nextValue; - nextValue.reset(); - } else { - result = generator(); - } - return result; -} - -template -inline void Generate(const Descriptor &harvest) { - static constexpr std::size_t minBits{ - std::max(PREC, 8 * sizeof(GeneratedWord))}; - using Int = common::HostUnsignedIntType; - static constexpr std::size_t words{ - static_cast(PREC + rangeBits - 1) / rangeBits}; - std::size_t elements{harvest.Elements()}; - SubscriptValue at[maxRank]; - harvest.GetLowerBounds(at); - { - CriticalSection critical{lock}; - for (std::size_t j{0}; j < elements; ++j) { - while (true) { - Int fraction{GetNextValue()}; - if constexpr (words > 1) { - for (std::size_t k{1}; k < words; ++k) { - static constexpr auto rangeMask{ - (GeneratedWord{1} << rangeBits) - 1}; - GeneratedWord word{(GetNextValue() - generator.min()) & rangeMask}; - fraction = (fraction << rangeBits) | word; - } - } - fraction >>= words * rangeBits - PREC; - REAL next{std::ldexp(static_cast(fraction), -(PREC + 1))}; - if (next >= 0.0 && next < 1.0) { - *harvest.Element(at) = next; - break; - } - } - harvest.IncrementSubscripts(at); - } - } -} +Lock lock; +Generator generator; +std::optional nextValue; extern "C" { @@ -130,14 +69,6 @@ void RTNAME(RandomNumber)( #if LDBL_MANT_DIG == 64 Generate, 64>(harvest); return; -#endif - } - break; - case 16: - if constexpr (HasCppTypeFor) { -#if LDBL_MANT_DIG == 113 - Generate, 113>(harvest); - return; #endif } break; @@ -263,4 +194,4 @@ void RTNAME(RandomSeed)(const Descriptor *size, const Descriptor *put, } } // extern "C" -} // namespace Fortran::runtime +} // namespace Fortran::runtime::random diff --git a/flang/test/Lower/Intrinsics/random_number_real16.f90 b/flang/test/Lower/Intrinsics/random_number_real16.f90 new file mode 100644 index 000000000000..76fed258d8af --- /dev/null +++ b/flang/test/Lower/Intrinsics/random_number_real16.f90 @@ -0,0 +1,16 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK-LABEL: func @_QPtest_scalar +! CHECK: fir.call @_FortranARandomNumber16({{.*}}){{.*}}: (!fir.box, !fir.ref, i32) -> none +subroutine test_scalar + real(16) :: r + call random_number(r) +end + +! CHECK-LABEL: func @_QPtest_array +! CHECK: fir.call @_FortranARandomNumber16({{.*}}){{.*}}: (!fir.box, !fir.ref, i32) -> none +subroutine test_array(r) + real(16) :: r(:) + call random_number(r) +end -- GitLab From 286c3b500dc36b2451683bde5d681bf6efea3e63 Mon Sep 17 00:00:00 2001 From: Slava Zakharin Date: Wed, 13 Mar 2024 08:26:49 -0700 Subject: [PATCH 0138/1407] [flang] Enable REAL(16) MODULO lowering. (#85005) The lowering currently relies on the trivial operations, so we should just lower it for REAL(16) the same way we do this for other trivial operations. --- flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 5 +- flang/test/Lower/Intrinsics/modulo.f90 | 82 +++++++++++-------- 2 files changed, 52 insertions(+), 35 deletions(-) diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp index ca5ab6fcea34..21d253624a1a 100644 --- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -5208,6 +5208,8 @@ mlir::Value IntrinsicLibrary::genMod(mlir::Type resultType, // MODULO mlir::Value IntrinsicLibrary::genModulo(mlir::Type resultType, llvm::ArrayRef args) { + // TODO: we'd better generate a runtime call here, when runtime error + // checking is needed (to detect 0 divisor) or when precise math is requested. assert(args.size() == 2); // No floored modulo op in LLVM/MLIR yet. TODO: add one to MLIR. // In the meantime, use a simple inlined implementation based on truncated @@ -5233,10 +5235,7 @@ mlir::Value IntrinsicLibrary::genModulo(mlir::Type resultType, return builder.create(loc, mustAddP, remPlusP, remainder); } - // Real case - if (resultType == mlir::FloatType::getF128(builder.getContext())) - TODO(loc, "REAL(KIND=16): in MODULO intrinsic"); auto remainder = builder.create(loc, args[0], args[1]); mlir::Value zero = builder.createRealZeroConstant(loc, remainder.getType()); auto remainderIsNotZero = builder.create( diff --git a/flang/test/Lower/Intrinsics/modulo.f90 b/flang/test/Lower/Intrinsics/modulo.f90 index 64a6607a09cc..001e307aa077 100644 --- a/flang/test/Lower/Intrinsics/modulo.f90 +++ b/flang/test/Lower/Intrinsics/modulo.f90 @@ -3,36 +3,54 @@ ! CHECK-LABEL: func @_QPmodulo_testr( ! CHECK-SAME: %[[arg0:.*]]: !fir.ref{{.*}}, %[[arg1:.*]]: !fir.ref{{.*}}, %[[arg2:.*]]: !fir.ref{{.*}}) { subroutine modulo_testr(r, a, p) - real(8) :: r, a, p - ! CHECK-DAG: %[[a:.*]] = fir.load %[[arg1]] : !fir.ref - ! CHECK-DAG: %[[p:.*]] = fir.load %[[arg2]] : !fir.ref - ! CHECK-DAG: %[[rem:.*]] = arith.remf %[[a]], %[[p]] {{.*}}: f64 - ! CHECK-DAG: %[[zero:.*]] = arith.constant 0.000000e+00 : f64 - ! CHECK-DAG: %[[remNotZero:.*]] = arith.cmpf une, %[[rem]], %[[zero]] {{.*}} : f64 - ! CHECK-DAG: %[[aNeg:.*]] = arith.cmpf olt, %[[a]], %[[zero]] {{.*}} : f64 - ! CHECK-DAG: %[[pNeg:.*]] = arith.cmpf olt, %[[p]], %[[zero]] {{.*}} : f64 - ! CHECK-DAG: %[[signDifferent:.*]] = arith.xori %[[aNeg]], %[[pNeg]] : i1 - ! CHECK-DAG: %[[mustAddP:.*]] = arith.andi %[[remNotZero]], %[[signDifferent]] : i1 - ! CHECK-DAG: %[[remPlusP:.*]] = arith.addf %[[rem]], %[[p]] {{.*}}: f64 - ! CHECK: %[[res:.*]] = arith.select %[[mustAddP]], %[[remPlusP]], %[[rem]] : f64 - ! CHECK: fir.store %[[res]] to %[[arg0]] : !fir.ref - r = modulo(a, p) - end subroutine - - ! CHECK-LABEL: func @_QPmodulo_testi( - ! CHECK-SAME: %[[arg0:.*]]: !fir.ref{{.*}}, %[[arg1:.*]]: !fir.ref{{.*}}, %[[arg2:.*]]: !fir.ref{{.*}}) { - subroutine modulo_testi(r, a, p) - integer(8) :: r, a, p - ! CHECK-DAG: %[[a:.*]] = fir.load %[[arg1]] : !fir.ref - ! CHECK-DAG: %[[p:.*]] = fir.load %[[arg2]] : !fir.ref - ! CHECK-DAG: %[[rem:.*]] = arith.remsi %[[a]], %[[p]] : i64 - ! CHECK-DAG: %[[argXor:.*]] = arith.xori %[[a]], %[[p]] : i64 - ! CHECK-DAG: %[[signDifferent:.*]] = arith.cmpi slt, %[[argXor]], %c0{{.*}} : i64 - ! CHECK-DAG: %[[remNotZero:.*]] = arith.cmpi ne, %[[rem]], %c0{{.*}} : i64 - ! CHECK-DAG: %[[mustAddP:.*]] = arith.andi %[[remNotZero]], %[[signDifferent]] : i1 - ! CHECK-DAG: %[[remPlusP:.*]] = arith.addi %[[rem]], %[[p]] : i64 - ! CHECK: %[[res:.*]] = arith.select %[[mustAddP]], %[[remPlusP]], %[[rem]] : i64 - ! CHECK: fir.store %[[res]] to %[[arg0]] : !fir.ref - r = modulo(a, p) - end subroutine + real(8) :: r, a, p + ! CHECK-DAG: %[[a:.*]] = fir.load %[[arg1]] : !fir.ref + ! CHECK-DAG: %[[p:.*]] = fir.load %[[arg2]] : !fir.ref + ! CHECK-DAG: %[[rem:.*]] = arith.remf %[[a]], %[[p]] {{.*}}: f64 + ! CHECK-DAG: %[[zero:.*]] = arith.constant 0.000000e+00 : f64 + ! CHECK-DAG: %[[remNotZero:.*]] = arith.cmpf une, %[[rem]], %[[zero]] {{.*}} : f64 + ! CHECK-DAG: %[[aNeg:.*]] = arith.cmpf olt, %[[a]], %[[zero]] {{.*}} : f64 + ! CHECK-DAG: %[[pNeg:.*]] = arith.cmpf olt, %[[p]], %[[zero]] {{.*}} : f64 + ! CHECK-DAG: %[[signDifferent:.*]] = arith.xori %[[aNeg]], %[[pNeg]] : i1 + ! CHECK-DAG: %[[mustAddP:.*]] = arith.andi %[[remNotZero]], %[[signDifferent]] : i1 + ! CHECK-DAG: %[[remPlusP:.*]] = arith.addf %[[rem]], %[[p]] {{.*}}: f64 + ! CHECK: %[[res:.*]] = arith.select %[[mustAddP]], %[[remPlusP]], %[[rem]] : f64 + ! CHECK: fir.store %[[res]] to %[[arg0]] : !fir.ref + r = modulo(a, p) +end subroutine +! CHECK-LABEL: func @_QPmodulo_testi( +! CHECK-SAME: %[[arg0:.*]]: !fir.ref{{.*}}, %[[arg1:.*]]: !fir.ref{{.*}}, %[[arg2:.*]]: !fir.ref{{.*}}) { +subroutine modulo_testi(r, a, p) + integer(8) :: r, a, p + ! CHECK-DAG: %[[a:.*]] = fir.load %[[arg1]] : !fir.ref + ! CHECK-DAG: %[[p:.*]] = fir.load %[[arg2]] : !fir.ref + ! CHECK-DAG: %[[rem:.*]] = arith.remsi %[[a]], %[[p]] : i64 + ! CHECK-DAG: %[[argXor:.*]] = arith.xori %[[a]], %[[p]] : i64 + ! CHECK-DAG: %[[signDifferent:.*]] = arith.cmpi slt, %[[argXor]], %c0{{.*}} : i64 + ! CHECK-DAG: %[[remNotZero:.*]] = arith.cmpi ne, %[[rem]], %c0{{.*}} : i64 + ! CHECK-DAG: %[[mustAddP:.*]] = arith.andi %[[remNotZero]], %[[signDifferent]] : i1 + ! CHECK-DAG: %[[remPlusP:.*]] = arith.addi %[[rem]], %[[p]] : i64 + ! CHECK: %[[res:.*]] = arith.select %[[mustAddP]], %[[remPlusP]], %[[rem]] : i64 + ! CHECK: fir.store %[[res]] to %[[arg0]] : !fir.ref + r = modulo(a, p) +end subroutine + +! CHECK-LABEL: func @_QPmodulo_testr16( +! CHECK-SAME: %[[arg0:.*]]: !fir.ref{{.*}}, %[[arg1:.*]]: !fir.ref{{.*}}, %[[arg2:.*]]: !fir.ref{{.*}}) { +subroutine modulo_testr16(r, a, p) + real(16) :: r, a, p + ! CHECK-DAG: %[[a:.*]] = fir.load %[[arg1]] : !fir.ref + ! CHECK-DAG: %[[p:.*]] = fir.load %[[arg2]] : !fir.ref + ! CHECK-DAG: %[[rem:.*]] = arith.remf %[[a]], %[[p]] {{.*}}: f128 + ! CHECK-DAG: %[[zero:.*]] = arith.constant 0.000000e+00 : f128 + ! CHECK-DAG: %[[remNotZero:.*]] = arith.cmpf une, %[[rem]], %[[zero]] {{.*}} : f128 + ! CHECK-DAG: %[[aNeg:.*]] = arith.cmpf olt, %[[a]], %[[zero]] {{.*}} : f128 + ! CHECK-DAG: %[[pNeg:.*]] = arith.cmpf olt, %[[p]], %[[zero]] {{.*}} : f128 + ! CHECK-DAG: %[[signDifferent:.*]] = arith.xori %[[aNeg]], %[[pNeg]] : i1 + ! CHECK-DAG: %[[mustAddP:.*]] = arith.andi %[[remNotZero]], %[[signDifferent]] : i1 + ! CHECK-DAG: %[[remPlusP:.*]] = arith.addf %[[rem]], %[[p]] {{.*}}: f128 + ! CHECK: %[[res:.*]] = arith.select %[[mustAddP]], %[[remPlusP]], %[[rem]] : f128 + ! CHECK: fir.store %[[res]] to %[[arg0]] : !fir.ref + r = modulo(a, p) +end subroutine -- GitLab From d24ff9aec4f2741804268a66d711d6d31cd06138 Mon Sep 17 00:00:00 2001 From: Slava Zakharin Date: Wed, 13 Mar 2024 08:27:15 -0700 Subject: [PATCH 0139/1407] [flang][runtime] Added lowering and runtime for REAL(16) IEEE_FMA. (#85017) --- flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 4 ++++ flang/runtime/Float128Math/CMakeLists.txt | 1 + flang/runtime/Float128Math/fma.cpp | 23 +++++++++++++++++++ flang/runtime/Float128Math/math-entries.h | 3 +++ flang/test/Lower/Intrinsics/fma_real16.f90 | 9 ++++++++ 5 files changed, 40 insertions(+) create mode 100644 flang/runtime/Float128Math/fma.cpp create mode 100644 flang/test/Lower/Intrinsics/fma_real16.f90 diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp index 21d253624a1a..94fcfa350311 100644 --- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -922,6 +922,8 @@ mlir::Value genComplexMathOp(fir::FirOpBuilder &builder, mlir::Location loc, constexpr auto FuncTypeReal16Real16 = genFuncType, Ty::Real<16>>; constexpr auto FuncTypeReal16Real16Real16 = genFuncType, Ty::Real<16>, Ty::Real<16>>; +constexpr auto FuncTypeReal16Real16Real16Real16 = + genFuncType, Ty::Real<16>, Ty::Real<16>, Ty::Real<16>>; constexpr auto FuncTypeReal16Integer4Real16 = genFuncType, Ty::Integer<4>, Ty::Real<16>>; constexpr auto FuncTypeInteger4Real16 = @@ -1143,6 +1145,8 @@ static constexpr MathOperation mathOperations[] = { {"fma", "llvm.fma.f64", genFuncType, Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>, genMathOp}, + {"fma", RTNAME_STRING(FmaF128), FuncTypeReal16Real16Real16Real16, + genLibF128Call}, {"gamma", "tgammaf", genFuncType, Ty::Real<4>>, genLibCall}, {"gamma", "tgamma", genFuncType, Ty::Real<8>>, genLibCall}, {"gamma", RTNAME_STRING(TgammaF128), FuncTypeReal16Real16, genLibF128Call}, diff --git a/flang/runtime/Float128Math/CMakeLists.txt b/flang/runtime/Float128Math/CMakeLists.txt index 33f73a9c5445..a5f5bec1e7e4 100644 --- a/flang/runtime/Float128Math/CMakeLists.txt +++ b/flang/runtime/Float128Math/CMakeLists.txt @@ -33,6 +33,7 @@ set(sources exp.cpp exponent.cpp floor.cpp + fma.cpp fraction.cpp hypot.cpp j0.cpp diff --git a/flang/runtime/Float128Math/fma.cpp b/flang/runtime/Float128Math/fma.cpp new file mode 100644 index 000000000000..ec67e8e6fba2 --- /dev/null +++ b/flang/runtime/Float128Math/fma.cpp @@ -0,0 +1,23 @@ +//===-- runtime/Float128Math/fma.cpp --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "math-entries.h" + +namespace Fortran::runtime { +extern "C" { + +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 +CppTypeFor RTDEF(FmaF128)( + CppTypeFor x, CppTypeFor y, + CppTypeFor z) { + return Fma::invoke(x, y, z); +} +#endif + +} // extern "C" +} // namespace Fortran::runtime diff --git a/flang/runtime/Float128Math/math-entries.h b/flang/runtime/Float128Math/math-entries.h index 1eab7c86f2ed..13fdab264700 100644 --- a/flang/runtime/Float128Math/math-entries.h +++ b/flang/runtime/Float128Math/math-entries.h @@ -77,6 +77,7 @@ DEFINE_FALLBACK_F128(Erf) DEFINE_FALLBACK_F128(Erfc) DEFINE_FALLBACK_F128(Exp) DEFINE_FALLBACK_F128(Floor) +DEFINE_FALLBACK_F128(Fma) DEFINE_FALLBACK_F128(Frexp) DEFINE_FALLBACK_F128(Hypot) DEFINE_FALLBACK_I32(Ilogb) @@ -124,6 +125,7 @@ DEFINE_SIMPLE_ALIAS(Erf, erfq) DEFINE_SIMPLE_ALIAS(Erfc, erfcq) DEFINE_SIMPLE_ALIAS(Exp, expq) DEFINE_SIMPLE_ALIAS(Floor, floorq) +DEFINE_SIMPLE_ALIAS(Fma, fmaq) DEFINE_SIMPLE_ALIAS(Frexp, frexpq) DEFINE_SIMPLE_ALIAS(Hypot, hypotq) DEFINE_SIMPLE_ALIAS(Ilogb, ilogbq) @@ -177,6 +179,7 @@ DEFINE_SIMPLE_ALIAS(Erf, std::erf) DEFINE_SIMPLE_ALIAS(Erfc, std::erfc) DEFINE_SIMPLE_ALIAS(Exp, std::exp) DEFINE_SIMPLE_ALIAS(Floor, std::floor) +DEFINE_SIMPLE_ALIAS(Fma, std::fma) DEFINE_SIMPLE_ALIAS(Frexp, std::frexp) DEFINE_SIMPLE_ALIAS(Hypot, std::hypot) DEFINE_SIMPLE_ALIAS(Ilogb, std::ilogb) diff --git a/flang/test/Lower/Intrinsics/fma_real16.f90 b/flang/test/Lower/Intrinsics/fma_real16.f90 new file mode 100644 index 000000000000..62cf2fbcefbf --- /dev/null +++ b/flang/test/Lower/Intrinsics/fma_real16.f90 @@ -0,0 +1,9 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranAFmaF128({{.*}}){{.*}}: (f128, f128, f128) -> f128 + use ieee_arithmetic, only: ieee_fma + real(16) :: x, y, z + x = ieee_fma(x, y, z) +end -- GitLab From 9a3000cf6700a711f3d81d071e0b6933cef46c36 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Wed, 13 Mar 2024 08:36:49 -0700 Subject: [PATCH 0140/1407] [libc] roll out rest of stdbit.h entrypoints to gpu,linux,baremetal (#84938) --- libc/config/gpu/entrypoints.txt | 30 +++++++++++++++++++++++ libc/config/linux/aarch64/entrypoints.txt | 30 +++++++++++++++++++++++ libc/config/linux/arm/entrypoints.txt | 30 +++++++++++++++++++++++ libc/config/linux/riscv/entrypoints.txt | 30 +++++++++++++++++++++++ 4 files changed, 120 insertions(+) diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt index fca5315fc4f0..4fb87cb9f5a3 100644 --- a/libc/config/gpu/entrypoints.txt +++ b/libc/config/gpu/entrypoints.txt @@ -106,6 +106,36 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdbit.stdc_first_trailing_one_ui libc.src.stdbit.stdc_first_trailing_one_ul libc.src.stdbit.stdc_first_trailing_one_ull + libc.src.stdbit.stdc_count_zeros_uc + libc.src.stdbit.stdc_count_zeros_us + libc.src.stdbit.stdc_count_zeros_ui + libc.src.stdbit.stdc_count_zeros_ul + libc.src.stdbit.stdc_count_zeros_ull + libc.src.stdbit.stdc_count_ones_uc + libc.src.stdbit.stdc_count_ones_us + libc.src.stdbit.stdc_count_ones_ui + libc.src.stdbit.stdc_count_ones_ul + libc.src.stdbit.stdc_count_ones_ull + libc.src.stdbit.stdc_has_single_bit_uc + libc.src.stdbit.stdc_has_single_bit_us + libc.src.stdbit.stdc_has_single_bit_ui + libc.src.stdbit.stdc_has_single_bit_ul + libc.src.stdbit.stdc_has_single_bit_ull + libc.src.stdbit.stdc_bit_width_uc + libc.src.stdbit.stdc_bit_width_us + libc.src.stdbit.stdc_bit_width_ui + libc.src.stdbit.stdc_bit_width_ul + libc.src.stdbit.stdc_bit_width_ull + libc.src.stdbit.stdc_bit_floor_uc + libc.src.stdbit.stdc_bit_floor_us + libc.src.stdbit.stdc_bit_floor_ui + libc.src.stdbit.stdc_bit_floor_ul + libc.src.stdbit.stdc_bit_floor_ull + libc.src.stdbit.stdc_bit_ceil_uc + libc.src.stdbit.stdc_bit_ceil_us + libc.src.stdbit.stdc_bit_ceil_ui + libc.src.stdbit.stdc_bit_ceil_ul + libc.src.stdbit.stdc_bit_ceil_ull # stdlib.h entrypoints libc.src.stdlib.abs diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index abd1f83794ed..7d69099e3cb9 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -131,6 +131,36 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdbit.stdc_first_trailing_one_ui libc.src.stdbit.stdc_first_trailing_one_ul libc.src.stdbit.stdc_first_trailing_one_ull + libc.src.stdbit.stdc_count_zeros_uc + libc.src.stdbit.stdc_count_zeros_us + libc.src.stdbit.stdc_count_zeros_ui + libc.src.stdbit.stdc_count_zeros_ul + libc.src.stdbit.stdc_count_zeros_ull + libc.src.stdbit.stdc_count_ones_uc + libc.src.stdbit.stdc_count_ones_us + libc.src.stdbit.stdc_count_ones_ui + libc.src.stdbit.stdc_count_ones_ul + libc.src.stdbit.stdc_count_ones_ull + libc.src.stdbit.stdc_has_single_bit_uc + libc.src.stdbit.stdc_has_single_bit_us + libc.src.stdbit.stdc_has_single_bit_ui + libc.src.stdbit.stdc_has_single_bit_ul + libc.src.stdbit.stdc_has_single_bit_ull + libc.src.stdbit.stdc_bit_width_uc + libc.src.stdbit.stdc_bit_width_us + libc.src.stdbit.stdc_bit_width_ui + libc.src.stdbit.stdc_bit_width_ul + libc.src.stdbit.stdc_bit_width_ull + libc.src.stdbit.stdc_bit_floor_uc + libc.src.stdbit.stdc_bit_floor_us + libc.src.stdbit.stdc_bit_floor_ui + libc.src.stdbit.stdc_bit_floor_ul + libc.src.stdbit.stdc_bit_floor_ull + libc.src.stdbit.stdc_bit_ceil_uc + libc.src.stdbit.stdc_bit_ceil_us + libc.src.stdbit.stdc_bit_ceil_ui + libc.src.stdbit.stdc_bit_ceil_ul + libc.src.stdbit.stdc_bit_ceil_ull # stdlib.h entrypoints libc.src.stdlib.abs diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt index 2fca96c5601b..bf1559b2f023 100644 --- a/libc/config/linux/arm/entrypoints.txt +++ b/libc/config/linux/arm/entrypoints.txt @@ -108,6 +108,36 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdbit.stdc_first_trailing_one_ui libc.src.stdbit.stdc_first_trailing_one_ul libc.src.stdbit.stdc_first_trailing_one_ull + libc.src.stdbit.stdc_count_zeros_uc + libc.src.stdbit.stdc_count_zeros_us + libc.src.stdbit.stdc_count_zeros_ui + libc.src.stdbit.stdc_count_zeros_ul + libc.src.stdbit.stdc_count_zeros_ull + libc.src.stdbit.stdc_count_ones_uc + libc.src.stdbit.stdc_count_ones_us + libc.src.stdbit.stdc_count_ones_ui + libc.src.stdbit.stdc_count_ones_ul + libc.src.stdbit.stdc_count_ones_ull + libc.src.stdbit.stdc_has_single_bit_uc + libc.src.stdbit.stdc_has_single_bit_us + libc.src.stdbit.stdc_has_single_bit_ui + libc.src.stdbit.stdc_has_single_bit_ul + libc.src.stdbit.stdc_has_single_bit_ull + libc.src.stdbit.stdc_bit_width_uc + libc.src.stdbit.stdc_bit_width_us + libc.src.stdbit.stdc_bit_width_ui + libc.src.stdbit.stdc_bit_width_ul + libc.src.stdbit.stdc_bit_width_ull + libc.src.stdbit.stdc_bit_floor_uc + libc.src.stdbit.stdc_bit_floor_us + libc.src.stdbit.stdc_bit_floor_ui + libc.src.stdbit.stdc_bit_floor_ul + libc.src.stdbit.stdc_bit_floor_ull + libc.src.stdbit.stdc_bit_ceil_uc + libc.src.stdbit.stdc_bit_ceil_us + libc.src.stdbit.stdc_bit_ceil_ui + libc.src.stdbit.stdc_bit_ceil_ul + libc.src.stdbit.stdc_bit_ceil_ull # stdlib.h entrypoints libc.src.stdlib.abs diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt index 006aa787ea6a..b1c9dd0428ee 100644 --- a/libc/config/linux/riscv/entrypoints.txt +++ b/libc/config/linux/riscv/entrypoints.txt @@ -132,6 +132,36 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdbit.stdc_first_trailing_one_ui libc.src.stdbit.stdc_first_trailing_one_ul libc.src.stdbit.stdc_first_trailing_one_ull + libc.src.stdbit.stdc_count_zeros_uc + libc.src.stdbit.stdc_count_zeros_us + libc.src.stdbit.stdc_count_zeros_ui + libc.src.stdbit.stdc_count_zeros_ul + libc.src.stdbit.stdc_count_zeros_ull + libc.src.stdbit.stdc_count_ones_uc + libc.src.stdbit.stdc_count_ones_us + libc.src.stdbit.stdc_count_ones_ui + libc.src.stdbit.stdc_count_ones_ul + libc.src.stdbit.stdc_count_ones_ull + libc.src.stdbit.stdc_has_single_bit_uc + libc.src.stdbit.stdc_has_single_bit_us + libc.src.stdbit.stdc_has_single_bit_ui + libc.src.stdbit.stdc_has_single_bit_ul + libc.src.stdbit.stdc_has_single_bit_ull + libc.src.stdbit.stdc_bit_width_uc + libc.src.stdbit.stdc_bit_width_us + libc.src.stdbit.stdc_bit_width_ui + libc.src.stdbit.stdc_bit_width_ul + libc.src.stdbit.stdc_bit_width_ull + libc.src.stdbit.stdc_bit_floor_uc + libc.src.stdbit.stdc_bit_floor_us + libc.src.stdbit.stdc_bit_floor_ui + libc.src.stdbit.stdc_bit_floor_ul + libc.src.stdbit.stdc_bit_floor_ull + libc.src.stdbit.stdc_bit_ceil_uc + libc.src.stdbit.stdc_bit_ceil_us + libc.src.stdbit.stdc_bit_ceil_ui + libc.src.stdbit.stdc_bit_ceil_ul + libc.src.stdbit.stdc_bit_ceil_ull # stdlib.h entrypoints libc.src.stdlib.abs -- GitLab From bb82092de71466728630050691fa9c20796b3cbc Mon Sep 17 00:00:00 2001 From: Han-Chung Wang Date: Wed, 13 Mar 2024 08:52:05 -0700 Subject: [PATCH 0141/1407] [mlir][tensor] Make getMixedPadImpl return static values when possible. (#85016) If low and high are constants (i.e., not attributes), users still prefer attributes. Otherwise, there could be failures in type inference. A failure is introduced by https://github.com/llvm/llvm-project/commit/60e562d11aeca8020de8d50ded7f0ba9e10e8843, see the drop_known_unit_constant_low_high test for more details. --- .../mlir/Dialect/Tensor/IR/TensorOps.td | 2 +- .../TensorToLinalg/tensor-ops-to-linalg.mlir | 3 +-- .../Dialect/Linalg/drop-unit-extent-dims.mlir | 20 +++++++++++++++++++ .../Dialect/Linalg/generalize-pad-tensor.mlir | 3 +-- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td index 670202fe4372..cf7f3e89079c 100644 --- a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td +++ b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td @@ -1364,7 +1364,7 @@ def Tensor_PadOp : Tensor_Op<"pad", [ unsigned count = staticAttrs.size(); for (unsigned idx = 0; idx < count; ++idx) { if (ShapedType::isDynamic(staticAttrs[idx])) - res.push_back(values[numDynamic++]); + res.push_back(getAsOpFoldResult(values[numDynamic++])); else res.push_back(builder.getI64IntegerAttr(staticAttrs[idx])); } diff --git a/mlir/test/Conversion/TensorToLinalg/tensor-ops-to-linalg.mlir b/mlir/test/Conversion/TensorToLinalg/tensor-ops-to-linalg.mlir index 238c0c51312a..a0a676edceb7 100644 --- a/mlir/test/Conversion/TensorToLinalg/tensor-ops-to-linalg.mlir +++ b/mlir/test/Conversion/TensorToLinalg/tensor-ops-to-linalg.mlir @@ -22,7 +22,6 @@ func.func @generalize_pad_tensor_static_shape(%arg0: tensor<1x28x28x1xf32>) -> t // CHECK-LABEL: func @generalize_pad_tensor_dynamic_shape( // CHECK-SAME: %[[IN:.*]]: tensor<4x?x2x?xf32>, // CHECK-SAME: %[[OFFSET:.*]]: index) -> tensor<4x?x?x?xf32> { -// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index // CHECK-DAG: %[[CST:.*]] = arith.constant 0.000000e+00 : f32 // CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index // CHECK: %[[DIM1:.*]] = tensor.dim %[[IN]], %[[C1]] : tensor<4x?x2x?xf32> @@ -33,7 +32,7 @@ func.func @generalize_pad_tensor_static_shape(%arg0: tensor<1x28x28x1xf32>) -> t // CHECK: %[[OUT_DIM3:.*]] = arith.addi %[[DIM3]], %[[OFFSET]] : index // CHECK: %[[INIT:.*]] = tensor.empty(%[[DIM1]], %[[OUT_DIM2]], %[[OUT_DIM3]]) : tensor<4x?x?x?xf32> // CHECK: %[[FILL:.*]] = linalg.fill ins(%[[CST]] : f32) outs(%[[INIT]] : tensor<4x?x?x?xf32>) -> tensor<4x?x?x?xf32> -// CHECK: %[[PADDED:.*]] = tensor.insert_slice %[[IN]] into %[[FILL]]{{\[}}%[[C0]], %[[C0]], %[[OFFSET]], %[[C0]]] [4, %[[DIM1]], 2, %[[DIM3]]] [1, 1, 1, 1] : tensor<4x?x2x?xf32> into tensor<4x?x?x?xf32> +// CHECK: %[[PADDED:.*]] = tensor.insert_slice %[[IN]] into %[[FILL]][0, 0, %[[OFFSET]], 0] [4, %[[DIM1]], 2, %[[DIM3]]] [1, 1, 1, 1] : tensor<4x?x2x?xf32> into tensor<4x?x?x?xf32> // CHECK: return %[[PADDED]] : tensor<4x?x?x?xf32> // CHECK: } func.func @generalize_pad_tensor_dynamic_shape(%arg0: tensor<4x?x2x?xf32>, %arg1: index) -> tensor<4x?x?x?xf32> { diff --git a/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir b/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir index f2c490b83207..c140b6abcc37 100644 --- a/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir +++ b/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir @@ -1033,3 +1033,23 @@ func.func @do_not_drop_non_constant_padding(%arg0: tensor<1x1x3x1x1xf32>, %pad: // CHECK-SLICES-LABEL: func @do_not_drop_non_constant_padding // CHECK-SLICES: tensor.pad %{{.*}} low[0, 1, 0, %c0, 0] high[0, 0, 0, %c0, 2] // CHECK-SLICES: } : tensor<1x1x3x1x1xf32> to tensor<1x2x3x1x3xf32> + +// ----- + +func.func @drop_known_unit_constant_low_high(%arg0: tensor<1x383x128xf32>) -> tensor<1x384x128xf32> { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %cst = arith.constant 0.000000e+00 : f32 + %padded = tensor.pad %arg0 low[%c0, %c1, %c0] high[%c0, %c0, %c0] { + ^bb0(%arg1: index, %arg2: index, %arg3: index): + tensor.yield %cst : f32 + } : tensor<1x383x128xf32> to tensor<1x384x128xf32> + return %padded : tensor<1x384x128xf32> +} +// CHECK-LABEL: func @drop_known_unit_constant_low_high +// CHECK: %[[COLLAPSE:.+]] = tensor.collapse_shape +// CHECK-SAME: {{\[}}[0, 1], [2]] : tensor<1x383x128xf32> into tensor<383x128xf32> +// CHECK: %[[PADDED:.+]] = tensor.pad %[[COLLAPSE]] low[1, 0] high[0, 0] +// CHECK: } : tensor<383x128xf32> to tensor<384x128xf32> +// CHECK: tensor.expand_shape %[[PADDED]] +// CHECK-SAME: {{\[}}[0, 1], [2]] : tensor<384x128xf32> into tensor<1x384x128xf32> diff --git a/mlir/test/Dialect/Linalg/generalize-pad-tensor.mlir b/mlir/test/Dialect/Linalg/generalize-pad-tensor.mlir index ac0eb48fb379..2beab31b613d 100644 --- a/mlir/test/Dialect/Linalg/generalize-pad-tensor.mlir +++ b/mlir/test/Dialect/Linalg/generalize-pad-tensor.mlir @@ -19,7 +19,6 @@ func.func @generalize_pad_tensor_static_shape(%arg0: tensor<1x28x28x1xf32>) -> t // CHECK-LABEL: func @generalize_pad_tensor_dynamic_shape( // CHECK-SAME: %[[IN:.*]]: tensor<4x?x2x?xf32>, // CHECK-SAME: %[[OFFSET:.*]]: index) -> tensor<4x?x?x?xf32> { -// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index // CHECK-DAG: %[[CST:.*]] = arith.constant 0.000000e+00 : f32 // CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index // CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index @@ -32,7 +31,7 @@ func.func @generalize_pad_tensor_static_shape(%arg0: tensor<1x28x28x1xf32>) -> t // CHECK: %[[FILL:.*]] = linalg.fill ins(%[[CST]] : f32) outs(%[[INIT]] : tensor<4x?x?x?xf32>) -> tensor<4x?x?x?xf32> // CHECK: %[[DIM1_1:.*]] = tensor.dim %[[IN]], %[[C1]] : tensor<4x?x2x?xf32> // CHECK: %[[DIM3_1:.*]] = tensor.dim %[[IN]], %[[C3]] : tensor<4x?x2x?xf32> -// CHECK: %[[PADDED:.*]] = tensor.insert_slice %[[IN]] into %[[FILL]]{{\[}}%[[C0]], %[[C0]], %[[OFFSET]], %[[C0]]] [4, %[[DIM1_1]], 2, %[[DIM3_1]]] [1, 1, 1, 1] : tensor<4x?x2x?xf32> into tensor<4x?x?x?xf32> +// CHECK: %[[PADDED:.*]] = tensor.insert_slice %[[IN]] into %[[FILL]][0, 0, %[[OFFSET]], 0] [4, %[[DIM1_1]], 2, %[[DIM3_1]]] [1, 1, 1, 1] : tensor<4x?x2x?xf32> into tensor<4x?x?x?xf32> // CHECK: return %[[PADDED]] : tensor<4x?x?x?xf32> // CHECK: } func.func @generalize_pad_tensor_dynamic_shape(%arg0: tensor<4x?x2x?xf32>, %arg1: index) -> tensor<4x?x?x?xf32> { -- GitLab From c3eccf03b365a705bc8dc043217478a82bc37a4d Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Wed, 13 Mar 2024 08:53:13 -0700 Subject: [PATCH 0142/1407] Avoid a potential exit(1) in LLVMContext::diagnose() (#84992) by handling *all* errors in IRExecDiagnosticHandler. The function that call this handles all unhandled errors with an `exit(1)`. rdar://124459751 I don't really have a testcase for this, since the crash report I got for this involved the Swift language plugin. --- lldb/source/Expression/IRExecutionUnit.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp index e4e131d70d43..cb9bee8733e1 100644 --- a/lldb/source/Expression/IRExecutionUnit.cpp +++ b/lldb/source/Expression/IRExecutionUnit.cpp @@ -212,18 +212,17 @@ struct IRExecDiagnosticHandler : public llvm::DiagnosticHandler { Status *err; IRExecDiagnosticHandler(Status *err) : err(err) {} bool handleDiagnostics(const llvm::DiagnosticInfo &DI) override { - if (DI.getKind() == llvm::DK_SrcMgr) { + if (DI.getSeverity() == llvm::DS_Error) { const auto &DISM = llvm::cast(DI); if (err && err->Success()) { err->SetErrorToGenericError(); err->SetErrorStringWithFormat( - "Inline assembly error: %s", + "IRExecution error: %s", DISM.getSMDiag().getMessage().str().c_str()); } - return true; } - return false; + return true; } }; } // namespace -- GitLab From cc761a7c356178009d186e70740ccb53bf0c6deb Mon Sep 17 00:00:00 2001 From: Zaara Syeda Date: Wed, 13 Mar 2024 11:57:07 -0400 Subject: [PATCH 0143/1407] [PowerPC][NFC] Rename ADDItocL to match the 64-bit naming convention (#85099) In preparation of adding a similar instruction for large code model on AIX for 32-bit, rename the exisitng ADDItocL 64-instruction to ADDItocL8 to match the naming convention of other instructions with 32-bit and 64-bit variants. --- .../Target/PowerPC/GISel/PPCInstructionSelector.cpp | 4 ++-- llvm/lib/Target/PowerPC/P10InstrResources.td | 2 +- llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp | 6 +++--- llvm/lib/Target/PowerPC/PPCBack2BackFusion.def | 4 ++-- llvm/lib/Target/PowerPC/PPCFastISel.cpp | 10 ++++++---- llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp | 10 +++++----- llvm/lib/Target/PowerPC/PPCInstr64Bit.td | 4 ++-- llvm/lib/Target/PowerPC/PPCInstrInfo.cpp | 12 ++++++------ llvm/lib/Target/PowerPC/PPCMacroFusion.def | 6 +++--- llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp | 3 +-- .../CodeGen/PowerPC/remove-copy-crunsetcrbit.mir | 2 +- 11 files changed, 32 insertions(+), 31 deletions(-) diff --git a/llvm/lib/Target/PowerPC/GISel/PPCInstructionSelector.cpp b/llvm/lib/Target/PowerPC/GISel/PPCInstructionSelector.cpp index 3fd7a1ad9efa..98cd3a82a6e0 100644 --- a/llvm/lib/Target/PowerPC/GISel/PPCInstructionSelector.cpp +++ b/llvm/lib/Target/PowerPC/GISel/PPCInstructionSelector.cpp @@ -695,8 +695,8 @@ bool PPCInstructionSelector::selectConstantPool( .addReg(HaAddrReg) .addMemOperand(MMO); else - // For medium code model, generate ADDItocL(CPI, ADDIStocHA8(X2, CPI)) - MI = BuildMI(MBB, I, DbgLoc, TII.get(PPC::ADDItocL), DstReg) + // For medium code model, generate ADDItocL8(CPI, ADDIStocHA8(X2, CPI)) + MI = BuildMI(MBB, I, DbgLoc, TII.get(PPC::ADDItocL8), DstReg) .addReg(HaAddrReg) .addConstantPoolIndex(CPI); } diff --git a/llvm/lib/Target/PowerPC/P10InstrResources.td b/llvm/lib/Target/PowerPC/P10InstrResources.td index 3bbc5a63ca7a..5015ba887d0b 100644 --- a/llvm/lib/Target/PowerPC/P10InstrResources.td +++ b/llvm/lib/Target/PowerPC/P10InstrResources.td @@ -881,7 +881,7 @@ def : InstRW<[P10W_FX_3C, P10W_DISP_ANY], // 3 Cycles ALU operations, 1 input operands def : InstRW<[P10W_FX_3C, P10W_DISP_ANY, P10FX_Read], (instrs - ADDI, ADDI8, ADDIdtprelL32, ADDItlsldLADDR32, ADDItocL, LI, LI8, + ADDI, ADDI8, ADDIdtprelL32, ADDItlsldLADDR32, ADDItocL8, LI, LI8, ADDIC, ADDIC8, ADDIS, ADDIS8, ADDISdtprelHA32, ADDIStocHA, ADDIStocHA8, LIS, LIS8, ADDME, ADDME8, diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp index 6f33b16f045a..542854ec9b99 100644 --- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -1236,8 +1236,8 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, TmpInst); return; } - case PPC::ADDItocL: { - // Transform %xd = ADDItocL %xs, @sym + case PPC::ADDItocL8: { + // Transform %xd = ADDItocL8 %xs, @sym LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); // Change the opcode to ADDI8. If the global address is external, then @@ -1246,7 +1246,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { TmpInst.setOpcode(PPC::ADDI8); const MachineOperand &MO = MI->getOperand(2); - assert((MO.isGlobal() || MO.isCPI()) && "Invalid operand for ADDItocL."); + assert((MO.isGlobal() || MO.isCPI()) && "Invalid operand for ADDItocL8."); LLVM_DEBUG(assert( !(MO.isGlobal() && Subtarget->isGVIndirectSymbol(MO.getGlobal())) && diff --git a/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def b/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def index 8bbe315a2bb9..6bb66bcc6c21 100644 --- a/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def +++ b/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def @@ -29,7 +29,7 @@ FUSION_FEATURE(GeneralBack2Back, hasBack2BackFusion, -1, ADDIStocHA8, ADDIdtprelL32, ADDItlsldLADDR32, - ADDItocL, + ADDItocL8, ADDME, ADDME8, ADDME8O, @@ -518,7 +518,7 @@ FUSION_FEATURE(GeneralBack2Back, hasBack2BackFusion, -1, ADDIStocHA8, ADDIdtprelL32, ADDItlsldLADDR32, - ADDItocL, + ADDItocL8, ADDME, ADDME8, ADDME8O, diff --git a/llvm/lib/Target/PowerPC/PPCFastISel.cpp b/llvm/lib/Target/PowerPC/PPCFastISel.cpp index 56af80f9cede..6e31cdae8476 100644 --- a/llvm/lib/Target/PowerPC/PPCFastISel.cpp +++ b/llvm/lib/Target/PowerPC/PPCFastISel.cpp @@ -2094,7 +2094,7 @@ unsigned PPCFastISel::PPCMaterializeGV(const GlobalValue *GV, MVT VT) { // for large code model, we generate: // LDtocL(GV, ADDIStocHA8(%x2, GV)) // Otherwise we generate: - // ADDItocL(ADDIStocHA8(%x2, GV), GV) + // ADDItocL8(ADDIStocHA8(%x2, GV), GV) // Either way, start with the ADDIStocHA8: Register HighPartReg = createResultReg(RC); BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(PPC::ADDIStocHA8), @@ -2104,9 +2104,11 @@ unsigned PPCFastISel::PPCMaterializeGV(const GlobalValue *GV, MVT VT) { BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(PPC::LDtocL), DestReg).addGlobalAddress(GV).addReg(HighPartReg); } else { - // Otherwise generate the ADDItocL. - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(PPC::ADDItocL), - DestReg).addReg(HighPartReg).addGlobalAddress(GV); + // Otherwise generate the ADDItocL8. + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(PPC::ADDItocL8), + DestReg) + .addReg(HighPartReg) + .addGlobalAddress(GV); } } diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp index 2462cbb19282..0c25accd1d6c 100644 --- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp @@ -6134,7 +6134,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) { // [64-bit ELF/AIX] // LDtocL(@sym, ADDIStocHA8(%x2, @sym)) // Otherwise we generate: - // ADDItocL(ADDIStocHA8(%x2, @sym), @sym) + // ADDItocL8(ADDIStocHA8(%x2, @sym), @sym) SDValue GA = N->getOperand(0); SDValue TOCbase = N->getOperand(1); @@ -6154,7 +6154,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) { } // Build the address relative to the TOC-pointer. - ReplaceNode(N, CurDAG->getMachineNode(PPC::ADDItocL, dl, MVT::i64, + ReplaceNode(N, CurDAG->getMachineNode(PPC::ADDItocL8, dl, MVT::i64, SDValue(Tmp, 0), GA)); return; } @@ -7707,7 +7707,7 @@ void PPCDAGToDAGISel::PeepholePPC64() { // target flags on the immediate operand when we fold it into the // load instruction. // - // For something like ADDItocL, the relocation information is + // For something like ADDItocL8, the relocation information is // inferred from the opcode; when we process it in the AsmPrinter, // we add the necessary relocation there. A load, though, can receive // relocation from various flavors of ADDIxxx, so we need to carry @@ -7728,7 +7728,7 @@ void PPCDAGToDAGISel::PeepholePPC64() { case PPC::ADDItlsldL: Flags = PPCII::MO_TLSLD_LO; break; - case PPC::ADDItocL: + case PPC::ADDItocL8: Flags = PPCII::MO_TOC_LO; break; } @@ -7755,7 +7755,7 @@ void PPCDAGToDAGISel::PeepholePPC64() { // If we have a addi(toc@l)/addis(toc@ha) pair, and the addis has only // one use, then we can do this for any offset, we just need to also // update the offset (i.e. the symbol addend) on the addis also. - if (Base.getMachineOpcode() != PPC::ADDItocL) + if (Base.getMachineOpcode() != PPC::ADDItocL8) continue; if (!HBase.isMachineOpcode() || diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td index 2949d58ab664..a9359794a641 100644 --- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td +++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td @@ -1480,8 +1480,8 @@ let hasSideEffects = 0 in { let isReMaterializable = 1 in { def ADDIStocHA8: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, tocentry:$disp), "#ADDIStocHA8", []>, isPPC64; -def ADDItocL: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, tocentry:$disp), - "#ADDItocL", []>, isPPC64; +def ADDItocL8: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, tocentry:$disp), + "#ADDItocL8", []>, isPPC64; } // Local Data Transform diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp index 5d37e929f875..5f5eb31a5a85 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp @@ -1077,7 +1077,7 @@ bool PPCInstrInfo::isReallyTriviallyReMaterializable( case PPC::LIS8: case PPC::ADDIStocHA: case PPC::ADDIStocHA8: - case PPC::ADDItocL: + case PPC::ADDItocL8: case PPC::LOAD_STACK_GUARD: case PPC::PPCLdFixedAddr: case PPC::XXLXORz: @@ -3453,7 +3453,7 @@ MachineInstr *PPCInstrInfo::getForwardingDefMI( break; case PPC::LI: case PPC::LI8: - case PPC::ADDItocL: + case PPC::ADDItocL8: case PPC::ADDI: case PPC::ADDI8: OpNoForForwarding = i; @@ -4420,7 +4420,7 @@ bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI, MachineOperand *&ImmMO, MachineOperand *&RegMO) const { unsigned Opc = DefMI.getOpcode(); - if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8) + if (Opc != PPC::ADDItocL8 && Opc != PPC::ADDI && Opc != PPC::ADDI8) return false; assert(DefMI.getNumOperands() >= 3 && @@ -4485,8 +4485,8 @@ bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO, int64_t &Imm, int64_t BaseImm) const { assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate"); - if (DefMI.getOpcode() == PPC::ADDItocL) { - // The operand for ADDItocL is CPI, which isn't imm at compiling time, + if (DefMI.getOpcode() == PPC::ADDItocL8) { + // The operand for ADDItocL8 is CPI, which isn't imm at compiling time, // However, we know that, it is 16-bit width, and has the alignment of 4. // Check if the instruction met the requirement. if (III.ImmMustBeMultipleOf > 4 || @@ -4899,7 +4899,7 @@ bool PPCInstrInfo::transformToImmFormFedByAdd( // register with ImmMO. // Before that, we need to fixup the target flags for imm. // For some reason, we miss to set the flag for the ImmMO if it is CPI. - if (DefMI.getOpcode() == PPC::ADDItocL) + if (DefMI.getOpcode() == PPC::ADDItocL8) ImmMO->setTargetFlags(PPCII::MO_TOC_LO); // MI didn't have the interface such as MI.setOperand(i) though diff --git a/llvm/lib/Target/PowerPC/PPCMacroFusion.def b/llvm/lib/Target/PowerPC/PPCMacroFusion.def index 6b8ad22639c8..fb6e656edb8b 100644 --- a/llvm/lib/Target/PowerPC/PPCMacroFusion.def +++ b/llvm/lib/Target/PowerPC/PPCMacroFusion.def @@ -32,7 +32,7 @@ // {addi} followed by one of these {lxvd2x, lxvw4x, lxvdsx, lvebx, lvehx, // lvewx, lvx, lxsdx} FUSION_FEATURE(AddiLoad, hasAddiLoadFusion, 2, \ - FUSION_OP_SET(ADDI, ADDI8, ADDItocL), \ + FUSION_OP_SET(ADDI, ADDI8, ADDItocL8), \ FUSION_OP_SET(LXVD2X, LXVW4X, LXVDSX, LVEBX, LVEHX, LVEWX, \ LVX, LXSDX)) @@ -135,11 +135,11 @@ FUSION_FEATURE(XorisXori, hasWideImmFusion, 1, FUSION_OP_SET(XORIS, XORIS8), // addis rx,ra,si - addi rt,rx,SI, SI >= 0 FUSION_FEATURE(AddisAddi, hasWideImmFusion, 1, FUSION_OP_SET(ADDIS, ADDIS8, ADDIStocHA8), - FUSION_OP_SET(ADDI, ADDI8, ADDItocL)) + FUSION_OP_SET(ADDI, ADDI8, ADDItocL8)) // addi rx,ra,si - addis rt,rx,SI, ra > 0, SI >= 2 FUSION_FEATURE(AddiAddis, hasWideImmFusion, 1, - FUSION_OP_SET(ADDI, ADDI8, ADDItocL), + FUSION_OP_SET(ADDI, ADDI8, ADDItocL8), FUSION_OP_SET(ADDIS, ADDIS8, ADDIStocHA8)) // mtctr - { bcctr,bcctrl } diff --git a/llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp b/llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp index 81f078ab246e..0527991b58ba 100644 --- a/llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp +++ b/llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp @@ -94,8 +94,7 @@ namespace { protected: bool hasTOCLoReloc(const MachineInstr &MI) { - if (MI.getOpcode() == PPC::LDtocL || - MI.getOpcode() == PPC::ADDItocL || + if (MI.getOpcode() == PPC::LDtocL || MI.getOpcode() == PPC::ADDItocL8 || MI.getOpcode() == PPC::LWZtocL) return true; diff --git a/llvm/test/CodeGen/PowerPC/remove-copy-crunsetcrbit.mir b/llvm/test/CodeGen/PowerPC/remove-copy-crunsetcrbit.mir index 3a312d2f4a8b..f3ef95bbb79a 100644 --- a/llvm/test/CodeGen/PowerPC/remove-copy-crunsetcrbit.mir +++ b/llvm/test/CodeGen/PowerPC/remove-copy-crunsetcrbit.mir @@ -130,7 +130,7 @@ body: | %22:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, @c %10:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, @e %13:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, @a - %14:g8rc_and_g8rc_nox0 = ADDItocL killed %13, @a, implicit $x2 + %14:g8rc_and_g8rc_nox0 = ADDItocL8 killed %13, @a, implicit $x2 bb.2.while.body: successors: %bb.4(0x30000000), %bb.3(0x50000000) -- GitLab From f15a790fd383665ec4defa0711e975476fd8b18b Mon Sep 17 00:00:00 2001 From: David Blaikie Date: Wed, 13 Mar 2024 15:55:18 +0000 Subject: [PATCH 0144/1407] Remove use of reference lifetime extension introduced in cdde0d9 Rather than dealing with which is more readable, the named variable doesn't seem to add value here - so omit it. --- clang/lib/AST/Interp/Interp.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h index bb220657c2da..db80e2d59753 100644 --- a/clang/lib/AST/Interp/Interp.h +++ b/clang/lib/AST/Interp/Interp.h @@ -846,8 +846,7 @@ bool CMP3(InterpState &S, CodePtr OpPC, const ComparisonCategoryInfo *CmpInfo) { CmpInfo->getValueInfo(CmpInfo->makeWeakResult(CmpResult)); assert(CmpValueInfo); assert(CmpValueInfo->hasValidIntValue()); - const APSInt &IntValue = CmpValueInfo->getIntValue(); - return SetThreeWayComparisonField(S, OpPC, P, IntValue); + return SetThreeWayComparisonField(S, OpPC, P, CmpValueInfo->getIntValue()); } template ::T> -- GitLab From 57b991ab39348d91d8552787958ba7db1e7ceb8a Mon Sep 17 00:00:00 2001 From: Usman Nadeem Date: Wed, 13 Mar 2024 09:05:55 -0700 Subject: [PATCH 0145/1407] [AArch64] Improve lowering of truncating uzp1 (#82457) There were two existing patterns: `concat_vectors(trunc(x), trunc(y)) -> uzp1(x, y)` `concat_vectors(assertzext(trunc(x)), assertzext(trunc(y))) -> uzp1(x, y)` Move them into a class and add the following `assertsext` pattern to it: `concat_vectors(assertsext(trunc(x)), assertsext(trunc(y))) -> uzp1(x, y)` Add the following transform for v8i8 and v4i16 result types to help with pattern matching: `truncating uzp1(x, y) -> trunc(concat(x, y))` And a pattern to go with it: `trunc(concat_vectors(x, y)) -> uzp1 (x, y)` Add another isel pattern for v8i8 and v4i16 result vector types, similar to the existing concat pattern, but with a trunc node in the begining: `trunc(concat_vectors(assertext_trunc(x), assertext_trunc(y))) -> xtn(uzp1(x, y))` --- .../Target/AArch64/AArch64ISelLowering.cpp | 39 +-- llvm/lib/Target/AArch64/AArch64InstrInfo.td | 53 ++-- .../CodeGen/AArch64/arm64-convert-v4f64.ll | 21 +- llvm/test/CodeGen/AArch64/extbinopload.ll | 31 ++- .../CodeGen/AArch64/fp-conversion-to-tbl.ll | 5 +- llvm/test/CodeGen/AArch64/fptoi.ll | 256 ++++++------------ llvm/test/CodeGen/AArch64/neon-truncstore.ll | 5 +- llvm/test/CodeGen/AArch64/sadd_sat_vec.ll | 2 +- llvm/test/CodeGen/AArch64/shuffle-tbl34.ll | 14 +- llvm/test/CodeGen/AArch64/ssub_sat_vec.ll | 2 +- llvm/test/CodeGen/AArch64/tbl-loops.ll | 4 +- llvm/test/CodeGen/AArch64/trunc-to-tbl.ll | 28 +- llvm/test/CodeGen/AArch64/uadd_sat_vec.ll | 2 +- llvm/test/CodeGen/AArch64/usub_sat_vec.ll | 2 +- llvm/test/CodeGen/AArch64/vcvt-oversize.ll | 5 +- .../vec-combine-compare-truncate-store.ll | 2 +- .../AArch64/vec3-loads-ext-trunc-stores.ll | 22 +- 17 files changed, 209 insertions(+), 284 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 5b7a36d2eba7..9665ae5ceb90 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -21423,12 +21423,8 @@ static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG, } } - // uzp1(xtn x, xtn y) -> xtn(uzp1 (x, y)) - // Only implemented on little-endian subtargets. - bool IsLittleEndian = DAG.getDataLayout().isLittleEndian(); - - // This optimization only works on little endian. - if (!IsLittleEndian) + // These optimizations only work on little endian. + if (!DAG.getDataLayout().isLittleEndian()) return SDValue(); // uzp1(bitcast(x), bitcast(y)) -> uzp1(x, y) @@ -21447,21 +21443,28 @@ static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG, if (ResVT != MVT::v2i32 && ResVT != MVT::v4i16 && ResVT != MVT::v8i8) return SDValue(); - auto getSourceOp = [](SDValue Operand) -> SDValue { - const unsigned Opcode = Operand.getOpcode(); - if (Opcode == ISD::TRUNCATE) - return Operand->getOperand(0); - if (Opcode == ISD::BITCAST && - Operand->getOperand(0).getOpcode() == ISD::TRUNCATE) - return Operand->getOperand(0)->getOperand(0); - return SDValue(); - }; + SDValue SourceOp0 = peekThroughBitcasts(Op0); + SDValue SourceOp1 = peekThroughBitcasts(Op1); - SDValue SourceOp0 = getSourceOp(Op0); - SDValue SourceOp1 = getSourceOp(Op1); + // truncating uzp1(x, y) -> xtn(concat (x, y)) + if (SourceOp0.getValueType() == SourceOp1.getValueType()) { + EVT Op0Ty = SourceOp0.getValueType(); + if ((ResVT == MVT::v4i16 && Op0Ty == MVT::v2i32) || + (ResVT == MVT::v8i8 && Op0Ty == MVT::v4i16)) { + SDValue Concat = + DAG.getNode(ISD::CONCAT_VECTORS, DL, + Op0Ty.getDoubleNumVectorElementsVT(*DAG.getContext()), + SourceOp0, SourceOp1); + return DAG.getNode(ISD::TRUNCATE, DL, ResVT, Concat); + } + } - if (!SourceOp0 || !SourceOp1) + // uzp1(xtn x, xtn y) -> xtn(uzp1 (x, y)) + if (SourceOp0.getOpcode() != ISD::TRUNCATE || + SourceOp1.getOpcode() != ISD::TRUNCATE) return SDValue(); + SourceOp0 = SourceOp0.getOperand(0); + SourceOp1 = SourceOp1.getOperand(0); if (SourceOp0.getValueType() != SourceOp1.getValueType() || !SourceOp0.getValueType().isSimple()) diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 6254e68326f7..b4b975cce007 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -6153,26 +6153,39 @@ defm UZP2 : SIMDZipVector<0b101, "uzp2", AArch64uzp2>; defm ZIP1 : SIMDZipVector<0b011, "zip1", AArch64zip1>; defm ZIP2 : SIMDZipVector<0b111, "zip2", AArch64zip2>; -def : Pat<(v16i8 (concat_vectors (v8i8 (trunc (v8i16 V128:$Vn))), - (v8i8 (trunc (v8i16 V128:$Vm))))), - (UZP1v16i8 V128:$Vn, V128:$Vm)>; -def : Pat<(v8i16 (concat_vectors (v4i16 (trunc (v4i32 V128:$Vn))), - (v4i16 (trunc (v4i32 V128:$Vm))))), - (UZP1v8i16 V128:$Vn, V128:$Vm)>; -def : Pat<(v4i32 (concat_vectors (v2i32 (trunc (v2i64 V128:$Vn))), - (v2i32 (trunc (v2i64 V128:$Vm))))), - (UZP1v4i32 V128:$Vn, V128:$Vm)>; -// These are the same as above, with an optional assertzext node that can be -// generated from fptoi lowering. -def : Pat<(v16i8 (concat_vectors (v8i8 (assertzext (trunc (v8i16 V128:$Vn)))), - (v8i8 (assertzext (trunc (v8i16 V128:$Vm)))))), - (UZP1v16i8 V128:$Vn, V128:$Vm)>; -def : Pat<(v8i16 (concat_vectors (v4i16 (assertzext (trunc (v4i32 V128:$Vn)))), - (v4i16 (assertzext (trunc (v4i32 V128:$Vm)))))), - (UZP1v8i16 V128:$Vn, V128:$Vm)>; -def : Pat<(v4i32 (concat_vectors (v2i32 (assertzext (trunc (v2i64 V128:$Vn)))), - (v2i32 (assertzext (trunc (v2i64 V128:$Vm)))))), - (UZP1v4i32 V128:$Vn, V128:$Vm)>; +def trunc_optional_assert_ext : PatFrags<(ops node:$op0), + [(trunc node:$op0), + (assertzext (trunc node:$op0)), + (assertsext (trunc node:$op0))]>; + +// concat_vectors(trunc(x), trunc(y)) -> uzp1(x, y) +// concat_vectors(assertzext(trunc(x)), assertzext(trunc(y))) -> uzp1(x, y) +// concat_vectors(assertsext(trunc(x)), assertsext(trunc(y))) -> uzp1(x, y) +class concat_trunc_to_uzp1_pat + : Pat<(ConcatTy (concat_vectors (TruncTy (trunc_optional_assert_ext (SrcTy V128:$Vn))), + (TruncTy (trunc_optional_assert_ext (SrcTy V128:$Vm))))), + (!cast("UZP1"#ConcatTy) V128:$Vn, V128:$Vm)>; +def : concat_trunc_to_uzp1_pat; +def : concat_trunc_to_uzp1_pat; +def : concat_trunc_to_uzp1_pat; + +// trunc(concat_vectors(trunc(x), trunc(y))) -> xtn(uzp1(x, y)) +// trunc(concat_vectors(assertzext(trunc(x)), assertzext(trunc(y)))) -> xtn(uzp1(x, y)) +// trunc(concat_vectors(assertsext(trunc(x)), assertsext(trunc(y)))) -> xtn(uzp1(x, y)) +class trunc_concat_trunc_to_xtn_uzp1_pat + : Pat<(Ty (trunc_optional_assert_ext + (ConcatTy (concat_vectors + (TruncTy (trunc_optional_assert_ext (SrcTy V128:$Vn))), + (TruncTy (trunc_optional_assert_ext (SrcTy V128:$Vm))))))), + (!cast("XTN"#Ty) (!cast("UZP1"#ConcatTy) V128:$Vn, V128:$Vm))>; +def : trunc_concat_trunc_to_xtn_uzp1_pat; +def : trunc_concat_trunc_to_xtn_uzp1_pat; + +def : Pat<(v8i8 (trunc (concat_vectors (v4i16 V64:$Vn), (v4i16 V64:$Vm)))), + (UZP1v8i8 V64:$Vn, V64:$Vm)>; +def : Pat<(v4i16 (trunc (concat_vectors (v2i32 V64:$Vn), (v2i32 V64:$Vm)))), + (UZP1v4i16 V64:$Vn, V64:$Vm)>; def : Pat<(v16i8 (concat_vectors (v8i8 (trunc (AArch64vlshr (v8i16 V128:$Vn), (i32 8)))), diff --git a/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll b/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll index 49325299f74a..3007e7ce771e 100644 --- a/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll +++ b/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll @@ -8,9 +8,8 @@ define <4 x i16> @fptosi_v4f64_to_v4i16(ptr %ptr) { ; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-NEXT: xtn v1.2s, v1.2d -; CHECK-NEXT: xtn v0.2s, v0.2d -; CHECK-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-NEXT: xtn v0.4h, v0.4s ; CHECK-NEXT: ret %tmp1 = load <4 x double>, ptr %ptr %tmp2 = fptosi <4 x double> %tmp1 to <4 x i16> @@ -26,13 +25,10 @@ define <8 x i8> @fptosi_v4f64_to_v4i8(ptr %ptr) { ; CHECK-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-NEXT: fcvtzs v3.2d, v3.2d ; CHECK-NEXT: fcvtzs v2.2d, v2.2d -; CHECK-NEXT: xtn v0.2s, v0.2d -; CHECK-NEXT: xtn v1.2s, v1.2d -; CHECK-NEXT: xtn v3.2s, v3.2d -; CHECK-NEXT: xtn v2.2s, v2.2d -; CHECK-NEXT: uzp1 v0.4h, v1.4h, v0.4h -; CHECK-NEXT: uzp1 v1.4h, v2.4h, v3.4h -; CHECK-NEXT: uzp1 v0.8b, v1.8b, v0.8b +; CHECK-NEXT: uzp1 v0.4s, v1.4s, v0.4s +; CHECK-NEXT: uzp1 v1.4s, v2.4s, v3.4s +; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h +; CHECK-NEXT: xtn v0.8b, v0.8h ; CHECK-NEXT: ret %tmp1 = load <8 x double>, ptr %ptr %tmp2 = fptosi <8 x double> %tmp1 to <8 x i8> @@ -96,9 +92,8 @@ define <4 x i16> @fptoui_v4f64_to_v4i16(ptr %ptr) { ; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-NEXT: xtn v1.2s, v1.2d -; CHECK-NEXT: xtn v0.2s, v0.2d -; CHECK-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-NEXT: xtn v0.4h, v0.4s ; CHECK-NEXT: ret %tmp1 = load <4 x double>, ptr %ptr %tmp2 = fptoui <4 x double> %tmp1 to <4 x i16> diff --git a/llvm/test/CodeGen/AArch64/extbinopload.ll b/llvm/test/CodeGen/AArch64/extbinopload.ll index 1f68c77611e1..dff4831330de 100644 --- a/llvm/test/CodeGen/AArch64/extbinopload.ll +++ b/llvm/test/CodeGen/AArch64/extbinopload.ll @@ -650,7 +650,7 @@ define <16 x i32> @extrause_load(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: add x11, x3, #12 ; CHECK-NEXT: str s1, [x4] ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: ldp s0, s5, [x2] +; CHECK-NEXT: ldp s0, s4, [x2] ; CHECK-NEXT: ushll v2.8h, v0.8b, #0 ; CHECK-NEXT: umov w9, v2.h[0] ; CHECK-NEXT: umov w10, v2.h[1] @@ -662,24 +662,25 @@ define <16 x i32> @extrause_load(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: ushll v2.8h, v2.8b, #0 ; CHECK-NEXT: mov v0.b[10], w9 ; CHECK-NEXT: add x9, x1, #4 -; CHECK-NEXT: uzp1 v1.8b, v1.8b, v2.8b +; CHECK-NEXT: mov v1.d[1], v2.d[0] ; CHECK-NEXT: mov v0.b[11], w10 ; CHECK-NEXT: add x10, x1, #12 +; CHECK-NEXT: bic v1.8h, #255, lsl #8 ; CHECK-NEXT: ld1 { v0.s }[3], [x3], #4 -; CHECK-NEXT: ldr s4, [x0, #12] -; CHECK-NEXT: ldp s3, s16, [x0, #4] -; CHECK-NEXT: ld1 { v5.s }[1], [x3] -; CHECK-NEXT: ldp s6, s7, [x2, #8] -; CHECK-NEXT: ld1 { v4.s }[1], [x10] -; CHECK-NEXT: ld1 { v3.s }[1], [x9] -; CHECK-NEXT: ld1 { v6.s }[1], [x8] -; CHECK-NEXT: ld1 { v7.s }[1], [x11] +; CHECK-NEXT: ldr s3, [x0, #12] +; CHECK-NEXT: ldp s2, s7, [x0, #4] +; CHECK-NEXT: ld1 { v4.s }[1], [x3] +; CHECK-NEXT: ldp s5, s6, [x2, #8] +; CHECK-NEXT: ld1 { v3.s }[1], [x10] +; CHECK-NEXT: ld1 { v2.s }[1], [x9] +; CHECK-NEXT: ld1 { v5.s }[1], [x8] +; CHECK-NEXT: ld1 { v6.s }[1], [x11] ; CHECK-NEXT: add x8, x1, #8 -; CHECK-NEXT: ld1 { v16.s }[1], [x8] -; CHECK-NEXT: uaddl v2.8h, v3.8b, v4.8b -; CHECK-NEXT: ushll v3.8h, v6.8b, #0 -; CHECK-NEXT: uaddl v4.8h, v5.8b, v7.8b -; CHECK-NEXT: uaddl v1.8h, v1.8b, v16.8b +; CHECK-NEXT: ld1 { v7.s }[1], [x8] +; CHECK-NEXT: uaddl v2.8h, v2.8b, v3.8b +; CHECK-NEXT: ushll v3.8h, v5.8b, #0 +; CHECK-NEXT: uaddl v4.8h, v4.8b, v6.8b +; CHECK-NEXT: uaddw v1.8h, v1.8h, v7.8b ; CHECK-NEXT: uaddw2 v5.8h, v3.8h, v0.16b ; CHECK-NEXT: ushll v0.4s, v2.4h, #3 ; CHECK-NEXT: ushll2 v2.4s, v2.8h, #3 diff --git a/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll b/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll index 1ea87bb6b04b..0a3b9a070c2b 100644 --- a/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll +++ b/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll @@ -73,9 +73,8 @@ define void @fptoui_v8f32_to_v8i8_no_loop(ptr %A, ptr %dst) { ; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: fcvtzs.4s v1, v1 ; CHECK-NEXT: fcvtzs.4s v0, v0 -; CHECK-NEXT: xtn.4h v1, v1 -; CHECK-NEXT: xtn.4h v0, v0 -; CHECK-NEXT: uzp1.8b v0, v0, v1 +; CHECK-NEXT: uzp1.8h v0, v0, v1 +; CHECK-NEXT: xtn.8b v0, v0 ; CHECK-NEXT: str d0, [x1] ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/fptoi.ll b/llvm/test/CodeGen/AArch64/fptoi.ll index 67190e8596c4..7af01b53dae7 100644 --- a/llvm/test/CodeGen/AArch64/fptoi.ll +++ b/llvm/test/CodeGen/AArch64/fptoi.ll @@ -1096,30 +1096,17 @@ entry: } define <3 x i16> @fptos_v3f64_v3i16(<3 x double> %a) { -; CHECK-SD-LABEL: fptos_v3f64_v3i16: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2 -; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] -; CHECK-SD-NEXT: fcvtzs v1.2d, v2.2d -; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: fptos_v3f64_v3i16: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2 -; CHECK-GI-NEXT: mov v0.d[1], v1.d[0] -; CHECK-GI-NEXT: fcvtzs v1.2d, v2.2d -; CHECK-GI-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-GI-NEXT: xtn v0.4h, v0.4s -; CHECK-GI-NEXT: ret +; CHECK-LABEL: fptos_v3f64_v3i16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2 +; CHECK-NEXT: mov v0.d[1], v1.d[0] +; CHECK-NEXT: fcvtzs v1.2d, v2.2d +; CHECK-NEXT: fcvtzs v0.2d, v0.2d +; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-NEXT: xtn v0.4h, v0.4s +; CHECK-NEXT: ret entry: %c = fptosi <3 x double> %a to <3 x i16> ret <3 x i16> %c @@ -1134,9 +1121,8 @@ define <3 x i16> @fptou_v3f64_v3i16(<3 x double> %a) { ; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] ; CHECK-SD-NEXT: fcvtzs v1.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v3f64_v3i16: @@ -1160,9 +1146,8 @@ define <4 x i16> @fptos_v4f64_v4i16(<4 x double> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v4f64_v4i16: @@ -1182,9 +1167,8 @@ define <4 x i16> @fptou_v4f64_v4i16(<4 x double> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v4f64_v4i16: @@ -1600,9 +1584,8 @@ define <3 x i8> @fptos_v3f64_v3i8(<3 x double> %a) { ; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] ; CHECK-SD-NEXT: fcvtzs v1.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: umov w0, v0.h[0] ; CHECK-SD-NEXT: umov w1, v0.h[1] ; CHECK-SD-NEXT: umov w2, v0.h[2] @@ -1638,9 +1621,8 @@ define <3 x i8> @fptou_v3f64_v3i8(<3 x double> %a) { ; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] ; CHECK-SD-NEXT: fcvtzs v1.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: umov w0, v0.h[0] ; CHECK-SD-NEXT: umov w1, v0.h[1] ; CHECK-SD-NEXT: umov w2, v0.h[2] @@ -1672,9 +1654,8 @@ define <4 x i8> @fptos_v4f64_v4i8(<4 x double> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v4f64_v4i8: @@ -1694,9 +1675,8 @@ define <4 x i8> @fptou_v4f64_v4i8(<4 x double> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v4f64_v4i8: @@ -1718,13 +1698,10 @@ define <8 x i8> @fptos_v8f64_v8i8(<8 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v2.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v3.2s, v3.2d -; CHECK-SD-NEXT: xtn v2.2s, v2.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v2.8b +; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h +; CHECK-SD-NEXT: xtn v0.8b, v0.8h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v8f64_v8i8: @@ -1750,13 +1727,10 @@ define <8 x i8> @fptou_v8f64_v8i8(<8 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v2.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v3.2s, v3.2d -; CHECK-SD-NEXT: xtn v2.2s, v2.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v2.8b +; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h +; CHECK-SD-NEXT: xtn v0.8b, v0.8h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v8f64_v8i8: @@ -1786,21 +1760,13 @@ define <16 x i8> @fptos_v16f64_v16i8(<16 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v2.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v7.2s, v7.2d -; CHECK-SD-NEXT: xtn v6.2s, v6.2d -; CHECK-SD-NEXT: xtn v5.2s, v5.2d -; CHECK-SD-NEXT: xtn v4.2s, v4.2d -; CHECK-SD-NEXT: xtn v3.2s, v3.2d -; CHECK-SD-NEXT: xtn v2.2s, v2.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v6.4h, v6.4h, v7.4h -; CHECK-SD-NEXT: uzp1 v4.4h, v4.4h, v5.4h -; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: mov v4.d[1], v6.d[0] -; CHECK-SD-NEXT: mov v0.d[1], v2.d[0] -; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v4.16b +; CHECK-SD-NEXT: uzp1 v6.4s, v6.4s, v7.4s +; CHECK-SD-NEXT: uzp1 v4.4s, v4.4s, v5.4s +; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: uzp1 v1.8h, v4.8h, v6.8h +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h +; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v16f64_v16i8: @@ -1837,21 +1803,13 @@ define <16 x i8> @fptou_v16f64_v16i8(<16 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v2.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v7.2s, v7.2d -; CHECK-SD-NEXT: xtn v6.2s, v6.2d -; CHECK-SD-NEXT: xtn v5.2s, v5.2d -; CHECK-SD-NEXT: xtn v4.2s, v4.2d -; CHECK-SD-NEXT: xtn v3.2s, v3.2d -; CHECK-SD-NEXT: xtn v2.2s, v2.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v6.4h, v6.4h, v7.4h -; CHECK-SD-NEXT: uzp1 v4.4h, v4.4h, v5.4h -; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: mov v4.d[1], v6.d[0] -; CHECK-SD-NEXT: mov v0.d[1], v2.d[0] -; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v4.16b +; CHECK-SD-NEXT: uzp1 v6.4s, v6.4s, v7.4s +; CHECK-SD-NEXT: uzp1 v4.4s, v4.4s, v5.4s +; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: uzp1 v1.8h, v4.8h, v6.8h +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h +; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v16f64_v16i8: @@ -1900,36 +1858,20 @@ define <32 x i8> @fptos_v32f64_v32i8(<32 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v18.2d, v18.2d ; CHECK-SD-NEXT: fcvtzs v17.2d, v17.2d ; CHECK-SD-NEXT: fcvtzs v16.2d, v16.2d -; CHECK-SD-NEXT: xtn v7.2s, v7.2d -; CHECK-SD-NEXT: xtn v6.2s, v6.2d -; CHECK-SD-NEXT: xtn v5.2s, v5.2d -; CHECK-SD-NEXT: xtn v4.2s, v4.2d -; CHECK-SD-NEXT: xtn v3.2s, v3.2d -; CHECK-SD-NEXT: xtn v2.2s, v2.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: xtn v23.2s, v23.2d -; CHECK-SD-NEXT: xtn v22.2s, v22.2d -; CHECK-SD-NEXT: xtn v21.2s, v21.2d -; CHECK-SD-NEXT: xtn v20.2s, v20.2d -; CHECK-SD-NEXT: xtn v19.2s, v19.2d -; CHECK-SD-NEXT: xtn v18.2s, v18.2d -; CHECK-SD-NEXT: xtn v17.2s, v17.2d -; CHECK-SD-NEXT: xtn v16.2s, v16.2d -; CHECK-SD-NEXT: uzp1 v6.4h, v6.4h, v7.4h -; CHECK-SD-NEXT: uzp1 v4.4h, v4.4h, v5.4h -; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: uzp1 v1.4h, v22.4h, v23.4h -; CHECK-SD-NEXT: uzp1 v3.4h, v20.4h, v21.4h -; CHECK-SD-NEXT: uzp1 v5.4h, v18.4h, v19.4h -; CHECK-SD-NEXT: uzp1 v7.4h, v16.4h, v17.4h -; CHECK-SD-NEXT: mov v4.d[1], v6.d[0] -; CHECK-SD-NEXT: mov v0.d[1], v2.d[0] -; CHECK-SD-NEXT: mov v3.d[1], v1.d[0] -; CHECK-SD-NEXT: mov v7.d[1], v5.d[0] +; CHECK-SD-NEXT: uzp1 v6.4s, v6.4s, v7.4s +; CHECK-SD-NEXT: uzp1 v4.4s, v4.4s, v5.4s +; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: uzp1 v3.4s, v20.4s, v21.4s +; CHECK-SD-NEXT: uzp1 v1.4s, v22.4s, v23.4s +; CHECK-SD-NEXT: uzp1 v5.4s, v18.4s, v19.4s +; CHECK-SD-NEXT: uzp1 v7.4s, v16.4s, v17.4s +; CHECK-SD-NEXT: uzp1 v4.8h, v4.8h, v6.8h +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h +; CHECK-SD-NEXT: uzp1 v1.8h, v3.8h, v1.8h +; CHECK-SD-NEXT: uzp1 v2.8h, v7.8h, v5.8h ; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v4.16b -; CHECK-SD-NEXT: uzp1 v1.16b, v7.16b, v3.16b +; CHECK-SD-NEXT: uzp1 v1.16b, v2.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v32f64_v32i8: @@ -1997,36 +1939,20 @@ define <32 x i8> @fptou_v32f64_v32i8(<32 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v18.2d, v18.2d ; CHECK-SD-NEXT: fcvtzs v17.2d, v17.2d ; CHECK-SD-NEXT: fcvtzs v16.2d, v16.2d -; CHECK-SD-NEXT: xtn v7.2s, v7.2d -; CHECK-SD-NEXT: xtn v6.2s, v6.2d -; CHECK-SD-NEXT: xtn v5.2s, v5.2d -; CHECK-SD-NEXT: xtn v4.2s, v4.2d -; CHECK-SD-NEXT: xtn v3.2s, v3.2d -; CHECK-SD-NEXT: xtn v2.2s, v2.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: xtn v23.2s, v23.2d -; CHECK-SD-NEXT: xtn v22.2s, v22.2d -; CHECK-SD-NEXT: xtn v21.2s, v21.2d -; CHECK-SD-NEXT: xtn v20.2s, v20.2d -; CHECK-SD-NEXT: xtn v19.2s, v19.2d -; CHECK-SD-NEXT: xtn v18.2s, v18.2d -; CHECK-SD-NEXT: xtn v17.2s, v17.2d -; CHECK-SD-NEXT: xtn v16.2s, v16.2d -; CHECK-SD-NEXT: uzp1 v6.4h, v6.4h, v7.4h -; CHECK-SD-NEXT: uzp1 v4.4h, v4.4h, v5.4h -; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: uzp1 v1.4h, v22.4h, v23.4h -; CHECK-SD-NEXT: uzp1 v3.4h, v20.4h, v21.4h -; CHECK-SD-NEXT: uzp1 v5.4h, v18.4h, v19.4h -; CHECK-SD-NEXT: uzp1 v7.4h, v16.4h, v17.4h -; CHECK-SD-NEXT: mov v4.d[1], v6.d[0] -; CHECK-SD-NEXT: mov v0.d[1], v2.d[0] -; CHECK-SD-NEXT: mov v3.d[1], v1.d[0] -; CHECK-SD-NEXT: mov v7.d[1], v5.d[0] +; CHECK-SD-NEXT: uzp1 v6.4s, v6.4s, v7.4s +; CHECK-SD-NEXT: uzp1 v4.4s, v4.4s, v5.4s +; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: uzp1 v3.4s, v20.4s, v21.4s +; CHECK-SD-NEXT: uzp1 v1.4s, v22.4s, v23.4s +; CHECK-SD-NEXT: uzp1 v5.4s, v18.4s, v19.4s +; CHECK-SD-NEXT: uzp1 v7.4s, v16.4s, v17.4s +; CHECK-SD-NEXT: uzp1 v4.8h, v4.8h, v6.8h +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h +; CHECK-SD-NEXT: uzp1 v1.8h, v3.8h, v1.8h +; CHECK-SD-NEXT: uzp1 v2.8h, v7.8h, v5.8h ; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v4.16b -; CHECK-SD-NEXT: uzp1 v1.16b, v7.16b, v3.16b +; CHECK-SD-NEXT: uzp1 v1.16b, v2.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v32f64_v32i8: @@ -3026,9 +2952,8 @@ define <8 x i8> @fptos_v8f32_v8i8(<8 x float> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.4s, v1.4s ; CHECK-SD-NEXT: fcvtzs v0.4s, v0.4s -; CHECK-SD-NEXT: xtn v1.4h, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s -; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v1.8b +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h +; CHECK-SD-NEXT: xtn v0.8b, v0.8h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v8f32_v8i8: @@ -3048,9 +2973,8 @@ define <8 x i8> @fptou_v8f32_v8i8(<8 x float> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.4s, v1.4s ; CHECK-SD-NEXT: fcvtzs v0.4s, v0.4s -; CHECK-SD-NEXT: xtn v1.4h, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s -; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v1.8b +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h +; CHECK-SD-NEXT: xtn v0.8b, v0.8h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v8f32_v8i8: @@ -3072,12 +2996,8 @@ define <16 x i8> @fptos_v16f32_v16i8(<16 x float> %a) { ; CHECK-SD-NEXT: fcvtzs v2.4s, v2.4s ; CHECK-SD-NEXT: fcvtzs v1.4s, v1.4s ; CHECK-SD-NEXT: fcvtzs v0.4s, v0.4s -; CHECK-SD-NEXT: xtn v3.4h, v3.4s -; CHECK-SD-NEXT: xtn v2.4h, v2.4s -; CHECK-SD-NEXT: xtn v1.4h, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s -; CHECK-SD-NEXT: mov v2.d[1], v3.d[0] -; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] +; CHECK-SD-NEXT: uzp1 v2.8h, v2.8h, v3.8h +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h ; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: ret ; @@ -3134,20 +3054,12 @@ define <32 x i8> @fptos_v32f32_v32i8(<32 x float> %a) { ; CHECK-SD-NEXT: fcvtzs v6.4s, v6.4s ; CHECK-SD-NEXT: fcvtzs v5.4s, v5.4s ; CHECK-SD-NEXT: fcvtzs v4.4s, v4.4s -; CHECK-SD-NEXT: xtn v3.4h, v3.4s -; CHECK-SD-NEXT: xtn v2.4h, v2.4s -; CHECK-SD-NEXT: xtn v1.4h, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s -; CHECK-SD-NEXT: xtn v7.4h, v7.4s -; CHECK-SD-NEXT: xtn v6.4h, v6.4s -; CHECK-SD-NEXT: xtn v5.4h, v5.4s -; CHECK-SD-NEXT: xtn v4.4h, v4.4s -; CHECK-SD-NEXT: mov v2.d[1], v3.d[0] -; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] -; CHECK-SD-NEXT: mov v6.d[1], v7.d[0] -; CHECK-SD-NEXT: mov v4.d[1], v5.d[0] +; CHECK-SD-NEXT: uzp1 v2.8h, v2.8h, v3.8h +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h +; CHECK-SD-NEXT: uzp1 v1.8h, v6.8h, v7.8h +; CHECK-SD-NEXT: uzp1 v3.8h, v4.8h, v5.8h ; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v2.16b -; CHECK-SD-NEXT: uzp1 v1.16b, v4.16b, v6.16b +; CHECK-SD-NEXT: uzp1 v1.16b, v3.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v32f32_v32i8: diff --git a/llvm/test/CodeGen/AArch64/neon-truncstore.ll b/llvm/test/CodeGen/AArch64/neon-truncstore.ll index b677d077b98c..5d78ad24eb33 100644 --- a/llvm/test/CodeGen/AArch64/neon-truncstore.ll +++ b/llvm/test/CodeGen/AArch64/neon-truncstore.ll @@ -104,7 +104,7 @@ define void @v4i32_v4i8(<4 x i32> %a, ptr %result) { ; CHECK-LABEL: v4i32_v4i8: ; CHECK: // %bb.0: ; CHECK-NEXT: xtn v0.4h, v0.4s -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: str s0, [x0] ; CHECK-NEXT: ret %b = trunc <4 x i32> %a to <4 x i8> @@ -170,8 +170,7 @@ define void @v2i16_v2i8(<2 x i16> %a, ptr %result) { define void @v4i16_v4i8(<4 x i16> %a, ptr %result) { ; CHECK-LABEL: v4i16_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: str s0, [x0] ; CHECK-NEXT: ret %b = trunc <4 x i16> %a to <4 x i8> diff --git a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll index 5f905d94e357..6f1ae023bf25 100644 --- a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll @@ -145,7 +145,7 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-NEXT: shl v0.4h, v0.4h, #8 ; CHECK-NEXT: sqadd v0.4h, v0.4h, v1.4h ; CHECK-NEXT: sshr v0.4h, v0.4h, #8 -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: str s0, [x2] ; CHECK-NEXT: ret %x = load <4 x i8>, ptr %px diff --git a/llvm/test/CodeGen/AArch64/shuffle-tbl34.ll b/llvm/test/CodeGen/AArch64/shuffle-tbl34.ll index 0ef64789ad97..fb571eff39fe 100644 --- a/llvm/test/CodeGen/AArch64/shuffle-tbl34.ll +++ b/llvm/test/CodeGen/AArch64/shuffle-tbl34.ll @@ -353,13 +353,17 @@ define <8 x i8> @shuffle4_v8i8_v8i8(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x define <8 x i16> @shuffle4_v4i8_zext(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) { ; CHECK-LABEL: shuffle4_v4i8_zext: ; CHECK: // %bb.0: -; CHECK-NEXT: uzp1 v0.8b, v0.8b, v1.8b -; CHECK-NEXT: uzp1 v1.8b, v2.8b, v3.8b +; CHECK-NEXT: fmov d5, d2 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3 ; CHECK-NEXT: adrp x8, .LCPI8_0 -; CHECK-NEXT: ushll v2.8h, v0.8b, #0 +; CHECK-NEXT: fmov d4, d0 ; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI8_0] -; CHECK-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b +; CHECK-NEXT: mov v4.d[1], v1.d[0] +; CHECK-NEXT: mov v5.d[1], v3.d[0] +; CHECK-NEXT: bic v4.8h, #255, lsl #8 +; CHECK-NEXT: bic v5.8h, #255, lsl #8 +; CHECK-NEXT: tbl v0.16b, { v4.16b, v5.16b }, v0.16b ; CHECK-NEXT: ret %x = shufflevector <4 x i8> %a, <4 x i8> %b, <8 x i32> %y = shufflevector <4 x i8> %c, <4 x i8> %d, <8 x i32> diff --git a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll index acec3e74d3e9..d1f843a09f74 100644 --- a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll @@ -146,7 +146,7 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-NEXT: shl v0.4h, v0.4h, #8 ; CHECK-NEXT: sqsub v0.4h, v0.4h, v1.4h ; CHECK-NEXT: sshr v0.4h, v0.4h, #8 -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: str s0, [x2] ; CHECK-NEXT: ret %x = load <4 x i8>, ptr %px diff --git a/llvm/test/CodeGen/AArch64/tbl-loops.ll b/llvm/test/CodeGen/AArch64/tbl-loops.ll index 4f8a4f7aede3..0ad990086551 100644 --- a/llvm/test/CodeGen/AArch64/tbl-loops.ll +++ b/llvm/test/CodeGen/AArch64/tbl-loops.ll @@ -41,8 +41,8 @@ define void @loop1(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n ; CHECK-NEXT: fcvtzs v2.4s, v2.4s ; CHECK-NEXT: xtn v1.4h, v1.4s ; CHECK-NEXT: xtn v2.4h, v2.4s -; CHECK-NEXT: xtn v1.8b, v1.8h -; CHECK-NEXT: xtn v2.8b, v2.8h +; CHECK-NEXT: uzp1 v1.8b, v1.8b, v0.8b +; CHECK-NEXT: uzp1 v2.8b, v2.8b, v0.8b ; CHECK-NEXT: mov v1.s[1], v2.s[0] ; CHECK-NEXT: stur d1, [x12, #-4] ; CHECK-NEXT: add x12, x12, #8 diff --git a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll index ba367b0dbfde..18cd4cc2111a 100644 --- a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll +++ b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll @@ -710,23 +710,23 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) { ; CHECK-NEXT: LBB6_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldp q4, q1, [x0, #48] -; CHECK-NEXT: add x9, x1, #8 -; CHECK-NEXT: ldp q3, q2, [x0] -; CHECK-NEXT: subs x8, x8, #1 +; CHECK-NEXT: add x9, x1, #10 ; CHECK-NEXT: ldr d0, [x0, #80] +; CHECK-NEXT: ldp q3, q2, [x0] ; CHECK-NEXT: ldr q5, [x0, #32] +; CHECK-NEXT: subs x8, x8, #1 ; CHECK-NEXT: add x0, x0, #128 -; CHECK-NEXT: uzp1.4s v4, v5, v4 -; CHECK-NEXT: uzp1.4s v2, v3, v2 ; CHECK-NEXT: uzp1.4s v0, v1, v0 -; CHECK-NEXT: uzp1.8h v1, v2, v4 +; CHECK-NEXT: uzp1.4s v1, v5, v4 +; CHECK-NEXT: uzp1.4s v2, v3, v2 ; CHECK-NEXT: xtn.4h v0, v0 -; CHECK-NEXT: uzp1.16b v1, v1, v0 -; CHECK-NEXT: xtn.8b v0, v0 -; CHECK-NEXT: st1.h { v1 }[4], [x9] -; CHECK-NEXT: add x9, x1, #10 -; CHECK-NEXT: st1.b { v0 }[2], [x9] -; CHECK-NEXT: str d1, [x1], #16 +; CHECK-NEXT: uzp1.8h v1, v2, v1 +; CHECK-NEXT: uzp1.8b v2, v0, v0 +; CHECK-NEXT: uzp1.16b v0, v1, v0 +; CHECK-NEXT: st1.b { v2 }[2], [x9] +; CHECK-NEXT: add x9, x1, #8 +; CHECK-NEXT: st1.h { v0 }[4], [x9] +; CHECK-NEXT: str d0, [x1], #16 ; CHECK-NEXT: b.eq LBB6_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret @@ -755,7 +755,7 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) { ; CHECK-BE-NEXT: xtn v0.4h, v0.4s ; CHECK-BE-NEXT: uzp1 v1.8h, v1.8h, v2.8h ; CHECK-BE-NEXT: uzp1 v1.16b, v1.16b, v0.16b -; CHECK-BE-NEXT: xtn v0.8b, v0.8h +; CHECK-BE-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-BE-NEXT: rev16 v2.16b, v1.16b ; CHECK-BE-NEXT: rev64 v1.16b, v1.16b ; CHECK-BE-NEXT: st1 { v0.b }[2], [x9] @@ -790,7 +790,7 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) { ; CHECK-DISABLE-NEXT: xtn v0.4h, v0.4s ; CHECK-DISABLE-NEXT: uzp1 v1.8h, v1.8h, v2.8h ; CHECK-DISABLE-NEXT: uzp1 v1.16b, v1.16b, v0.16b -; CHECK-DISABLE-NEXT: xtn v0.8b, v0.8h +; CHECK-DISABLE-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-DISABLE-NEXT: rev16 v2.16b, v1.16b ; CHECK-DISABLE-NEXT: rev64 v1.16b, v1.16b ; CHECK-DISABLE-NEXT: st1 { v0.b }[2], [x9] diff --git a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll index e05c65daf50a..f0bbed59405e 100644 --- a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll @@ -142,7 +142,7 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-NEXT: movi d0, #0xff00ff00ff00ff ; CHECK-NEXT: uaddl v1.8h, v1.8b, v2.8b ; CHECK-NEXT: umin v0.4h, v1.4h, v0.4h -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: str s0, [x2] ; CHECK-NEXT: ret %x = load <4 x i8>, ptr %px diff --git a/llvm/test/CodeGen/AArch64/usub_sat_vec.ll b/llvm/test/CodeGen/AArch64/usub_sat_vec.ll index 05f43e7d8427..82c0327219f5 100644 --- a/llvm/test/CodeGen/AArch64/usub_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/usub_sat_vec.ll @@ -143,7 +143,7 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 ; CHECK-NEXT: uqsub v0.4h, v0.4h, v1.4h -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: str s0, [x2] ; CHECK-NEXT: ret %x = load <4 x i8>, ptr %px diff --git a/llvm/test/CodeGen/AArch64/vcvt-oversize.ll b/llvm/test/CodeGen/AArch64/vcvt-oversize.ll index 380bdbcc7f74..611940546bc1 100644 --- a/llvm/test/CodeGen/AArch64/vcvt-oversize.ll +++ b/llvm/test/CodeGen/AArch64/vcvt-oversize.ll @@ -9,9 +9,8 @@ define <8 x i8> @float_to_i8(ptr %in) { ; CHECK-NEXT: fadd v0.4s, v0.4s, v0.4s ; CHECK-NEXT: fcvtzs v0.4s, v0.4s ; CHECK-NEXT: fcvtzs v1.4s, v1.4s -; CHECK-NEXT: xtn v0.4h, v0.4s -; CHECK-NEXT: xtn v1.4h, v1.4s -; CHECK-NEXT: uzp1 v0.8b, v1.8b, v0.8b +; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h +; CHECK-NEXT: xtn v0.8b, v0.8h ; CHECK-NEXT: ret %l = load <8 x float>, ptr %in %scale = fmul <8 x float> %l, diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll index 9c6ab8da0fa7..dd7a9c6d7768 100644 --- a/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll +++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll @@ -210,7 +210,7 @@ define void @no_combine_for_non_bool_truncate(<4 x i32> %vec, ptr %out) { ; CHECK-LABEL: no_combine_for_non_bool_truncate: ; CHECK: ; %bb.0: ; CHECK-NEXT: xtn.4h v0, v0 -; CHECK-NEXT: xtn.8b v0, v0 +; CHECK-NEXT: uzp1.8b v0, v0, v0 ; CHECK-NEXT: str s0, [x0] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll index 90328f73f86b..71d55df66517 100644 --- a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll +++ b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll @@ -410,7 +410,7 @@ define void @store_trunc_from_64bits(ptr %src, ptr %dst) { ; BE-NEXT: ldrh w8, [x0, #4] ; BE-NEXT: rev32 v0.4h, v0.4h ; BE-NEXT: mov v0.h[2], w8 -; BE-NEXT: xtn v0.8b, v0.8h +; BE-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; BE-NEXT: rev32 v0.16b, v0.16b ; BE-NEXT: str s0, [sp, #12] ; BE-NEXT: ldrh w9, [sp, #12] @@ -456,7 +456,7 @@ define void @store_trunc_add_from_64bits(ptr %src, ptr %dst) { ; BE-NEXT: add x8, x8, :lo12:.LCPI11_0 ; BE-NEXT: ld1 { v1.4h }, [x8] ; BE-NEXT: add v0.4h, v0.4h, v1.4h -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -638,7 +638,7 @@ define void @shift_trunc_store(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -672,7 +672,7 @@ define void @shift_trunc_store_default_align(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -706,7 +706,7 @@ define void @shift_trunc_store_align_4(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -741,7 +741,7 @@ define void @shift_trunc_store_const_offset_1(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -777,7 +777,7 @@ define void @shift_trunc_store_const_offset_3(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -801,7 +801,7 @@ define void @shift_trunc_volatile_store(ptr %src, ptr %dst) { ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: shrn.4h v0, v0, #16 -; CHECK-NEXT: xtn.8b v1, v0 +; CHECK-NEXT: uzp1.8b v1, v0, v0 ; CHECK-NEXT: umov.h w8, v0[2] ; CHECK-NEXT: str s1, [sp, #12] ; CHECK-NEXT: ldrh w9, [sp, #12] @@ -816,7 +816,7 @@ define void @shift_trunc_volatile_store(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -868,7 +868,7 @@ define void @load_v3i8_zext_to_3xi32_add_trunc_store(ptr %src) { ; BE-NEXT: ushll v0.8h, v0.8b, #0 ; BE-NEXT: ld1 { v0.b }[4], [x9] ; BE-NEXT: add v0.4h, v0.4h, v1.4h -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #8] @@ -921,7 +921,7 @@ define void @load_v3i8_sext_to_3xi32_add_trunc_store(ptr %src) { ; BE-NEXT: ushll v0.8h, v0.8b, #0 ; BE-NEXT: ld1 { v0.b }[4], [x9] ; BE-NEXT: add v0.4h, v0.4h, v1.4h -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #8] -- GitLab From f1015d1701d86c4e640cdbfd1928a958aea921d6 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 13 Mar 2024 16:08:01 +0000 Subject: [PATCH 0146/1407] [VPlan] Use VPBuilder to create ActiveLaneMask (NFC). --- .../Vectorize/LoopVectorizationPlanner.h | 16 ++++++++++++---- .../lib/Transforms/Vectorize/VPlanTransforms.cpp | 8 ++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index a7ebf78e54ce..e86705e89889 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -79,6 +79,13 @@ public: VPBasicBlock *getInsertBlock() const { return BB; } VPBasicBlock::iterator getInsertPoint() const { return InsertPt; } + /// Create a VPBuilder to insert after \p R. + static VPBuilder getToInsertAfter(VPRecipeBase *R) { + VPBuilder B; + B.setInsertPoint(R->getParent(), std::next(R->getIterator())); + return B; + } + /// InsertPoint - A saved insertion point. class VPInsertPoint { VPBasicBlock *Block = nullptr; @@ -131,8 +138,9 @@ public: /// Create an N-ary operation with \p Opcode, \p Operands and set \p Inst as /// its underlying Instruction. - VPValue *createNaryOp(unsigned Opcode, ArrayRef Operands, - Instruction *Inst = nullptr, const Twine &Name = "") { + VPInstruction *createNaryOp(unsigned Opcode, ArrayRef Operands, + Instruction *Inst = nullptr, + const Twine &Name = "") { DebugLoc DL; if (Inst) DL = Inst->getDebugLoc(); @@ -140,8 +148,8 @@ public: NewVPInst->setUnderlyingValue(Inst); return NewVPInst; } - VPValue *createNaryOp(unsigned Opcode, ArrayRef Operands, - DebugLoc DL, const Twine &Name = "") { + VPInstruction *createNaryOp(unsigned Opcode, ArrayRef Operands, + DebugLoc DL, const Twine &Name = "") { return createInstruction(Opcode, Operands, DL, Name); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index f6b564ad931c..3b19db9f0d30 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1192,10 +1192,10 @@ void VPlanTransforms::addActiveLaneMask( LaneMask = addVPLaneMaskPhiAndUpdateExitBranch( Plan, DataAndControlFlowWithoutRuntimeCheck); } else { - LaneMask = new VPInstruction(VPInstruction::ActiveLaneMask, - {WideCanonicalIV, Plan.getTripCount()}, - nullptr, "active.lane.mask"); - LaneMask->insertAfter(WideCanonicalIV); + VPBuilder B = VPBuilder::getToInsertAfter(WideCanonicalIV); + LaneMask = B.createNaryOp(VPInstruction::ActiveLaneMask, + {WideCanonicalIV, Plan.getTripCount()}, nullptr, + "active.lane.mask"); } // Walk users of WideCanonicalIV and replace all compares of the form -- GitLab From 8a8ef1cacfcd7745d2b6ad00431e6fa9ab9a2fb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Wed, 13 Mar 2024 09:14:40 -0700 Subject: [PATCH 0147/1407] [flang][cuda] Enable cuda with -x cuda option (#84944) Flang driver was already able to enable the CUDA language feature base on the file extension but there was no command line option. This PR adds one. --- flang/lib/Frontend/CompilerInvocation.cpp | 9 +++++++++ flang/lib/Frontend/FrontendAction.cpp | 11 ++++++++--- flang/test/Driver/cuda-option.f90 | 15 +++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 flang/test/Driver/cuda-option.f90 diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp index 4707de0e976c..2e3fa1f6e660 100644 --- a/flang/lib/Frontend/CompilerInvocation.cpp +++ b/flang/lib/Frontend/CompilerInvocation.cpp @@ -581,6 +581,8 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args, // pre-processed inputs. .Case("f95", Language::Fortran) .Case("f95-cpp-input", Language::Fortran) + // CUDA Fortran + .Case("cuda", Language::Fortran) .Default(Language::Unknown); // Flang's intermediate representations. @@ -877,6 +879,13 @@ static bool parseDialectArgs(CompilerInvocation &res, llvm::opt::ArgList &args, if (args.hasArg(clang::driver::options::OPT_flarge_sizes)) res.getDefaultKinds().set_sizeIntegerKind(8); + // -x cuda + auto language = args.getLastArgValue(clang::driver::options::OPT_x); + if (language.equals("cuda")) { + res.getFrontendOpts().features.Enable( + Fortran::common::LanguageFeature::CUDA); + } + // -fopenmp and -fopenacc if (args.hasArg(clang::driver::options::OPT_fopenacc)) { res.getFrontendOpts().features.Enable( diff --git a/flang/lib/Frontend/FrontendAction.cpp b/flang/lib/Frontend/FrontendAction.cpp index 599b4e11f0cf..bb1c239540d9 100644 --- a/flang/lib/Frontend/FrontendAction.cpp +++ b/flang/lib/Frontend/FrontendAction.cpp @@ -86,9 +86,14 @@ bool FrontendAction::beginSourceFile(CompilerInstance &ci, invoc.collectMacroDefinitions(); } - // Enable CUDA Fortran if source file is *.cuf/*.CUF. - invoc.getFortranOpts().features.Enable(Fortran::common::LanguageFeature::CUDA, - getCurrentInput().getIsCUDAFortran()); + if (!invoc.getFortranOpts().features.IsEnabled( + Fortran::common::LanguageFeature::CUDA)) { + // Enable CUDA Fortran if source file is *.cuf/*.CUF and not already + // enabled. + invoc.getFortranOpts().features.Enable( + Fortran::common::LanguageFeature::CUDA, + getCurrentInput().getIsCUDAFortran()); + } // Decide between fixed and free form (if the user didn't express any // preference, use the file extension to decide) diff --git a/flang/test/Driver/cuda-option.f90 b/flang/test/Driver/cuda-option.f90 new file mode 100644 index 000000000000..112e1cb6c77f --- /dev/null +++ b/flang/test/Driver/cuda-option.f90 @@ -0,0 +1,15 @@ +! Test -fcuda option +! RUN: %flang -fc1 -cpp -x cuda -fdebug-unparse %s -o - | FileCheck %s +! RUN: not %flang -fc1 -cpp %s -o - 2>&1 | FileCheck %s --check-prefix=ERROR +program main +#if _CUDA + integer :: var = _CUDA +#endif + integer, device :: dvar +end program + +! CHECK-LABEL: PROGRAM main +! CHECK: INTEGER :: var = 1 +! CHECK: INTEGER, DEVICE :: dvar + +! ERROR: cuda-option.f90:8:19: error: expected end of statement -- GitLab From 5facb406e6417987ac5dfabd8f04510d7bc3fbc6 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Wed, 13 Mar 2024 09:26:36 -0700 Subject: [PATCH 0148/1407] [libc][docs] document gpu support for stdbit.h (#85103) Via: https://github.com/llvm/llvm-project/pull/84938#issuecomment-1992120095 --------- Co-authored-by: Joseph Huber --- libc/docs/gpu/support.rst | 73 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/libc/docs/gpu/support.rst b/libc/docs/gpu/support.rst index 250f0a7794de..31bbdc41090b 100644 --- a/libc/docs/gpu/support.rst +++ b/libc/docs/gpu/support.rst @@ -89,6 +89,79 @@ strtok_r |check| strxfrm |check| ============= ========= ============ +stdbit.h +-------- + +============================ ========= ============ +Function Name Available RPC Required +============================ ========= ============ +stdc_leading_zeros_uc |check| +stdc_leading_zeros_us |check| +stdc_leading_zeros_ui |check| +stdc_leading_zeros_ul |check| +stdc_leading_zeros_ull |check| +stdc_trailing_zeros_uc |check| +stdc_trailing_zeros_us |check| +stdc_trailing_zeros_ui |check| +stdc_trailing_zeros_ul |check| +stdc_trailing_zeros_ull |check| +stdc_trailing_ones_uc |check| +stdc_trailing_ones_us |check| +stdc_trailing_ones_ui |check| +stdc_trailing_ones_ul |check| +stdc_trailing_ones_ull |check| +stdc_first_leading_zero_uc |check| +stdc_first_leading_zero_us |check| +stdc_first_leading_zero_ui |check| +stdc_first_leading_zero_ul |check| +stdc_first_leading_zero_ull |check| +stdc_first_leading_one_uc |check| +stdc_first_leading_one_us |check| +stdc_first_leading_one_ui |check| +stdc_first_leading_one_ul |check| +stdc_first_leading_one_ull |check| +stdc_first_trailing_zero_uc |check| +stdc_first_trailing_zero_us |check| +stdc_first_trailing_zero_ui |check| +stdc_first_trailing_zero_ul |check| +stdc_first_trailing_zero_ull |check| +stdc_first_trailing_one_uc |check| +stdc_first_trailing_one_us |check| +stdc_first_trailing_one_ui |check| +stdc_first_trailing_one_ul |check| +stdc_first_trailing_one_ull |check| +stdc_count_zeros_uc |check| +stdc_count_zeros_us |check| +stdc_count_zeros_ui |check| +stdc_count_zeros_ul |check| +stdc_count_zeros_ull |check| +stdc_count_ones_uc |check| +stdc_count_ones_us |check| +stdc_count_ones_ui |check| +stdc_count_ones_ul |check| +stdc_count_ones_ull |check| +stdc_has_single_bit_uc |check| +stdc_has_single_bit_us |check| +stdc_has_single_bit_ui |check| +stdc_has_single_bit_ul |check| +stdc_has_single_bit_ull |check| +stdc_bit_width_uc |check| +stdc_bit_width_us |check| +stdc_bit_width_ui |check| +stdc_bit_width_ul |check| +stdc_bit_width_ull |check| +stdc_bit_floor_uc |check| +stdc_bit_floor_us |check| +stdc_bit_floor_ui |check| +stdc_bit_floor_ul |check| +stdc_bit_floor_ull |check| +stdc_bit_ceil_uc |check| +stdc_bit_ceil_us |check| +stdc_bit_ceil_ui |check| +stdc_bit_ceil_ul |check| +stdc_bit_ceil_ull |check| +============================ ========= ============ + stdlib.h -------- -- GitLab From bb893fa23f6c851d957d82e14bc1aa6fbbffcaaa Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Wed, 13 Mar 2024 17:26:50 +0100 Subject: [PATCH 0149/1407] [mlir] Fix inlining-threshold.mlir test for NDEBUG builds. --- mlir/test/Transforms/inlining-threshold.mlir | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlir/test/Transforms/inlining-threshold.mlir b/mlir/test/Transforms/inlining-threshold.mlir index b94115d8f264..649408aab577 100644 --- a/mlir/test/Transforms/inlining-threshold.mlir +++ b/mlir/test/Transforms/inlining-threshold.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s --mlir-disable-threading -inline='default-pipeline='' inlining-threshold=100' -debug-only=inliner-pass 2>&1 | FileCheck %s +// RUN: mlir-opt %s -inline='default-pipeline= inlining-threshold=100' | FileCheck %s // Check that inlining does not happen when the threshold is exceeded. func.func @callee1(%arg : i32) -> i32 { -- GitLab From ccd16085f70105d457f052543d731dd51089945b Mon Sep 17 00:00:00 2001 From: Bhuminjay Soni Date: Wed, 13 Mar 2024 21:58:25 +0530 Subject: [PATCH 0150/1407] Diagnose misuse of the cleanup attribute (#80040) This pull request fixes #79443 when the cleanup attribute is intended to be applied to a variable declaration, passing its address to a specified function. The problem arises when standard functions like free, closedir, fclose, etc., are used incorrectly with this attribute, leading to incorrect behavior. Fixes #79443 --- clang/docs/ReleaseNotes.rst | 4 ++++ clang/include/clang/Sema/Sema.h | 5 +++-- clang/lib/Sema/SemaDeclAttr.cpp | 24 ++++++++++++++++++++++++ clang/test/Sema/attr-cleanup.c | 28 ++++++++++++++++++++++++++-- 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 64a9fe0d8bcc..7173c1400f53 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -237,6 +237,10 @@ Improvements to Clang's diagnostics - Clang now diagnoses lambda function expressions being implicitly cast to boolean values, under ``-Wpointer-bool-conversion``. Fixes #GH82512. +- Clang now provides improved warnings for the ``cleanup`` attribute to detect misuse scenarios, + such as attempting to call ``free`` on an unallocated object. Fixes + `#79443 `_. + Improvements to Clang's time-trace ---------------------------------- diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index d6ab2b0c2def..4a853119a2bb 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -2152,14 +2152,15 @@ public: bool IsLayoutCompatible(QualType T1, QualType T2) const; + bool CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall, + const FunctionProtoType *Proto); + private: void CheckArrayAccess(const Expr *BaseExpr, const Expr *IndexExpr, const ArraySubscriptExpr *ASE = nullptr, bool AllowOnePastEnd = true, bool IndexNegated = false); void CheckArrayAccess(const Expr *E); - bool CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall, - const FunctionProtoType *Proto); bool CheckObjCMethodCall(ObjCMethodDecl *Method, SourceLocation loc, ArrayRef Args); bool CheckPointerCall(NamedDecl *NDecl, CallExpr *TheCall, diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index e3da3e606435..ec00fdf3f88d 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -3787,6 +3787,30 @@ static void handleCleanupAttr(Sema &S, Decl *D, const ParsedAttr &AL) { << NI.getName() << ParamTy << Ty; return; } + VarDecl *VD = cast(D); + // Create a reference to the variable declaration. This is a fake/dummy + // reference. + DeclRefExpr *VariableReference = DeclRefExpr::Create( + S.Context, NestedNameSpecifierLoc{}, FD->getLocation(), VD, false, + DeclarationNameInfo{VD->getDeclName(), VD->getLocation()}, VD->getType(), + VK_LValue); + + // Create a unary operator expression that represents taking the address of + // the variable. This is a fake/dummy expression. + Expr *AddressOfVariable = UnaryOperator::Create( + S.Context, VariableReference, UnaryOperatorKind::UO_AddrOf, + S.Context.getPointerType(VD->getType()), VK_PRValue, OK_Ordinary, Loc, + +false, FPOptionsOverride{}); + + // Create a function call expression. This is a fake/dummy call expression. + CallExpr *FunctionCallExpression = + CallExpr::Create(S.Context, E, ArrayRef{AddressOfVariable}, + S.Context.VoidTy, VK_PRValue, Loc, FPOptionsOverride{}); + + if (S.CheckFunctionCall(FD, FunctionCallExpression, + FD->getType()->getAs())) { + return; + } D->addAttr(::new (S.Context) CleanupAttr(S.Context, AL, FD)); } diff --git a/clang/test/Sema/attr-cleanup.c b/clang/test/Sema/attr-cleanup.c index 2c38687622c2..95baf2e675a0 100644 --- a/clang/test/Sema/attr-cleanup.c +++ b/clang/test/Sema/attr-cleanup.c @@ -1,7 +1,7 @@ -// RUN: %clang_cc1 %s -verify -fsyntax-only +// RUN: %clang_cc1 -Wfree-nonheap-object -fsyntax-only -verify %s void c1(int *a); - +typedef __typeof__(sizeof(0)) size_t; extern int g1 __attribute((cleanup(c1))); // expected-warning {{'cleanup' attribute only applies to local variables}} int g2 __attribute((cleanup(c1))); // expected-warning {{'cleanup' attribute only applies to local variables}} static int g3 __attribute((cleanup(c1))); // expected-warning {{'cleanup' attribute only applies to local variables}} @@ -48,3 +48,27 @@ void t6(void) { } void t7(__attribute__((cleanup(c4))) int a) {} // expected-warning {{'cleanup' attribute only applies to local variables}} + +extern void free(void *); +extern void *malloc(size_t size); +void t8(void) { + void *p + __attribute__(( + cleanup( + free // expected-warning{{attempt to call free on non-heap object 'p'}} + ) + )) + = malloc(10); +} +typedef __attribute__((aligned(2))) int Aligned2Int; +void t9(void){ + Aligned2Int __attribute__((cleanup(c1))) xwarn; // expected-warning{{passing 2-byte aligned argument to 4-byte aligned parameter 1 of 'c1' may result in an unaligned pointer access}} +} + +__attribute__((enforce_tcb("TCB1"))) void func1(int *x) { + *x = 5; +} +__attribute__((enforce_tcb("TCB2"))) void t10() { + int __attribute__((cleanup(func1))) x = 5; // expected-warning{{calling 'func1' is a violation of trusted computing base 'TCB2'}} +} + -- GitLab From 69afb9d7875d79fdacaaa2f22b5ee3a06faf5373 Mon Sep 17 00:00:00 2001 From: Sirraide Date: Wed, 13 Mar 2024 17:39:23 +0100 Subject: [PATCH 0151/1407] [Clang] [Sema] Fix bug in `_Complex float`+`int` arithmetic (#83063) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C23 6.3.1.8 ‘Usual arithmetic conversions’ p1 states (emphasis mine): > Otherwise, if the corresponding real type of either operand is `float`, the other operand is converted, *without change of type domain*, to a type whose corresponding real type is `float`. ‘type domain’ here refers to `_Complex` vs real (i.e. non-`_Complex`); there is another clause that states the same for `double`. Consider the following code: ```c++ _Complex float f; int x; f / x; ``` After talking this over with @AaronBallman, we came to the conclusion that `x` should be converted to `float` and *not* `_Complex float` (that is, we should perform a division of `_Complex float / float`, and *not* `_Complex float / _Complex float`; the same also applies to `-+*`). This was already being done correctly for cases where `x` was already a `float`; it’s just mixed `_Complex float`+`int` operations that currently suffer from this problem. This pr removes the extra `FloatingRealToComplex` conversion that we were erroneously inserting and adds some tests to make sure we’re actually doing `_Complex float / float` and not `_Complex float / _Complex float` (and analogously for `double` and `-+*`). The only exception here is `float / _Complex float`, which calls a library function (`__divsc3`) that takes 4 `float`s, so we end up having to convert the `float` to a `_Complex float` after all (and analogously for `double`); I don’t believe there is a way around this. Lastly, we were also missing tests for `_Complex` arithmetic at compile time, so this adds some tests for that as well. --- clang/docs/ReleaseNotes.rst | 7 ++ clang/lib/Sema/SemaExpr.cpp | 15 ++- clang/test/CodeGen/complex-math-mixed.c | 146 ++++++++++++++++++++++++ clang/test/CodeGen/volatile.cpp | 48 ++++---- clang/test/Sema/complex-arithmetic.c | 115 +++++++++++++++++++ 5 files changed, 298 insertions(+), 33 deletions(-) create mode 100644 clang/test/CodeGen/complex-math-mixed.c create mode 100644 clang/test/Sema/complex-arithmetic.c diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 7173c1400f53..c5488e8742f6 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -278,6 +278,13 @@ Bug Fixes in This Version - Clang now correctly generates overloads for bit-precise integer types for builtin operators in C++. Fixes #GH82998. +- When performing mixed arithmetic between ``_Complex`` floating-point types and integers, + Clang now correctly promotes the integer to its corresponding real floating-point + type only rather than to the complex type (e.g. ``_Complex float / int`` is now evaluated + as ``_Complex float / float`` rather than ``_Complex float / _Complex float``), as mandated + by the C standard. This significantly improves codegen of `*` and `/` especially. + Fixes (`#31205 `_). + Bug Fixes to Compiler Builtins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp index 93f82e68ab64..8725b09f8546 100644 --- a/clang/lib/Sema/SemaExpr.cpp +++ b/clang/lib/Sema/SemaExpr.cpp @@ -1099,12 +1099,13 @@ ExprResult Sema::DefaultVariadicArgumentPromotion(Expr *E, VariadicCallType CT, return E; } -/// Converts an integer to complex float type. Helper function of +/// Convert complex integers to complex floats and real integers to +/// real floats as required for complex arithmetic. Helper function of /// UsualArithmeticConversions() /// /// \return false if the integer expression is an integer type and is -/// successfully converted to the complex type. -static bool handleIntegerToComplexFloatConversion(Sema &S, ExprResult &IntExpr, +/// successfully converted to the (complex) float type. +static bool handleComplexIntegerToFloatConversion(Sema &S, ExprResult &IntExpr, ExprResult &ComplexExpr, QualType IntTy, QualType ComplexTy, @@ -1114,8 +1115,6 @@ static bool handleIntegerToComplexFloatConversion(Sema &S, ExprResult &IntExpr, if (IntTy->isIntegerType()) { QualType fpTy = ComplexTy->castAs()->getElementType(); IntExpr = S.ImpCastExprToType(IntExpr.get(), fpTy, CK_IntegralToFloating); - IntExpr = S.ImpCastExprToType(IntExpr.get(), ComplexTy, - CK_FloatingRealToComplex); } else { assert(IntTy->isComplexIntegerType()); IntExpr = S.ImpCastExprToType(IntExpr.get(), ComplexTy, @@ -1160,11 +1159,11 @@ static QualType handleComplexFloatConversion(Sema &S, ExprResult &Shorter, static QualType handleComplexConversion(Sema &S, ExprResult &LHS, ExprResult &RHS, QualType LHSType, QualType RHSType, bool IsCompAssign) { - // if we have an integer operand, the result is the complex type. - if (!handleIntegerToComplexFloatConversion(S, RHS, LHS, RHSType, LHSType, + // Handle (complex) integer types. + if (!handleComplexIntegerToFloatConversion(S, RHS, LHS, RHSType, LHSType, /*SkipCast=*/false)) return LHSType; - if (!handleIntegerToComplexFloatConversion(S, LHS, RHS, LHSType, RHSType, + if (!handleComplexIntegerToFloatConversion(S, LHS, RHS, LHSType, RHSType, /*SkipCast=*/IsCompAssign)) return RHSType; diff --git a/clang/test/CodeGen/complex-math-mixed.c b/clang/test/CodeGen/complex-math-mixed.c new file mode 100644 index 000000000000..050163cca80a --- /dev/null +++ b/clang/test/CodeGen/complex-math-mixed.c @@ -0,0 +1,146 @@ +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown -o - | FileCheck %s --check-prefix=X86 +// RUN: %clang_cc1 %s -O0 -triple x86_64-unknown-unknown -fsyntax-only -ast-dump | FileCheck %s --check-prefix=AST + +// Check that for 'F _Complex + int' (F = real floating-point type), we emit an +// implicit cast from 'int' to 'F', but NOT to 'F _Complex' (i.e. that we do +// 'F _Complex + F', NOT 'F _Complex + F _Complex'), and likewise for -/*. + +// AST-NOT: FloatingRealToComplex + +float _Complex add_float_ci(float _Complex a, int b) { + // X86-LABEL: @add_float_ci + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to float + // X86: fadd float {{.*}}, [[I]] + // X86-NOT: fadd + return a + b; +} + +float _Complex add_float_ic(int a, float _Complex b) { + // X86-LABEL: @add_float_ic + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to float + // X86: fadd float [[I]] + // X86-NOT: fadd + return a + b; +} + +float _Complex sub_float_ci(float _Complex a, int b) { + // X86-LABEL: @sub_float_ci + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to float + // X86: fsub float {{.*}}, [[I]] + // X86-NOT: fsub + return a - b; +} + +float _Complex sub_float_ic(int a, float _Complex b) { + // X86-LABEL: @sub_float_ic + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to float + // X86: fsub float [[I]] + // X86: fneg + // X86-NOT: fsub + return a - b; +} + +float _Complex mul_float_ci(float _Complex a, int b) { + // X86-LABEL: @mul_float_ci + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to float + // X86: fmul float {{.*}}, [[I]] + // X86: fmul float {{.*}}, [[I]] + // X86-NOT: fmul + return a * b; +} + +float _Complex mul_float_ic(int a, float _Complex b) { + // X86-LABEL: @mul_float_ic + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to float + // X86: fmul float [[I]] + // X86: fmul float [[I]] + // X86-NOT: fmul + return a * b; +} + +float _Complex div_float_ci(float _Complex a, int b) { + // X86-LABEL: @div_float_ci + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to float + // X86: fdiv float {{.*}}, [[I]] + // X86: fdiv float {{.*}}, [[I]] + // X86-NOT: @__divsc3 + return a / b; +} + +// There is no good way of doing this w/o converting the 'int' to a complex +// number, so we expect complex division here. +float _Complex div_float_ic(int a, float _Complex b) { + // X86-LABEL: @div_float_ic + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to float + // X86: call {{.*}} @__divsc3(float {{.*}} [[I]], float noundef 0.{{0+}}e+00, float {{.*}}, float {{.*}}) + return a / b; +} + +double _Complex add_double_ci(double _Complex a, int b) { + // X86-LABEL: @add_double_ci + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to double + // X86: fadd double {{.*}}, [[I]] + // X86-NOT: fadd + return a + b; +} + +double _Complex add_double_ic(int a, double _Complex b) { + // X86-LABEL: @add_double_ic + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to double + // X86: fadd double [[I]] + // X86-NOT: fadd + return a + b; +} + +double _Complex sub_double_ci(double _Complex a, int b) { + // X86-LABEL: @sub_double_ci + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to double + // X86: fsub double {{.*}}, [[I]] + // X86-NOT: fsub + return a - b; +} + +double _Complex sub_double_ic(int a, double _Complex b) { + // X86-LABEL: @sub_double_ic + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to double + // X86: fsub double [[I]] + // X86: fneg + // X86-NOT: fsub + return a - b; +} + +double _Complex mul_double_ci(double _Complex a, int b) { + // X86-LABEL: @mul_double_ci + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to double + // X86: fmul double {{.*}}, [[I]] + // X86: fmul double {{.*}}, [[I]] + // X86-NOT: fmul + return a * b; +} + +double _Complex mul_double_ic(int a, double _Complex b) { + // X86-LABEL: @mul_double_ic + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to double + // X86: fmul double [[I]] + // X86: fmul double [[I]] + // X86-NOT: fmul + return a * b; +} + +double _Complex div_double_ci(double _Complex a, int b) { + // X86-LABEL: @div_double_ci + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to double + // X86: fdiv double {{.*}}, [[I]] + // X86: fdiv double {{.*}}, [[I]] + // X86-NOT: @__divdc3 + return a / b; +} + +// There is no good way of doing this w/o converting the 'int' to a complex +// number, so we expect complex division here. +double _Complex div_double_ic(int a, double _Complex b) { + // X86-LABEL: @div_double_ic + // X86: [[I:%.*]] = sitofp i32 {{%.*}} to double + // X86: call {{.*}} @__divdc3(double {{.*}} [[I]], double noundef 0.{{0+}}e+00, double {{.*}}, double {{.*}}) + return a / b; +} diff --git a/clang/test/CodeGen/volatile.cpp b/clang/test/CodeGen/volatile.cpp index 38724659ad8a..70f523b93852 100644 --- a/clang/test/CodeGen/volatile.cpp +++ b/clang/test/CodeGen/volatile.cpp @@ -1,4 +1,4 @@ -// RUN: %clang_cc1 -O2 -triple=x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK +// RUN: %clang_cc1 -O2 -triple=x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s struct agg { int a ; @@ -10,34 +10,32 @@ _Complex float cf; int volatile vol =10; void f0() { const_cast(cf) = const_cast(cf) + 1; -// CHECK: %cf.real = load volatile float, ptr @cf -// CHECK: %cf.imag = load volatile float, ptr getelementptr -// CHECK: %add.r = fadd float %cf.real, 1.000000e+00 -// CHECK: %add.i = fadd float %cf.imag, 0.000000e+00 -// CHECK: store volatile float %add.r -// CHECK: store volatile float %add.i, ptr getelementptr +// CHECK: [[Re1:%.*]] = load volatile float, ptr @cf +// CHECK: [[Im1:%.*]] = load volatile float, ptr getelementptr +// CHECK: [[Add1:%.*]] = fadd float [[Re1]], 1.000000e+00 +// CHECK: store volatile float [[Add1]], ptr @cf +// CHECK: store volatile float [[Im1]], ptr getelementptr static_cast(cf) = static_cast(cf) + 1; -// CHECK: %cf.real1 = load volatile float, ptr @cf -// CHECK: %cf.imag2 = load volatile float, ptr getelementptr -// CHECK: %add.r3 = fadd float %cf.real1, 1.000000e+00 -// CHECK: %add.i4 = fadd float %cf.imag2, 0.000000e+00 -// CHECK: store volatile float %add.r3, ptr @cf -// CHECK: store volatile float %add.i4, ptr getelementptr +// CHECK: [[Re2:%.*]] = load volatile float, ptr @cf +// CHECK: [[Im2:%.*]] = load volatile float, ptr getelementptr +// CHECK: [[Add2:%.*]] = fadd float [[Re2]], 1.000000e+00 +// CHECK: store volatile float [[Add2]], ptr @cf +// CHECK: store volatile float [[Im2]], ptr getelementptr const_cast(a.a) = const_cast(t.a) ; -// CHECK: %0 = load volatile i32, ptr @t -// CHECK: store volatile i32 %0, ptr @a +// CHECK: [[I1:%.*]] = load volatile i32, ptr @t +// CHECK: store volatile i32 [[I1]], ptr @a static_cast(a.b) = static_cast(t.a) ; -// CHECK: %1 = load volatile i32, ptr @t -// CHECK: store volatile i32 %1, ptr getelementptr +// CHECK: [[I2:%.*]] = load volatile i32, ptr @t +// CHECK: store volatile i32 [[I2]], ptr getelementptr const_cast(vt) = const_cast(vt) + 1; -// CHECK: %2 = load volatile i32, ptr @vt -// CHECK: %add = add nsw i32 %2, 1 -// CHECK: store volatile i32 %add, ptr @vt +// CHECK: [[I3:%.*]] = load volatile i32, ptr @vt +// CHECK: [[Add3:%.*]] = add nsw i32 [[I3]], 1 +// CHECK: store volatile i32 [[Add3]], ptr @vt static_cast(vt) = static_cast(vt) + 1; -// CHECK: %3 = load volatile i32, ptr @vt -// CHECK: %add5 = add nsw i32 %3, 1 -// CHECK: store volatile i32 %add5, ptr @vt +// CHECK: [[I4:%.*]] = load volatile i32, ptr @vt +// CHECK: [[Add4:%.*]] = add nsw i32 [[I4]], 1 +// CHECK: store volatile i32 [[Add4]], ptr @vt vt = const_cast(vol); -// %4 = load i32, ptr @vol -// store i32 %4, ptr @vt +// [[I5:%.*]] = load i32, ptr @vol +// store i32 [[I5]], ptr @vt } diff --git a/clang/test/Sema/complex-arithmetic.c b/clang/test/Sema/complex-arithmetic.c new file mode 100644 index 000000000000..c9e84da6daa9 --- /dev/null +++ b/clang/test/Sema/complex-arithmetic.c @@ -0,0 +1,115 @@ +// RUN: %clang_cc1 -verify %s +// expected-no-diagnostics + +// This tests evaluation of _Complex arithmetic at compile time. + +#define APPROX_EQ(a, b) ( \ + __builtin_fabs(__real (a) - __real (b)) < 0.0001 && \ + __builtin_fabs(__imag (a) - __imag (b)) < 0.0001 \ +) + +#define EVAL(a, b) _Static_assert(a == b, "") +#define EVALF(a, b) _Static_assert(APPROX_EQ(a, b), "") + +// _Complex float + _Complex float +void a() { + EVALF((2.f + 3i) + (4.f + 5i), 6.f + 8i); + EVALF((2.f + 3i) - (4.f + 5i), -2.f - 2i); + EVALF((2.f + 3i) * (4.f + 5i), -7.f + 22i); + EVALF((2.f + 3i) / (4.f + 5i), 0.5609f + 0.0487i); + + EVALF((2. + 3i) + (4. + 5i), 6. + 8i); + EVALF((2. + 3i) - (4. + 5i), -2. - 2i); + EVALF((2. + 3i) * (4. + 5i), -7. + 22i); + EVALF((2. + 3i) / (4. + 5i), .5609 + .0487i); +} + +// _Complex int + _Complex int +void b() { + EVAL((2 + 3i) + (4 + 5i), 6 + 8i); + EVAL((2 + 3i) - (4 + 5i), -2 - 2i); + EVAL((2 + 3i) * (4 + 5i), -7 + 22i); + EVAL((8 + 30i) / (4 + 5i), 4 + 1i); +} + +// _Complex float + float +void c() { + EVALF((2.f + 4i) + 3.f, 5.f + 4i); + EVALF((2.f + 4i) - 3.f, -1.f + 4i); + EVALF((2.f + 4i) * 3.f, 6.f + 12i); + EVALF((2.f + 4i) / 2.f, 1.f + 2i); + + EVALF(3.f + (2.f + 4i), 5.f + 4i); + EVALF(3.f - (2.f + 4i), 1.f - 4i); + EVALF(3.f * (2.f + 4i), 6.f + 12i); + EVALF(3.f / (2.f + 4i), .3f - 0.6i); + + EVALF((2. + 4i) + 3., 5. + 4i); + EVALF((2. + 4i) - 3., -1. + 4i); + EVALF((2. + 4i) * 3., 6. + 12i); + EVALF((2. + 4i) / 2., 1. + 2i); + + EVALF(3. + (2. + 4i), 5. + 4i); + EVALF(3. - (2. + 4i), 1. - 4i); + EVALF(3. * (2. + 4i), 6. + 12i); + EVALF(3. / (2. + 4i), .3 - 0.6i); +} + +// _Complex int + int +void d() { + EVAL((2 + 4i) + 3, 5 + 4i); + EVAL((2 + 4i) - 3, -1 + 4i); + EVAL((2 + 4i) * 3, 6 + 12i); + EVAL((2 + 4i) / 2, 1 + 2i); + + EVAL(3 + (2 + 4i), 5 + 4i); + EVAL(3 - (2 + 4i), 1 - 4i); + EVAL(3 * (2 + 4i), 6 + 12i); + EVAL(20 / (2 + 4i), 2 - 4i); +} + +// _Complex float + int +void e() { + EVALF((2.f + 4i) + 3, 5.f + 4i); + EVALF((2.f + 4i) - 3, -1.f + 4i); + EVALF((2.f + 4i) * 3, 6.f + 12i); + EVALF((2.f + 4i) / 2, 1.f + 2i); + + EVALF(3 + (2.f + 4i), 5.f + 4i); + EVALF(3 - (2.f + 4i), 1.f - 4i); + EVALF(3 * (2.f + 4i), 6.f + 12i); + EVALF(3 / (2.f + 4i), .3f - 0.6i); + + EVALF((2. + 4i) + 3, 5. + 4i); + EVALF((2. + 4i) - 3, -1. + 4i); + EVALF((2. + 4i) * 3, 6. + 12i); + EVALF((2. + 4i) / 2, 1. + 2i); + + EVALF(3 + (2. + 4i), 5. + 4i); + EVALF(3 - (2. + 4i), 1. - 4i); + EVALF(3 * (2. + 4i), 6. + 12i); + EVALF(3 / (2. + 4i), .3 - 0.6i); +} + +// _Complex int + float +void f() { + EVALF((2 + 4i) + 3.f, 5.f + 4i); + EVALF((2 + 4i) - 3.f, -1.f + 4i); + EVALF((2 + 4i) * 3.f, 6.f + 12i); + EVALF((2 + 4i) / 2.f, 1.f + 2i); + + EVALF(3.f + (2 + 4i), 5.f + 4i); + EVALF(3.f - (2 + 4i), 1.f - 4i); + EVALF(3.f * (2 + 4i), 6.f + 12i); + EVALF(3.f / (2 + 4i), .3f - 0.6i); + + EVALF((2 + 4i) + 3., 5. + 4i); + EVALF((2 + 4i) - 3., -1. + 4i); + EVALF((2 + 4i) * 3., 6. + 12i); + EVALF((2 + 4i) / 2., 1. + 2i); + + EVALF(3. + (2 + 4i), 5. + 4i); + EVALF(3. - (2 + 4i), 1. - 4i); + EVALF(3. * (2 + 4i), 6. + 12i); + EVALF(3. / (2 + 4i), .3 - 0.6i); +} -- GitLab From 360da83858655ad8297f3c0467c8c97ebedab5ed Mon Sep 17 00:00:00 2001 From: Stephen Tozer Date: Wed, 13 Mar 2024 16:39:35 +0000 Subject: [PATCH 0152/1407] [RemoveDI][NFC] Rename DPValue->DbgRecord in comments and varnames (#84939) This patch continues the ongoing rename work, replacing DPValue with DbgRecord in comments and the names of variables, both members and fn-local. This is the most labour-intensive part of the rename, as it is where the most decisions have to be made about whether a given comment or variable is referring to DPValues (equivalent to debug variable intrinsics) or DbgRecords (a catch-all for all debug intrinsics); these decisions are not individually difficult, but comprise a fairly large amount of text to review. This patch still largely performs basic string substitutions followed by clang-format; there are almost* no places where, for example, a comment has been expanded or modified to reflect the semantic difference between DPValues and DbgRecords. I don't believe such a change is generally necessary in LLVM, but it may be useful in the docs, and so I'll be submitting docs changes as a separate patch. *In a few places, `dbg.values` was replaced with `debug intrinsics`. --- .../llvm/CodeGen/GlobalISel/IRTranslator.h | 2 +- llvm/include/llvm/IR/BasicBlock.h | 32 ++-- .../include/llvm/IR/DebugProgramInstruction.h | 62 +++--- llvm/include/llvm/IR/Instruction.h | 31 +-- llvm/include/llvm/IR/PassManager.h | 2 +- llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp | 6 +- .../CodeGen/AssignmentTrackingAnalysis.cpp | 35 ++-- llvm/lib/CodeGen/CodeGenPrepare.cpp | 8 +- llvm/lib/CodeGen/MIRPrinter.cpp | 4 +- llvm/lib/CodeGen/SelectOptimize.cpp | 15 +- llvm/lib/IR/AsmWriter.cpp | 2 +- llvm/lib/IR/BasicBlock.cpp | 176 +++++++++--------- llvm/lib/IR/DebugInfo.cpp | 2 +- llvm/lib/IR/DebugProgramInstruction.cpp | 70 +++---- llvm/lib/IR/Instruction.cpp | 37 ++-- llvm/lib/IR/LLVMContextImpl.cpp | 2 +- llvm/lib/IR/LLVMContextImpl.h | 12 +- llvm/lib/Transforms/Utils/Local.cpp | 2 +- .../Transforms/Utils/LoopRotationUtils.cpp | 28 +-- llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 9 +- llvm/lib/Transforms/Utils/ValueMapper.cpp | 8 +- llvm/unittests/IR/BasicBlockDbgInfoTest.cpp | 76 ++++---- llvm/unittests/IR/DebugInfoTest.cpp | 26 +-- .../Transforms/Utils/DebugifyTest.cpp | 2 +- 24 files changed, 330 insertions(+), 319 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h index bfac54a65c5b..29f675b2203b 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -205,7 +205,7 @@ private: bool translate(const Constant &C, Register Reg); /// Examine any debug-info attached to the instruction (in the form of - /// DPValues) and translate it. + /// DbgRecords) and translate it. void translateDbgInfo(const Instruction &Inst, MachineIRBuilder &MIRBuilder); diff --git a/llvm/include/llvm/IR/BasicBlock.h b/llvm/include/llvm/IR/BasicBlock.h index 5bac113c9b7b..71c1a8394896 100644 --- a/llvm/include/llvm/IR/BasicBlock.h +++ b/llvm/include/llvm/IR/BasicBlock.h @@ -78,13 +78,13 @@ public: DPMarker *createMarker(InstListType::iterator It); /// Convert variable location debugging information stored in dbg.value - /// intrinsics into DPMarker / DPValue records. Deletes all dbg.values in + /// intrinsics into DPMarkers / DbgRecords. Deletes all dbg.values in /// the process and sets IsNewDbgInfoFormat = true. Only takes effect if /// the UseNewDbgInfoFormat LLVM command line option is given. void convertToNewDbgValues(); /// Convert variable location debugging information stored in DPMarkers and - /// DPValues into the dbg.value intrinsic representation. Sets + /// DbgRecords into the dbg.value intrinsic representation. Sets /// IsNewDbgInfoFormat = false. void convertFromNewDbgValues(); @@ -93,50 +93,50 @@ public: /// if necessary. void setIsNewDbgInfoFormat(bool NewFlag); - /// Record that the collection of DPValues in \p M "trails" after the last + /// Record that the collection of DbgRecords in \p M "trails" after the last /// instruction of this block. These are equivalent to dbg.value intrinsics /// that exist at the end of a basic block with no terminator (a transient /// state that occurs regularly). void setTrailingDbgRecords(DPMarker *M); - /// Fetch the collection of DPValues that "trail" after the last instruction + /// Fetch the collection of DbgRecords that "trail" after the last instruction /// of this block, see \ref setTrailingDbgRecords. If there are none, returns /// nullptr. DPMarker *getTrailingDbgRecords(); - /// Delete any trailing DPValues at the end of this block, see + /// Delete any trailing DbgRecords at the end of this block, see /// \ref setTrailingDbgRecords. void deleteTrailingDbgRecords(); void dumpDbgValues() const; - /// Return the DPMarker for the position given by \p It, so that DPValues can - /// be inserted there. This will either be nullptr if not present, a DPMarker, - /// or TrailingDPValues if It is end(). + /// Return the DPMarker for the position given by \p It, so that DbgRecords + /// can be inserted there. This will either be nullptr if not present, a + /// DPMarker, or TrailingDbgRecords if It is end(). DPMarker *getMarker(InstListType::iterator It); /// Return the DPMarker for the position that comes after \p I. \see /// BasicBlock::getMarker, this can be nullptr, a DPMarker, or - /// TrailingDPValues if there is no next instruction. + /// TrailingDbgRecords if there is no next instruction. DPMarker *getNextMarker(Instruction *I); - /// Insert a DPValue into a block at the position given by \p I. + /// Insert a DbgRecord into a block at the position given by \p I. void insertDbgRecordAfter(DbgRecord *DPV, Instruction *I); - /// Insert a DPValue into a block at the position given by \p Here. + /// Insert a DbgRecord into a block at the position given by \p Here. void insertDbgRecordBefore(DbgRecord *DPV, InstListType::iterator Here); - /// Eject any debug-info trailing at the end of a block. DPValues can + /// Eject any debug-info trailing at the end of a block. DbgRecords can /// transiently be located "off the end" of a block if the blocks terminator /// is temporarily removed. Once a terminator is re-inserted this method will - /// move such DPValues back to the right place (ahead of the terminator). - void flushTerminatorDbgValues(); + /// move such DbgRecords back to the right place (ahead of the terminator). + void flushTerminatorDbgRecords(); /// In rare circumstances instructions can be speculatively removed from /// blocks, and then be re-inserted back into that position later. When this /// happens in RemoveDIs debug-info mode, some special patching-up needs to /// occur: inserting into the middle of a sequence of dbg.value intrinsics - /// does not have an equivalent with DPValues. + /// does not have an equivalent with DbgRecords. void reinsertInstInDbgRecords(Instruction *I, std::optional Pos); @@ -522,7 +522,7 @@ private: BasicBlock::iterator FromEndIt); /// Perform any debug-info specific maintenence for the given splice - /// activity. In the DPValue debug-info representation, debug-info is not + /// activity. In the DbgRecord debug-info representation, debug-info is not /// in instructions, and so it does not automatically move from one block /// to another. void spliceDebugInfo(BasicBlock::iterator ToIt, BasicBlock *FromBB, diff --git a/llvm/include/llvm/IR/DebugProgramInstruction.h b/llvm/include/llvm/IR/DebugProgramInstruction.h index 507b652feeb0..1afc9259241d 100644 --- a/llvm/include/llvm/IR/DebugProgramInstruction.h +++ b/llvm/include/llvm/IR/DebugProgramInstruction.h @@ -1,4 +1,4 @@ -//===-- llvm/DebugProgramInstruction.h - Stream of debug info -------*- C++ -*-===// +//===-- llvm/DebugProgramInstruction.h - Stream of debug info ---*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -15,10 +15,10 @@ // %bar = void call @ext(%foo); // // and all information is stored in the Value / Metadata hierachy defined -// elsewhere in LLVM. In the "DPValue" design, each instruction /may/ have a -// connection with a DPMarker, which identifies a position immediately before the -// instruction, and each DPMarker /may/ then have connections to DPValues which -// record the variable assignment information. To illustrate: +// elsewhere in LLVM. In the "DbgRecord" design, each instruction /may/ have a +// connection with a DPMarker, which identifies a position immediately before +// the instruction, and each DPMarker /may/ then have connections to DbgRecords +// which record the variable assignment information. To illustrate: // // %foo = add i32 1, %0 // ; foo->DbgMarker == nullptr @@ -26,7 +26,7 @@ // ;; the instruction for %foo, therefore it has no DbgMarker. // %bar = void call @ext(%foo) // ; bar->DbgMarker = { -// ; StoredDPValues = { +// ; StoredDbgRecords = { // ; DPValue(metadata i32 %foo, ...) // ; } // ; } @@ -119,7 +119,7 @@ public: /// Base class for non-instruction debug metadata records that have positions /// within IR. Features various methods copied across from the Instruction /// class to aid ease-of-use. DbgRecords should always be linked into a -/// DPMarker's StoredDPValues list. The marker connects a DbgRecord back to +/// DPMarker's StoredDbgRecords list. The marker connects a DbgRecord back to /// it's position in the BasicBlock. /// /// We need a discriminator for dyn/isa casts. In order to avoid paying for a @@ -557,8 +557,8 @@ public: /// intrinsics. There is a one-to-one relationship between each debug /// intrinsic in a block and each DbgRecord once the representation has been /// converted, and the ordering is meaningful in the same way. - simple_ilist StoredDPValues; - bool empty() const { return StoredDPValues.empty(); } + simple_ilist StoredDbgRecords; + bool empty() const { return StoredDbgRecords.empty(); } const BasicBlock *getParent() const; BasicBlock *getParent(); @@ -576,54 +576,56 @@ public: void print(raw_ostream &O, bool IsForDebug = false) const; void print(raw_ostream &ROS, ModuleSlotTracker &MST, bool IsForDebug) const; - /// Produce a range over all the DPValues in this Marker. + /// Produce a range over all the DbgRecords in this Marker. iterator_range::iterator> getDbgRecordRange(); iterator_range::const_iterator> getDbgRecordRange() const; - /// Transfer any DPValues from \p Src into this DPMarker. If \p InsertAtHead - /// is true, place them before existing DPValues, otherwise afterwards. + /// Transfer any DbgRecords from \p Src into this DPMarker. If \p InsertAtHead + /// is true, place them before existing DbgRecords, otherwise afterwards. void absorbDebugValues(DPMarker &Src, bool InsertAtHead); - /// Transfer the DPValues in \p Range from \p Src into this DPMarker. If - /// \p InsertAtHead is true, place them before existing DPValues, otherwise + /// Transfer the DbgRecords in \p Range from \p Src into this DPMarker. If + /// \p InsertAtHead is true, place them before existing DbgRecords, otherwise // afterwards. void absorbDebugValues(iterator_range Range, DPMarker &Src, bool InsertAtHead); - /// Insert a DPValue into this DPMarker, at the end of the list. If + /// Insert a DbgRecord into this DPMarker, at the end of the list. If /// \p InsertAtHead is true, at the start. void insertDbgRecord(DbgRecord *New, bool InsertAtHead); - /// Insert a DPValue prior to a DPValue contained within this marker. + /// Insert a DbgRecord prior to a DbgRecord contained within this marker. void insertDbgRecord(DbgRecord *New, DbgRecord *InsertBefore); - /// Insert a DPValue after a DPValue contained within this marker. + /// Insert a DbgRecord after a DbgRecord contained within this marker. void insertDbgRecordAfter(DbgRecord *New, DbgRecord *InsertAfter); /// Clone all DPMarkers from \p From into this marker. There are numerous /// options to customise the source/destination, due to gnarliness, see class /// comment. - /// \p FromHere If non-null, copy from FromHere to the end of From's DPValues - /// \p InsertAtHead Place the cloned DPValues at the start of StoredDPValues - /// \returns Range over all the newly cloned DPValues + /// \p FromHere If non-null, copy from FromHere to the end of From's + /// DbgRecords + /// \p InsertAtHead Place the cloned DbgRecords at the start of + /// StoredDbgRecords + /// \returns Range over all the newly cloned DbgRecords iterator_range::iterator> cloneDebugInfoFrom(DPMarker *From, std::optional::iterator> FromHere, bool InsertAtHead = false); - /// Erase all DPValues in this DPMarker. + /// Erase all DbgRecords in this DPMarker. void dropDbgRecords(); /// Erase a single DbgRecord from this marker. In an ideal future, we would /// never erase an assignment in this way, but it's the equivalent to /// erasing a debug intrinsic from a block. void dropOneDbgRecord(DbgRecord *DR); - /// We generally act like all llvm Instructions have a range of DPValues + /// We generally act like all llvm Instructions have a range of DbgRecords /// attached to them, but in reality sometimes we don't allocate the DPMarker - /// to save time and memory, but still have to return ranges of DPValues. When - /// we need to describe such an unallocated DPValue range, use this static - /// markers range instead. This will bite us if someone tries to insert a - /// DPValue in that range, but they should be using the Official (TM) API for - /// that. + /// to save time and memory, but still have to return ranges of DbgRecords. + /// When we need to describe such an unallocated DbgRecord range, use this + /// static markers range instead. This will bite us if someone tries to insert + /// a DbgRecord in that range, but they should be using the Official (TM) API + /// for that. static DPMarker EmptyDPMarker; static iterator_range::iterator> getEmptyDbgRecordRange() { - return make_range(EmptyDPMarker.StoredDPValues.end(), - EmptyDPMarker.StoredDPValues.end()); + return make_range(EmptyDPMarker.StoredDbgRecords.end(), + EmptyDPMarker.StoredDbgRecords.end()); } }; @@ -632,7 +634,7 @@ inline raw_ostream &operator<<(raw_ostream &OS, const DPMarker &Marker) { return OS; } -/// Inline helper to return a range of DPValues attached to a marker. It needs +/// Inline helper to return a range of DbgRecords attached to a marker. It needs /// to be inlined as it's frequently called, but also come after the declaration /// of DPMarker. Thus: it's pre-declared by users like Instruction, then an /// inlineable body defined here. diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h index 817abd6afbca..d6cf15577523 100644 --- a/llvm/include/llvm/IR/Instruction.h +++ b/llvm/include/llvm/IR/Instruction.h @@ -64,47 +64,48 @@ public: /// Clone any debug-info attached to \p From onto this instruction. Used to /// copy debugging information from one block to another, when copying entire - /// blocks. \see DebugProgramInstruction.h , because the ordering of DPValues - /// is still important, fine grain control of which instructions are moved and - /// where they go is necessary. + /// blocks. \see DebugProgramInstruction.h , because the ordering of + /// DbgRecords is still important, fine grain control of which instructions + /// are moved and where they go is necessary. /// \p From The instruction to clone debug-info from. - /// \p from_here Optional iterator to limit DPValues cloned to be a range from + /// \p from_here Optional iterator to limit DbgRecords cloned to be a range + /// from /// from_here to end(). - /// \p InsertAtHead Whether the cloned DPValues should be placed at the end - /// or the beginning of existing DPValues attached to this. - /// \returns A range over the newly cloned DPValues. + /// \p InsertAtHead Whether the cloned DbgRecords should be placed at the end + /// or the beginning of existing DbgRecords attached to this. + /// \returns A range over the newly cloned DbgRecords. iterator_range::iterator> cloneDebugInfoFrom( const Instruction *From, std::optional::iterator> FromHere = std::nullopt, bool InsertAtHead = false); - /// Return a range over the DPValues attached to this instruction. + /// Return a range over the DbgRecords attached to this instruction. iterator_range::iterator> getDbgRecordRange() const { return llvm::getDbgRecordRange(DbgMarker); } - /// Return an iterator to the position of the "Next" DPValue after this + /// Return an iterator to the position of the "Next" DbgRecord after this /// instruction, or std::nullopt. This is the position to pass to /// BasicBlock::reinsertInstInDbgRecords when re-inserting an instruction. std::optional::iterator> getDbgReinsertionPosition(); - /// Returns true if any DPValues are attached to this instruction. + /// Returns true if any DbgRecords are attached to this instruction. bool hasDbgRecords() const; - /// Transfer any DPValues on the position \p It onto this instruction, - /// by simply adopting the sequence of DPValues (which is efficient) if + /// Transfer any DbgRecords on the position \p It onto this instruction, + /// by simply adopting the sequence of DbgRecords (which is efficient) if /// possible, by merging two sequences otherwise. void adoptDbgRecords(BasicBlock *BB, InstListType::iterator It, bool InsertAtHead); - /// Erase any DPValues attached to this instruction. + /// Erase any DbgRecords attached to this instruction. void dropDbgRecords(); - /// Erase a single DPValue \p I that is attached to this instruction. + /// Erase a single DbgRecord \p I that is attached to this instruction. void dropOneDbgRecord(DbgRecord *I); /// Handle the debug-info implications of this instruction being removed. Any - /// attached DPValues need to "fall" down onto the next instruction. + /// attached DbgRecords need to "fall" down onto the next instruction. void handleMarkerRemoval(); protected: diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h index c03d49c3b7b9..ec8b809d40bf 100644 --- a/llvm/include/llvm/IR/PassManager.h +++ b/llvm/include/llvm/IR/PassManager.h @@ -227,7 +227,7 @@ public: detail::getAnalysisResult( AM, IR, std::tuple(ExtraArgs...)); - // RemoveDIs: if requested, convert debug-info to DPValue representation + // RemoveDIs: if requested, convert debug-info to DbgRecord representation // for duration of these passes. bool ShouldConvertDbgInfo = shouldConvertDbgInfo(IR); if (ShouldConvertDbgInfo) diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp index 93fb2a821dee..0eb9c246f2a9 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp @@ -19,7 +19,7 @@ using namespace llvm; PreservedAnalyses BitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) { - // RemoveDIs: there's no bitcode representation of the DPValue debug-info, + // RemoveDIs: there's no bitcode representation of the DbgRecord debug-info, // convert to dbg.values before writing out. bool IsNewDbgInfoFormat = M.IsNewDbgInfoFormat; if (IsNewDbgInfoFormat) @@ -56,8 +56,8 @@ namespace { StringRef getPassName() const override { return "Bitcode Writer"; } bool runOnModule(Module &M) override { - // RemoveDIs: there's no bitcode representation of the DPValue debug-info, - // convert to dbg.values before writing out. + // RemoveDIs: there's no bitcode representation of the DbgRecord + // debug-info, convert to dbg.values before writing out. bool IsNewDbgInfoFormat = M.IsNewDbgInfoFormat; if (IsNewDbgInfoFormat) M.convertFromNewDbgValues(); diff --git a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp index a4b819a735c6..746926e56f2e 100644 --- a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp +++ b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp @@ -217,13 +217,14 @@ void FunctionVarLocs::init(FunctionVarLocsBuilder &Builder) { // to the start and end position in the vector with VarLocsBeforeInst. This // block includes VarLocs for any DPValues attached to that instruction. for (auto &P : Builder.VarLocsBeforeInst) { - // Process VarLocs attached to a DPValue alongside their marker Instruction. + // Process VarLocs attached to a DbgRecord alongside their marker + // Instruction. if (isa(P.first)) continue; const Instruction *I = cast(P.first); unsigned BlockStart = VarLocRecords.size(); - // Any VarLocInfos attached to a DPValue should now be remapped to their - // marker Instruction, in order of DPValue appearance and prior to any + // Any VarLocInfos attached to a DbgRecord should now be remapped to their + // marker Instruction, in order of DbgRecord appearance and prior to any // VarLocInfos attached directly to that instruction. for (const DPValue &DPV : DPValue::filter(I->getDbgRecordRange())) { // Even though DPV defines a variable location, VarLocsBeforeInst can @@ -1649,7 +1650,7 @@ void AssignmentTrackingLowering::processUntaggedInstruction( Ops.push_back(dwarf::DW_OP_deref); DIE = DIExpression::prependOpcodes(DIE, Ops, /*StackValue=*/false, /*EntryValue=*/false); - // Find a suitable insert point, before the next instruction or DPValue + // Find a suitable insert point, before the next instruction or DbgRecord // after I. auto InsertBefore = getNextNode(&I); assert(InsertBefore && "Shouldn't be inserting after a terminator"); @@ -1886,21 +1887,21 @@ void AssignmentTrackingLowering::resetInsertionPoint(DPValue &After) { } void AssignmentTrackingLowering::process(BasicBlock &BB, BlockInfo *LiveSet) { - // If the block starts with DPValues, we need to process those DPValues as + // If the block starts with DbgRecords, we need to process those DbgRecords as // their own frame without processing any instructions first. - bool ProcessedLeadingDPValues = !BB.begin()->hasDbgRecords(); + bool ProcessedLeadingDbgRecords = !BB.begin()->hasDbgRecords(); for (auto II = BB.begin(), EI = BB.end(); II != EI;) { assert(VarsTouchedThisFrame.empty()); // Process the instructions in "frames". A "frame" includes a single // non-debug instruction followed any debug instructions before the // next non-debug instruction. - // Skip the current instruction if it has unprocessed DPValues attached (see - // comment above `ProcessedLeadingDPValues`). - if (ProcessedLeadingDPValues) { + // Skip the current instruction if it has unprocessed DbgRecords attached + // (see comment above `ProcessedLeadingDbgRecords`). + if (ProcessedLeadingDbgRecords) { // II is now either a debug intrinsic, a non-debug instruction with no - // attached DPValues, or a non-debug instruction with attached processed - // DPValues. + // attached DbgRecords, or a non-debug instruction with attached processed + // DbgRecords. // II has not been processed. if (!isa(&*II)) { if (II->isTerminator()) @@ -1912,8 +1913,8 @@ void AssignmentTrackingLowering::process(BasicBlock &BB, BlockInfo *LiveSet) { } } // II is now either a debug intrinsic, a non-debug instruction with no - // attached DPValues, or a non-debug instruction with attached unprocessed - // DPValues. + // attached DbgRecords, or a non-debug instruction with attached unprocessed + // DbgRecords. if (II != EI && II->hasDbgRecords()) { // Skip over non-variable debug records (i.e., labels). They're going to // be read from IR (possibly re-ordering them within the debug record @@ -1924,7 +1925,7 @@ void AssignmentTrackingLowering::process(BasicBlock &BB, BlockInfo *LiveSet) { assert(LiveSet->isValid()); } } - ProcessedLeadingDPValues = true; + ProcessedLeadingDbgRecords = true; while (II != EI) { auto *Dbg = dyn_cast(&*II); if (!Dbg) @@ -1934,9 +1935,9 @@ void AssignmentTrackingLowering::process(BasicBlock &BB, BlockInfo *LiveSet) { assert(LiveSet->isValid()); ++II; } - // II is now a non-debug instruction either with no attached DPValues, or - // with attached processed DPValues. II has not been processed, and all - // debug instructions or DPValues in the frame preceding II have been + // II is now a non-debug instruction either with no attached DbgRecords, or + // with attached processed DbgRecords. II has not been processed, and all + // debug instructions or DbgRecords in the frame preceding II have been // processed. // We've processed everything in the "frame". Now determine which variables diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index 59a0c64d3c9f..055e275e143d 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -2946,7 +2946,7 @@ class TypePromotionTransaction { Instruction *PrevInst; BasicBlock *BB; } Point; - std::optional BeforeDPValue = std::nullopt; + std::optional BeforeDbgRecord = std::nullopt; /// Remember whether or not the instruction had a previous instruction. bool HasPrevInstruction; @@ -2958,9 +2958,9 @@ class TypePromotionTransaction { BasicBlock *BB = Inst->getParent(); // Record where we would have to re-insert the instruction in the sequence - // of DPValues, if we ended up reinserting. + // of DbgRecords, if we ended up reinserting. if (BB->IsNewDbgInfoFormat) - BeforeDPValue = Inst->getDbgReinsertionPosition(); + BeforeDbgRecord = Inst->getDbgReinsertionPosition(); if (HasPrevInstruction) { Point.PrevInst = &*std::prev(Inst->getIterator()); @@ -2983,7 +2983,7 @@ class TypePromotionTransaction { Inst->insertBefore(*Point.BB, Position); } - Inst->getParent()->reinsertInstInDbgRecords(Inst, BeforeDPValue); + Inst->getParent()->reinsertInstInDbgRecords(Inst, BeforeDbgRecord); } }; diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp index 4ed44d1c06f4..8efe67a9a72b 100644 --- a/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/llvm/lib/CodeGen/MIRPrinter.cpp @@ -982,7 +982,7 @@ void MIRFormatter::printIRValue(raw_ostream &OS, const Value &V, } void llvm::printMIR(raw_ostream &OS, const Module &M) { - // RemoveDIs: as there's no textual form for DPValues yet, print debug-info + // RemoveDIs: as there's no textual form for DbgRecords yet, print debug-info // in dbg.value format. bool IsNewDbgInfoFormat = M.IsNewDbgInfoFormat; if (IsNewDbgInfoFormat) @@ -996,7 +996,7 @@ void llvm::printMIR(raw_ostream &OS, const Module &M) { } void llvm::printMIR(raw_ostream &OS, const MachineFunction &MF) { - // RemoveDIs: as there's no textual form for DPValues yet, print debug-info + // RemoveDIs: as there's no textual form for DbgRecords yet, print debug-info // in dbg.value format. bool IsNewDbgInfoFormat = MF.getFunction().IsNewDbgInfoFormat; if (IsNewDbgInfoFormat) diff --git a/llvm/lib/CodeGen/SelectOptimize.cpp b/llvm/lib/CodeGen/SelectOptimize.cpp index 40898d284a09..f65d5320bab9 100644 --- a/llvm/lib/CodeGen/SelectOptimize.cpp +++ b/llvm/lib/CodeGen/SelectOptimize.cpp @@ -645,12 +645,13 @@ void SelectOptimizeImpl::convertProfitableSIGroups(SelectGroups &ProfSIGroups) { DI->moveBeforePreserving(&*EndBlock->getFirstInsertionPt()); } - // Duplicate implementation for DPValues, the non-instruction debug-info - // record. Helper lambda for moving DPValues to the end block. - auto TransferDPValues = [&](Instruction &I) { - for (auto &DPValue : llvm::make_early_inc_range(I.getDbgRecordRange())) { - DPValue.removeFromParent(); - EndBlock->insertDbgRecordBefore(&DPValue, + // Duplicate implementation for DbgRecords, the non-instruction debug-info + // format. Helper lambda for moving DbgRecords to the end block. + auto TransferDbgRecords = [&](Instruction &I) { + for (auto &DbgRecord : + llvm::make_early_inc_range(I.getDbgRecordRange())) { + DbgRecord.removeFromParent(); + EndBlock->insertDbgRecordBefore(&DbgRecord, EndBlock->getFirstInsertionPt()); } }; @@ -660,7 +661,7 @@ void SelectOptimizeImpl::convertProfitableSIGroups(SelectGroups &ProfSIGroups) { // middle" of the select group. auto R = make_range(std::next(SI.getI()->getIterator()), std::next(LastSI.getI()->getIterator())); - llvm::for_each(R, TransferDPValues); + llvm::for_each(R, TransferDbgRecords); // These are the new basic blocks for the conditional branch. // At least one will become an actual new basic block. diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 1beb4c069a69..11383ea6214b 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -4592,7 +4592,7 @@ void AssemblyWriter::printInstruction(const Instruction &I) { void AssemblyWriter::printDPMarker(const DPMarker &Marker) { // There's no formal representation of a DPMarker -- print purely as a // debugging aid. - for (const DbgRecord &DPR : Marker.StoredDPValues) { + for (const DbgRecord &DPR : Marker.StoredDbgRecords) { printDbgRecord(DPR); Out << "\n"; } diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp index 7ead7ce3bf08..4dd1bdd6e2f4 100644 --- a/llvm/lib/IR/BasicBlock.cpp +++ b/llvm/lib/IR/BasicBlock.cpp @@ -63,9 +63,9 @@ DPMarker *BasicBlock::createMarker(InstListType::iterator It) { void BasicBlock::convertToNewDbgValues() { IsNewDbgInfoFormat = true; - // Iterate over all instructions in the instruction list, collecting dbg.value - // instructions and converting them to DPValues. Once we find a "real" - // instruction, attach all those DPValues to a DPMarker in that instruction. + // Iterate over all instructions in the instruction list, collecting debug + // info intrinsics and converting them to DbgRecords. Once we find a "real" + // instruction, attach all those DbgRecords to a DPMarker in that instruction. SmallVector DPVals; for (Instruction &I : make_early_inc_range(InstList)) { assert(!I.DbgMarker && "DbgMarker already set on old-format instrs?"); @@ -86,7 +86,7 @@ void BasicBlock::convertToNewDbgValues() { if (DPVals.empty()) continue; - // Create a marker to store DPValues in. + // Create a marker to store DbgRecords in. createMarker(&I); DPMarker *Marker = I.DbgMarker; @@ -102,7 +102,7 @@ void BasicBlock::convertFromNewDbgValues() { IsNewDbgInfoFormat = false; // Iterate over the block, finding instructions annotated with DPMarkers. - // Convert any attached DPValues to dbg.values and insert ahead of the + // Convert any attached DbgRecords to debug intrinsics and insert ahead of the // instruction. for (auto &Inst : *this) { if (!Inst.DbgMarker) @@ -116,7 +116,7 @@ void BasicBlock::convertFromNewDbgValues() { Marker.eraseFromParent(); } - // Assume no trailing DPValues: we could technically create them at the end + // Assume no trailing DbgRecords: we could technically create them at the end // of the block, after a terminator, but this would be non-cannonical and // indicates that something else is broken somewhere. assert(!getTrailingDbgRecords()); @@ -691,15 +691,15 @@ void BasicBlock::renumberInstructions() { NumInstrRenumberings++; } -void BasicBlock::flushTerminatorDbgValues() { - // If we erase the terminator in a block, any DPValues will sink and "fall +void BasicBlock::flushTerminatorDbgRecords() { + // If we erase the terminator in a block, any DbgRecords will sink and "fall // off the end", existing after any terminator that gets inserted. With // dbg.value intrinsics we would just insert the terminator at end() and - // the dbg.values would come before the terminator. With DPValues, we must + // the dbg.values would come before the terminator. With DbgRecords, we must // do this manually. // To get out of this unfortunate form, whenever we insert a terminator, - // check whether there's anything trailing at the end and move those DPValues - // in front of the terminator. + // check whether there's anything trailing at the end and move those + // DbgRecords in front of the terminator. // Do nothing if we're not in new debug-info format. if (!IsNewDbgInfoFormat) @@ -710,15 +710,15 @@ void BasicBlock::flushTerminatorDbgValues() { if (!Term) return; - // Are there any dangling DPValues? - DPMarker *TrailingDPValues = getTrailingDbgRecords(); - if (!TrailingDPValues) + // Are there any dangling DbgRecords? + DPMarker *TrailingDbgRecords = getTrailingDbgRecords(); + if (!TrailingDbgRecords) return; - // Transfer DPValues from the trailing position onto the terminator. + // Transfer DbgRecords from the trailing position onto the terminator. createMarker(Term); - Term->DbgMarker->absorbDebugValues(*TrailingDPValues, false); - TrailingDPValues->eraseFromParent(); + Term->DbgMarker->absorbDebugValues(*TrailingDbgRecords, false); + TrailingDbgRecords->eraseFromParent(); deleteTrailingDbgRecords(); } @@ -735,7 +735,7 @@ void BasicBlock::spliceDebugInfoEmptyBlock(BasicBlock::iterator Dest, // If an optimisation pass attempts to splice the contents of the block from // BB1->begin() to BB1->getTerminator(), then the dbg.value will be // transferred to the destination. - // However, in the "new" DPValue format for debug-info, that range is empty: + // However, in the "new" DbgRecord format for debug-info, that range is empty: // begin() returns an iterator to the terminator, as there will only be a // single instruction in the block. We must piece together from the bits set // in the iterators whether there was the intention to transfer any debug @@ -750,16 +750,16 @@ void BasicBlock::spliceDebugInfoEmptyBlock(BasicBlock::iterator Dest, bool ReadFromHead = First.getHeadBit(); // If the source block is completely empty, including no terminator, then - // transfer any trailing DPValues that are still hanging around. This can + // transfer any trailing DbgRecords that are still hanging around. This can // occur when a block is optimised away and the terminator has been moved // somewhere else. if (Src->empty()) { - DPMarker *SrcTrailingDPValues = Src->getTrailingDbgRecords(); - if (!SrcTrailingDPValues) + DPMarker *SrcTrailingDbgRecords = Src->getTrailingDbgRecords(); + if (!SrcTrailingDbgRecords) return; Dest->adoptDbgRecords(Src, Src->end(), InsertAtHead); - // adoptDbgRecords should have released the trailing DPValues. + // adoptDbgRecords should have released the trailing DbgRecords. assert(!Src->getTrailingDbgRecords()); return; } @@ -785,8 +785,8 @@ void BasicBlock::spliceDebugInfo(BasicBlock::iterator Dest, BasicBlock *Src, /* Do a quick normalisation before calling the real splice implementation. We might be operating on a degenerate basic block that has no instructions in it, a legitimate transient state. In that case, Dest will be end() and - any DPValues temporarily stored in the TrailingDPValues map in LLVMContext. - We might illustrate it thus: + any DbgRecords temporarily stored in the TrailingDbgRecords map in + LLVMContext. We might illustrate it thus: Dest | @@ -795,35 +795,35 @@ void BasicBlock::spliceDebugInfo(BasicBlock::iterator Dest, BasicBlock *Src, | | First Last - However: does the caller expect the "~" DPValues to end up before or after - the spliced segment? This is communciated in the "Head" bit of Dest, which - signals whether the caller called begin() or end() on this block. + However: does the caller expect the "~" DbgRecords to end up before or + after the spliced segment? This is communciated in the "Head" bit of Dest, + which signals whether the caller called begin() or end() on this block. - If the head bit is set, then all is well, we leave DPValues trailing just + If the head bit is set, then all is well, we leave DbgRecords trailing just like how dbg.value instructions would trail after instructions spliced to the beginning of this block. - If the head bit isn't set, then try to jam the "~" DPValues onto the front - of the First instruction, then splice like normal, which joins the "~" - DPValues with the "+" DPValues. However if the "+" DPValues are supposed to - be left behind in Src, then: - * detach the "+" DPValues, - * move the "~" DPValues onto First, + If the head bit isn't set, then try to jam the "~" DbgRecords onto the + front of the First instruction, then splice like normal, which joins the + "~" DbgRecords with the "+" DbgRecords. However if the "+" DbgRecords are + supposed to be left behind in Src, then: + * detach the "+" DbgRecords, + * move the "~" DbgRecords onto First, * splice like normal, - * replace the "+" DPValues onto the Last position. + * replace the "+" DbgRecords onto the Last position. Complicated, but gets the job done. */ - // If we're inserting at end(), and not in front of dangling DPValues, then - // move the DPValues onto "First". They'll then be moved naturally in the + // If we're inserting at end(), and not in front of dangling DbgRecords, then + // move the DbgRecords onto "First". They'll then be moved naturally in the // splice process. - DPMarker *MoreDanglingDPValues = nullptr; - DPMarker *OurTrailingDPValues = getTrailingDbgRecords(); - if (Dest == end() && !Dest.getHeadBit() && OurTrailingDPValues) { - // Are the "+" DPValues not supposed to move? If so, detach them + DPMarker *MoreDanglingDbgRecords = nullptr; + DPMarker *OurTrailingDbgRecords = getTrailingDbgRecords(); + if (Dest == end() && !Dest.getHeadBit() && OurTrailingDbgRecords) { + // Are the "+" DbgRecords not supposed to move? If so, detach them // temporarily. if (!First.getHeadBit() && First->hasDbgRecords()) { - MoreDanglingDPValues = Src->getMarker(First); - MoreDanglingDPValues->removeFromParent(); + MoreDanglingDbgRecords = Src->getMarker(First); + MoreDanglingDbgRecords->removeFromParent(); } if (First->hasDbgRecords()) { @@ -839,8 +839,8 @@ void BasicBlock::spliceDebugInfo(BasicBlock::iterator Dest, BasicBlock *Src, // No current marker, create one and absorb in. (FIXME: we can avoid an // allocation in the future). DPMarker *CurMarker = Src->createMarker(&*First); - CurMarker->absorbDebugValues(*OurTrailingDPValues, false); - OurTrailingDPValues->eraseFromParent(); + CurMarker->absorbDebugValues(*OurTrailingDbgRecords, false); + OurTrailingDbgRecords->eraseFromParent(); } deleteTrailingDbgRecords(); First.setHeadBit(true); @@ -849,16 +849,16 @@ void BasicBlock::spliceDebugInfo(BasicBlock::iterator Dest, BasicBlock *Src, // Call the main debug-info-splicing implementation. spliceDebugInfoImpl(Dest, Src, First, Last); - // Do we have some "+" DPValues hanging around that weren't supposed to move, - // and we detached to make things easier? - if (!MoreDanglingDPValues) + // Do we have some "+" DbgRecords hanging around that weren't supposed to + // move, and we detached to make things easier? + if (!MoreDanglingDbgRecords) return; // FIXME: we could avoid an allocation here sometimes. (adoptDbgRecords // requires an iterator). DPMarker *LastMarker = Src->createMarker(Last); - LastMarker->absorbDebugValues(*MoreDanglingDPValues, true); - MoreDanglingDPValues->eraseFromParent(); + LastMarker->absorbDebugValues(*MoreDanglingDbgRecords, true); + MoreDanglingDbgRecords->eraseFromParent(); } void BasicBlock::spliceDebugInfoImpl(BasicBlock::iterator Dest, BasicBlock *Src, @@ -870,15 +870,16 @@ void BasicBlock::spliceDebugInfoImpl(BasicBlock::iterator Dest, BasicBlock *Src, bool InsertAtHead = Dest.getHeadBit(); bool ReadFromHead = First.getHeadBit(); // Use this flag to signal the abnormal case, where we don't want to copy the - // DPValues ahead of the "Last" position. + // DbgRecords ahead of the "Last" position. bool ReadFromTail = !Last.getTailBit(); bool LastIsEnd = (Last == Src->end()); /* Here's an illustration of what we're about to do. We have two blocks, this and Src, and two segments of list. Each instruction is marked by a capital - while potential DPValue debug-info is marked out by "-" characters and a few - other special characters (+:=) where I want to highlight what's going on. + while potential DbgRecord debug-info is marked out by "-" characters and a + few other special characters (+:=) where I want to highlight what's going + on. Dest | @@ -889,18 +890,18 @@ void BasicBlock::spliceDebugInfoImpl(BasicBlock::iterator Dest, BasicBlock *Src, The splice method is going to take all the instructions from First up to (but not including) Last and insert them in _front_ of Dest, forming one - long list. All the DPValues attached to instructions _between_ First and + long list. All the DbgRecords attached to instructions _between_ First and Last need no maintenence. However, we have to do special things with the - DPValues marked with the +:= characters. We only have three positions: - should the "+" DPValues be transferred, and if so to where? Do we move the - ":" DPValues? Would they go in front of the "=" DPValues, or should the "=" - DPValues go before "+" DPValues? + DbgRecords marked with the +:= characters. We only have three positions: + should the "+" DbgRecords be transferred, and if so to where? Do we move the + ":" DbgRecords? Would they go in front of the "=" DbgRecords, or should the + "=" DbgRecords go before "+" DbgRecords? We're told which way it should be by the bits carried in the iterators. The "Head" bit indicates whether the specified position is supposed to be at the - front of the attached DPValues (true) or not (false). The Tail bit is true - on the other end of a range: is the range intended to include DPValues up to - the end (false) or not (true). + front of the attached DbgRecords (true) or not (false). The Tail bit is true + on the other end of a range: is the range intended to include DbgRecords up + to the end (false) or not (true). FIXME: the tail bit doesn't need to be distinct from the head bit, we could combine them. @@ -934,15 +935,16 @@ void BasicBlock::spliceDebugInfoImpl(BasicBlock::iterator Dest, BasicBlock *Src, */ - // Detach the marker at Dest -- this lets us move the "====" DPValues around. + // Detach the marker at Dest -- this lets us move the "====" DbgRecords + // around. DPMarker *DestMarker = nullptr; if (Dest != end()) { if ((DestMarker = getMarker(Dest))) DestMarker->removeFromParent(); } - // If we're moving the tail range of DPValues (":::"), absorb them into the - // front of the DPValues at Dest. + // If we're moving the tail range of DbgRecords (":::"), absorb them into the + // front of the DbgRecords at Dest. if (ReadFromTail && Src->getMarker(Last)) { DPMarker *FromLast = Src->getMarker(Last); if (LastIsEnd) { @@ -956,7 +958,7 @@ void BasicBlock::spliceDebugInfoImpl(BasicBlock::iterator Dest, BasicBlock *Src, } } - // If we're _not_ reading from the head of First, i.e. the "++++" DPValues, + // If we're _not_ reading from the head of First, i.e. the "++++" DbgRecords, // move their markers onto Last. They remain in the Src block. No action // needed. if (!ReadFromHead && First->hasDbgRecords()) { @@ -970,16 +972,16 @@ void BasicBlock::spliceDebugInfoImpl(BasicBlock::iterator Dest, BasicBlock *Src, } } - // Finally, do something with the "====" DPValues we detached. + // Finally, do something with the "====" DbgRecords we detached. if (DestMarker) { if (InsertAtHead) { - // Insert them at the end of the DPValues at Dest. The "::::" DPValues + // Insert them at the end of the DbgRecords at Dest. The "::::" DbgRecords // might be in front of them. DPMarker *NewDestMarker = createMarker(Dest); NewDestMarker->absorbDebugValues(*DestMarker, false); } else { // Insert them right at the start of the range we moved, ahead of First - // and the "++++" DPValues. + // and the "++++" DbgRecords. DPMarker *FirstMarker = createMarker(First); FirstMarker->absorbDebugValues(*DestMarker, true); } @@ -990,10 +992,10 @@ void BasicBlock::spliceDebugInfoImpl(BasicBlock::iterator Dest, BasicBlock *Src, // any trailing debug-info at the end of the block would "normally" have // been pushed in front of "First". Move it there now. DPMarker *FirstMarker = getMarker(First); - DPMarker *TrailingDPValues = getTrailingDbgRecords(); - if (TrailingDPValues) { - FirstMarker->absorbDebugValues(*TrailingDPValues, true); - TrailingDPValues->eraseFromParent(); + DPMarker *TrailingDbgRecords = getTrailingDbgRecords(); + if (TrailingDbgRecords) { + FirstMarker->absorbDebugValues(*TrailingDbgRecords, true); + TrailingDbgRecords->eraseFromParent(); deleteTrailingDbgRecords(); } } @@ -1024,7 +1026,7 @@ void BasicBlock::splice(iterator Dest, BasicBlock *Src, iterator First, // And move the instructions. getInstList().splice(Dest, Src->getInstList(), First, Last); - flushTerminatorDbgValues(); + flushTerminatorDbgRecords(); } void BasicBlock::insertDbgRecordAfter(DbgRecord *DPV, Instruction *I) { @@ -1057,38 +1059,40 @@ DPMarker *BasicBlock::getMarker(InstListType::iterator It) { } void BasicBlock::reinsertInstInDbgRecords( - Instruction *I, std::optional Pos) { + Instruction *I, std::optional Pos) { // "I" was originally removed from a position where it was - // immediately in front of Pos. Any DPValues on that position then "fell down" - // onto Pos. "I" has been re-inserted at the front of that wedge of DPValues, - // shuffle them around to represent the original positioning. To illustrate: + // immediately in front of Pos. Any DbgRecords on that position then "fell + // down" onto Pos. "I" has been re-inserted at the front of that wedge of + // DbgRecords, shuffle them around to represent the original positioning. To + // illustrate: // // Instructions: I1---I---I0 - // DPValues: DDD DDD + // DbgRecords: DDD DDD // // Instruction "I" removed, // // Instructions: I1------I0 - // DPValues: DDDDDD + // DbgRecords: DDDDDD // ^Pos // // Instruction "I" re-inserted (now): // // Instructions: I1---I------I0 - // DPValues: DDDDDD + // DbgRecords: DDDDDD // ^Pos // // After this method completes: // // Instructions: I1---I---I0 - // DPValues: DDD DDD + // DbgRecords: DDD DDD - // This happens if there were no DPValues on I0. Are there now DPValues there? + // This happens if there were no DbgRecords on I0. Are there now DbgRecords + // there? if (!Pos) { DPMarker *NextMarker = getNextMarker(I); if (!NextMarker) return; - if (NextMarker->StoredDPValues.empty()) + if (NextMarker->StoredDbgRecords.empty()) return; // There are DPMarkers there now -- they fell down from "I". DPMarker *ThisMarker = createMarker(I); @@ -1096,15 +1100,15 @@ void BasicBlock::reinsertInstInDbgRecords( return; } - // Is there even a range of DPValues to move? + // Is there even a range of DbgRecords to move? DPMarker *DPM = (*Pos)->getMarker(); - auto Range = make_range(DPM->StoredDPValues.begin(), (*Pos)); + auto Range = make_range(DPM->StoredDbgRecords.begin(), (*Pos)); if (Range.begin() == Range.end()) return; // Otherwise: splice. DPMarker *ThisMarker = createMarker(I); - assert(ThisMarker->StoredDPValues.empty()); + assert(ThisMarker->StoredDbgRecords.empty()); ThisMarker->absorbDebugValues(Range, *DPM, true); } diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index e63b1e67dad7..d16895058ba2 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -895,7 +895,7 @@ bool llvm::stripNonLineTableDebugInfo(Module &M) { if (I.hasMetadataOtherThanDebugLoc()) I.setMetadata("heapallocsite", nullptr); - // Strip any DPValues attached. + // Strip any DbgRecords attached. I.dropDbgRecords(); } } diff --git a/llvm/lib/IR/DebugProgramInstruction.cpp b/llvm/lib/IR/DebugProgramInstruction.cpp index 019b00c2e208..f34d3aecfa0a 100644 --- a/llvm/lib/IR/DebugProgramInstruction.cpp +++ b/llvm/lib/IR/DebugProgramInstruction.cpp @@ -1,4 +1,4 @@ -//======-- DebugProgramInstruction.cpp - Implement DPValues/DPMarkers --======// +//=====-- DebugProgramInstruction.cpp - Implement DbgRecords/DPMarkers --=====// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -541,21 +541,21 @@ void DbgRecord::moveAfter(DbgRecord *MoveAfter) { /////////////////////////////////////////////////////////////////////////////// // An empty, global, DPMarker for the purpose of describing empty ranges of -// DPValues. +// DbgRecords. DPMarker DPMarker::EmptyDPMarker; void DPMarker::dropDbgRecords() { - while (!StoredDPValues.empty()) { - auto It = StoredDPValues.begin(); + while (!StoredDbgRecords.empty()) { + auto It = StoredDbgRecords.begin(); DbgRecord *DR = &*It; - StoredDPValues.erase(It); + StoredDbgRecords.erase(It); DR->deleteRecord(); } } void DPMarker::dropOneDbgRecord(DbgRecord *DR) { assert(DR->getMarker() == this); - StoredDPValues.erase(DR->getIterator()); + StoredDbgRecords.erase(DR->getIterator()); DR->deleteRecord(); } @@ -566,15 +566,15 @@ const BasicBlock *DPMarker::getParent() const { BasicBlock *DPMarker::getParent() { return MarkedInstr->getParent(); } void DPMarker::removeMarker() { - // Are there any DPValues in this DPMarker? If not, nothing to preserve. + // Are there any DbgRecords in this DPMarker? If not, nothing to preserve. Instruction *Owner = MarkedInstr; - if (StoredDPValues.empty()) { + if (StoredDbgRecords.empty()) { eraseFromParent(); Owner->DbgMarker = nullptr; return; } - // The attached DPValues need to be preserved; attach them to the next + // The attached DbgRecords need to be preserved; attach them to the next // instruction. If there isn't a next instruction, put them on the // "trailing" list. DPMarker *NextMarker = Owner->getParent()->getNextMarker(Owner); @@ -610,15 +610,15 @@ void DPMarker::eraseFromParent() { } iterator_range DPMarker::getDbgRecordRange() { - return make_range(StoredDPValues.begin(), StoredDPValues.end()); + return make_range(StoredDbgRecords.begin(), StoredDbgRecords.end()); } iterator_range DPMarker::getDbgRecordRange() const { - return make_range(StoredDPValues.begin(), StoredDPValues.end()); + return make_range(StoredDbgRecords.begin(), StoredDbgRecords.end()); } void DbgRecord::removeFromParent() { - getMarker()->StoredDPValues.erase(getIterator()); + getMarker()->StoredDbgRecords.erase(getIterator()); Marker = nullptr; } @@ -628,29 +628,29 @@ void DbgRecord::eraseFromParent() { } void DPMarker::insertDbgRecord(DbgRecord *New, bool InsertAtHead) { - auto It = InsertAtHead ? StoredDPValues.begin() : StoredDPValues.end(); - StoredDPValues.insert(It, *New); + auto It = InsertAtHead ? StoredDbgRecords.begin() : StoredDbgRecords.end(); + StoredDbgRecords.insert(It, *New); New->setMarker(this); } void DPMarker::insertDbgRecord(DbgRecord *New, DbgRecord *InsertBefore) { assert(InsertBefore->getMarker() == this && - "DPValue 'InsertBefore' must be contained in this DPMarker!"); - StoredDPValues.insert(InsertBefore->getIterator(), *New); + "DbgRecord 'InsertBefore' must be contained in this DPMarker!"); + StoredDbgRecords.insert(InsertBefore->getIterator(), *New); New->setMarker(this); } void DPMarker::insertDbgRecordAfter(DbgRecord *New, DbgRecord *InsertAfter) { assert(InsertAfter->getMarker() == this && - "DPValue 'InsertAfter' must be contained in this DPMarker!"); - StoredDPValues.insert(++(InsertAfter->getIterator()), *New); + "DbgRecord 'InsertAfter' must be contained in this DPMarker!"); + StoredDbgRecords.insert(++(InsertAfter->getIterator()), *New); New->setMarker(this); } void DPMarker::absorbDebugValues(DPMarker &Src, bool InsertAtHead) { - auto It = InsertAtHead ? StoredDPValues.begin() : StoredDPValues.end(); - for (DbgRecord &DPV : Src.StoredDPValues) + auto It = InsertAtHead ? StoredDbgRecords.begin() : StoredDbgRecords.end(); + for (DbgRecord &DPV : Src.StoredDbgRecords) DPV.setMarker(this); - StoredDPValues.splice(It, Src.StoredDPValues); + StoredDbgRecords.splice(It, Src.StoredDbgRecords); } void DPMarker::absorbDebugValues(iterator_range Range, @@ -659,45 +659,45 @@ void DPMarker::absorbDebugValues(iterator_range Range, DR.setMarker(this); auto InsertPos = - (InsertAtHead) ? StoredDPValues.begin() : StoredDPValues.end(); + (InsertAtHead) ? StoredDbgRecords.begin() : StoredDbgRecords.end(); - StoredDPValues.splice(InsertPos, Src.StoredDPValues, Range.begin(), - Range.end()); + StoredDbgRecords.splice(InsertPos, Src.StoredDbgRecords, Range.begin(), + Range.end()); } iterator_range::iterator> DPMarker::cloneDebugInfoFrom( DPMarker *From, std::optional::iterator> from_here, bool InsertAtHead) { DbgRecord *First = nullptr; - // Work out what range of DPValues to clone: normally all the contents of the - // "From" marker, optionally we can start from the from_here position down to - // end(). + // Work out what range of DbgRecords to clone: normally all the contents of + // the "From" marker, optionally we can start from the from_here position down + // to end(). auto Range = - make_range(From->StoredDPValues.begin(), From->StoredDPValues.end()); + make_range(From->StoredDbgRecords.begin(), From->StoredDbgRecords.end()); if (from_here.has_value()) - Range = make_range(*from_here, From->StoredDPValues.end()); + Range = make_range(*from_here, From->StoredDbgRecords.end()); // Clone each DPValue and insert into StoreDPValues; optionally place them at // the start or the end of the list. - auto Pos = (InsertAtHead) ? StoredDPValues.begin() : StoredDPValues.end(); + auto Pos = (InsertAtHead) ? StoredDbgRecords.begin() : StoredDbgRecords.end(); for (DbgRecord &DR : Range) { DbgRecord *New = DR.clone(); New->setMarker(this); - StoredDPValues.insert(Pos, *New); + StoredDbgRecords.insert(Pos, *New); if (!First) First = New; } if (!First) - return {StoredDPValues.end(), StoredDPValues.end()}; + return {StoredDbgRecords.end(), StoredDbgRecords.end()}; if (InsertAtHead) // If InsertAtHead is set, we cloned a range onto the front of of the - // StoredDPValues collection, return that range. - return {StoredDPValues.begin(), Pos}; + // StoredDbgRecords collection, return that range. + return {StoredDbgRecords.begin(), Pos}; else // We inserted a block at the end, return that range. - return {First->getIterator(), StoredDPValues.end()}; + return {First->getIterator(), StoredDbgRecords.end()}; } } // end namespace llvm diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp index e0892398f434..7a677d7f3c2b 100644 --- a/llvm/lib/IR/Instruction.cpp +++ b/llvm/lib/IR/Instruction.cpp @@ -143,7 +143,7 @@ void Instruction::insertBefore(BasicBlock &BB, return; // We've inserted "this": if InsertAtHead is set then it comes before any - // DPValues attached to InsertPos. But if it's not set, then any DPValues + // DPValues attached to InsertPos. But if it's not set, then any DbgRecords // should now come before "this". bool InsertAtHead = InsertPos.getHeadBit(); if (!InsertAtHead) { @@ -166,10 +166,10 @@ void Instruction::insertBefore(BasicBlock &BB, } // If we're inserting a terminator, check if we need to flush out - // TrailingDPValues. Inserting instructions at the end of an incomplete + // TrailingDbgRecords. Inserting instructions at the end of an incomplete // block is handled by the code block above. if (isTerminator()) - getParent()->flushTerminatorDbgValues(); + getParent()->flushTerminatorDbgRecords(); } /// Unlink this instruction from its current basic block and insert it into the @@ -212,12 +212,12 @@ void Instruction::moveBeforeImpl(BasicBlock &BB, InstListType::iterator I, assert(I == BB.end() || I->getParent() == &BB); bool InsertAtHead = I.getHeadBit(); - // If we've been given the "Preserve" flag, then just move the DPValues with + // If we've been given the "Preserve" flag, then just move the DbgRecords with // the instruction, no more special handling needed. if (BB.IsNewDbgInfoFormat && DbgMarker && !Preserve) { if (I != this->getIterator() || InsertAtHead) { // "this" is definitely moving in the list, or it's moving ahead of its - // attached DPValues. Detach any existing DPValues. + // attached DPValues. Detach any existing DbgRecords. handleMarkerRemoval(); } } @@ -229,15 +229,15 @@ void Instruction::moveBeforeImpl(BasicBlock &BB, InstListType::iterator I, if (BB.IsNewDbgInfoFormat && !Preserve) { DPMarker *NextMarker = getParent()->getNextMarker(this); - // If we're inserting at point I, and not in front of the DPValues attached - // there, then we should absorb the DPValues attached to I. + // If we're inserting at point I, and not in front of the DbgRecords + // attached there, then we should absorb the DbgRecords attached to I. if (!InsertAtHead && NextMarker && !NextMarker->empty()) { adoptDbgRecords(&BB, I, false); } } if (isTerminator()) - getParent()->flushTerminatorDbgValues(); + getParent()->flushTerminatorDbgRecords(); } iterator_range Instruction::cloneDebugInfoFrom( @@ -263,11 +263,11 @@ Instruction::getDbgReinsertionPosition() { if (!NextMarker) return std::nullopt; - // Are there any DPValues in the next marker? - if (NextMarker->StoredDPValues.empty()) + // Are there any DbgRecords in the next marker? + if (NextMarker->StoredDbgRecords.empty()) return std::nullopt; - return NextMarker->StoredDPValues.begin(); + return NextMarker->StoredDbgRecords.begin(); } bool Instruction::hasDbgRecords() const { return !getDbgRecordRange().empty(); } @@ -275,20 +275,20 @@ bool Instruction::hasDbgRecords() const { return !getDbgRecordRange().empty(); } void Instruction::adoptDbgRecords(BasicBlock *BB, BasicBlock::iterator It, bool InsertAtHead) { DPMarker *SrcMarker = BB->getMarker(It); - auto ReleaseTrailingDPValues = [BB, It, SrcMarker]() { + auto ReleaseTrailingDbgRecords = [BB, It, SrcMarker]() { if (BB->end() == It) { SrcMarker->eraseFromParent(); BB->deleteTrailingDbgRecords(); } }; - if (!SrcMarker || SrcMarker->StoredDPValues.empty()) { - ReleaseTrailingDPValues(); + if (!SrcMarker || SrcMarker->StoredDbgRecords.empty()) { + ReleaseTrailingDbgRecords(); return; } // If we have DPMarkers attached to this instruction, we have to honour the - // ordering of DPValues between this and the other marker. Fall back to just + // ordering of DbgRecords between this and the other marker. Fall back to just // absorbing from the source. if (DbgMarker || It == BB->end()) { // Ensure we _do_ have a marker. @@ -304,10 +304,11 @@ void Instruction::adoptDbgRecords(BasicBlock *BB, BasicBlock::iterator It, // block, it's important to not leave the empty marker trailing. It will // give a misleading impression that some debug records have been left // trailing. - ReleaseTrailingDPValues(); + ReleaseTrailingDbgRecords(); } else { - // Optimisation: we're transferring all the DPValues from the source marker - // onto this empty location: just adopt the other instructions marker. + // Optimisation: we're transferring all the DbgRecords from the source + // marker onto this empty location: just adopt the other instructions + // marker. DbgMarker = SrcMarker; DbgMarker->MarkedInstr = this; It->DbgMarker = nullptr; diff --git a/llvm/lib/IR/LLVMContextImpl.cpp b/llvm/lib/IR/LLVMContextImpl.cpp index a0bf9cae7926..a471314ccb94 100644 --- a/llvm/lib/IR/LLVMContextImpl.cpp +++ b/llvm/lib/IR/LLVMContextImpl.cpp @@ -50,7 +50,7 @@ LLVMContextImpl::~LLVMContextImpl() { // when it's terminator was removed were eventually replaced. This assertion // firing indicates that DPValues went missing during the lifetime of the // LLVMContext. - assert(TrailingDPValues.empty() && "DPValue records in blocks not cleaned"); + assert(TrailingDbgRecords.empty() && "DbgRecords in blocks not cleaned"); #endif // NOTE: We need to delete the contents of OwnedModules, but Module's dtor diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h index c841b28ca438..b1dcb262fb65 100644 --- a/llvm/lib/IR/LLVMContextImpl.h +++ b/llvm/lib/IR/LLVMContextImpl.h @@ -1684,19 +1684,19 @@ public: /// such a way. These are stored in LLVMContext because typically LLVM only /// edits a small number of blocks at a time, so there's no need to bloat /// BasicBlock with such a data structure. - SmallDenseMap TrailingDPValues; + SmallDenseMap TrailingDbgRecords; - // Set, get and delete operations for TrailingDPValues. + // Set, get and delete operations for TrailingDbgRecords. void setTrailingDbgRecords(BasicBlock *B, DPMarker *M) { - assert(!TrailingDPValues.count(B)); - TrailingDPValues[B] = M; + assert(!TrailingDbgRecords.count(B)); + TrailingDbgRecords[B] = M; } DPMarker *getTrailingDbgRecords(BasicBlock *B) { - return TrailingDPValues.lookup(B); + return TrailingDbgRecords.lookup(B); } - void deleteTrailingDbgRecords(BasicBlock *B) { TrailingDPValues.erase(B); } + void deleteTrailingDbgRecords(BasicBlock *B) { TrailingDbgRecords.erase(B); } }; } // end namespace llvm diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index 7b74caac9e08..a87e5a3a923a 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -2848,7 +2848,7 @@ unsigned llvm::changeToUnreachable(Instruction *I, bool PreserveLCSSA, Updates.push_back({DominatorTree::Delete, BB, UniqueSuccessor}); DTU->applyUpdates(Updates); } - BB->flushTerminatorDbgValues(); + BB->flushTerminatorDbgRecords(); return NumInstrsRemoved; } diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp index 8c6af7afa875..acfd87c64b79 100644 --- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp @@ -577,28 +577,28 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) { Module *M = OrigHeader->getModule(); - // Track the next DPValue to clone. If we have a sequence where an + // Track the next DbgRecord to clone. If we have a sequence where an // instruction is hoisted instead of being cloned: - // DPValue blah + // DbgRecord blah // %foo = add i32 0, 0 - // DPValue xyzzy + // DbgRecord xyzzy // %bar = call i32 @foobar() - // where %foo is hoisted, then the DPValue "blah" will be seen twice, once + // where %foo is hoisted, then the DbgRecord "blah" will be seen twice, once // attached to %foo, then when %foo his hoisted it will "fall down" onto the // function call: - // DPValue blah - // DPValue xyzzy + // DbgRecord blah + // DbgRecord xyzzy // %bar = call i32 @foobar() // causing it to appear attached to the call too. // // To avoid this, cloneDebugInfoFrom takes an optional "start cloning from - // here" position to account for this behaviour. We point it at any DPValues - // on the next instruction, here labelled xyzzy, before we hoist %foo. - // Later, we only only clone DPValues from that position (xyzzy) onwards, - // which avoids cloning DPValue "blah" multiple times. - // (Stored as a range because it gives us a natural way of testing whether - // there were DPValues on the next instruction before we hoisted things). - iterator_range NextDbgInsts = + // here" position to account for this behaviour. We point it at any + // DbgRecords on the next instruction, here labelled xyzzy, before we hoist + // %foo. Later, we only only clone DbgRecords from that position (xyzzy) + // onwards, which avoids cloning DbgRecord "blah" multiple times. (Stored as + // a range because it gives us a natural way of testing whether + // there were DbgRecords on the next instruction before we hoisted things). + iterator_range NextDbgInsts = (I != E) ? I->getDbgRecordRange() : DPMarker::getEmptyDbgRecordRange(); while (I != E) { @@ -777,7 +777,7 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) { // OrigPreHeader's old terminator (the original branch into the loop), and // remove the corresponding incoming values from the PHI nodes in OrigHeader. LoopEntryBranch->eraseFromParent(); - OrigPreheader->flushTerminatorDbgValues(); + OrigPreheader->flushTerminatorDbgRecords(); // Update MemorySSA before the rewrite call below changes the 1:1 // instruction:cloned_instruction_or_value mapping. diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 0f3d1403481d..6d2a6a3e7f11 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -1572,7 +1572,8 @@ hoistLockstepIdenticalDPValues(Instruction *TI, Instruction *I1, while (none_of(Itrs, atEnd)) { bool HoistDPVs = allIdentical(Itrs); for (CurrentAndEndIt &Pair : Itrs) { - // Increment Current iterator now as we may be about to move the DPValue. + // Increment Current iterator now as we may be about to move the + // DbgRecord. DbgRecord &DR = *Pair.first++; if (HoistDPVs) { DR.removeFromParent(); @@ -5304,7 +5305,7 @@ bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) { // Ensure that any debug-info records that used to occur after the Unreachable // are moved to in front of it -- otherwise they'll "dangle" at the end of // the block. - BB->flushTerminatorDbgValues(); + BB->flushTerminatorDbgRecords(); // Debug-info records on the unreachable inst itself should be deleted, as // below we delete everything past the final executable instruction. @@ -5326,8 +5327,8 @@ bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) { // block will be the unwind edges of Invoke/CatchSwitch/CleanupReturn, // and we can therefore guarantee this block will be erased. - // If we're deleting this, we're deleting any subsequent dbg.values, so - // delete DPValue records of variable information. + // If we're deleting this, we're deleting any subsequent debug info, so + // delete DbgRecords. BBI->dropDbgRecords(); // Delete this instruction (any uses are guaranteed to be dead) diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp index 3da161043d6c..abb7a4452a37 100644 --- a/llvm/lib/Transforms/Utils/ValueMapper.cpp +++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp @@ -146,7 +146,7 @@ public: Value *mapValue(const Value *V); void remapInstruction(Instruction *I); void remapFunction(Function &F); - void remapDPValue(DbgRecord &DPV); + void remapDbgRecord(DbgRecord &DPV); Constant *mapConstant(const Constant *C) { return cast_or_null(mapValue(C)); @@ -537,7 +537,7 @@ Value *Mapper::mapValue(const Value *V) { return getVM()[V] = ConstantPointerNull::get(cast(NewTy)); } -void Mapper::remapDPValue(DbgRecord &DR) { +void Mapper::remapDbgRecord(DbgRecord &DR) { if (DPLabel *DPL = dyn_cast(&DR)) { DPL->setLabel(cast(mapMetadata(DPL->getLabel()))); return; @@ -1067,7 +1067,7 @@ void Mapper::remapFunction(Function &F) { for (Instruction &I : BB) { remapInstruction(&I); for (DbgRecord &DR : I.getDbgRecordRange()) - remapDPValue(DR); + remapDbgRecord(DR); } } } @@ -1234,7 +1234,7 @@ void ValueMapper::remapInstruction(Instruction &I) { } void ValueMapper::remapDPValue(Module *M, DPValue &V) { - FlushingMapper(pImpl)->remapDPValue(V); + FlushingMapper(pImpl)->remapDbgRecord(V); } void ValueMapper::remapDPValueRange( diff --git a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp index e23c7eaa4930..bfc64cb84143 100644 --- a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp +++ b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp @@ -149,7 +149,7 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) { Instruction *Instr2 = Instr1->getNextNode(); DPMarker *Marker1 = Instr1->DbgMarker; DPMarker *Marker2 = Instr2->DbgMarker; - // There's no TrailingDPValues marker allocated yet. + // There's no TrailingDbgRecords marker allocated yet. DPMarker *EndMarker = nullptr; // Check that the "getMarker" utilities operate as expected. @@ -159,26 +159,26 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) { EXPECT_EQ(BB.getNextMarker(Instr2), EndMarker); // Is nullptr. // There should be two DPValues, - EXPECT_EQ(Marker1->StoredDPValues.size(), 1u); - EXPECT_EQ(Marker2->StoredDPValues.size(), 1u); + EXPECT_EQ(Marker1->StoredDbgRecords.size(), 1u); + EXPECT_EQ(Marker2->StoredDbgRecords.size(), 1u); // Unlink them and try to re-insert them through the basic block. - DbgRecord *DPV1 = &*Marker1->StoredDPValues.begin(); - DbgRecord *DPV2 = &*Marker2->StoredDPValues.begin(); + DbgRecord *DPV1 = &*Marker1->StoredDbgRecords.begin(); + DbgRecord *DPV2 = &*Marker2->StoredDbgRecords.begin(); DPV1->removeFromParent(); DPV2->removeFromParent(); - EXPECT_TRUE(Marker1->StoredDPValues.empty()); - EXPECT_TRUE(Marker2->StoredDPValues.empty()); + EXPECT_TRUE(Marker1->StoredDbgRecords.empty()); + EXPECT_TRUE(Marker2->StoredDbgRecords.empty()); // This should appear in Marker1. BB.insertDbgRecordBefore(DPV1, BB.begin()); - EXPECT_EQ(Marker1->StoredDPValues.size(), 1u); - EXPECT_EQ(DPV1, &*Marker1->StoredDPValues.begin()); + EXPECT_EQ(Marker1->StoredDbgRecords.size(), 1u); + EXPECT_EQ(DPV1, &*Marker1->StoredDbgRecords.begin()); // This should attach to Marker2. BB.insertDbgRecordAfter(DPV2, &*BB.begin()); - EXPECT_EQ(Marker2->StoredDPValues.size(), 1u); - EXPECT_EQ(DPV2, &*Marker2->StoredDPValues.begin()); + EXPECT_EQ(Marker2->StoredDbgRecords.size(), 1u); + EXPECT_EQ(DPV2, &*Marker2->StoredDbgRecords.begin()); // Now, how about removing instructions? That should cause any DPValues to // "fall down". @@ -186,7 +186,7 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) { Marker1 = nullptr; // DPValues should now be in Marker2. EXPECT_EQ(BB.size(), 1u); - EXPECT_EQ(Marker2->StoredDPValues.size(), 2u); + EXPECT_EQ(Marker2->StoredDbgRecords.size(), 2u); // They should also be in the correct order. SmallVector DPVs; for (DbgRecord &DPV : Marker2->getDbgRecordRange()) @@ -201,7 +201,7 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) { EXPECT_TRUE(BB.empty()); EndMarker = BB.getTrailingDbgRecords(); ASSERT_NE(EndMarker, nullptr); - EXPECT_EQ(EndMarker->StoredDPValues.size(), 2u); + EXPECT_EQ(EndMarker->StoredDbgRecords.size(), 2u); // Again, these should arrive in the correct order. DPVs.clear(); @@ -213,13 +213,13 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) { // Inserting a normal instruction at the beginning: shouldn't dislodge the // DPValues. It's intended to not go at the start. Instr1->insertBefore(BB, BB.begin()); - EXPECT_EQ(EndMarker->StoredDPValues.size(), 2u); + EXPECT_EQ(EndMarker->StoredDbgRecords.size(), 2u); Instr1->removeFromParent(); // Inserting at end(): should dislodge the DPValues, if they were dbg.values // then they would sit "above" the new instruction. Instr1->insertBefore(BB, BB.end()); - EXPECT_EQ(Instr1->DbgMarker->StoredDPValues.size(), 2u); + EXPECT_EQ(Instr1->DbgMarker->StoredDbgRecords.size(), 2u); // We should de-allocate the trailing marker when something is inserted // at end(). EXPECT_EQ(BB.getTrailingDbgRecords(), nullptr); @@ -227,14 +227,14 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) { // Remove Instr1: now the DPValues will fall down again, Instr1->removeFromParent(); EndMarker = BB.getTrailingDbgRecords(); - EXPECT_EQ(EndMarker->StoredDPValues.size(), 2u); + EXPECT_EQ(EndMarker->StoredDbgRecords.size(), 2u); // Inserting a terminator, however it's intended, should dislodge the // trailing DPValues, as it's the clear intention of the caller that this be // the final instr in the block, and DPValues aren't allowed to live off the // end forever. Instr2->insertBefore(BB, BB.begin()); - EXPECT_EQ(Instr2->DbgMarker->StoredDPValues.size(), 2u); + EXPECT_EQ(Instr2->DbgMarker->StoredDbgRecords.size(), 2u); EXPECT_EQ(BB.getTrailingDbgRecords(), nullptr); // Teardown, @@ -298,24 +298,24 @@ TEST(BasicBlockDbgInfoTest, HeadBitOperations) { Instruction *DInst = CInst->getNextNode(); // CInst should have debug-info. ASSERT_TRUE(CInst->DbgMarker); - EXPECT_FALSE(CInst->DbgMarker->StoredDPValues.empty()); + EXPECT_FALSE(CInst->DbgMarker->StoredDbgRecords.empty()); // If we move "c" to the start of the block, just normally, then the DPValues // should fall down to "d". CInst->moveBefore(BB, BeginIt2); - EXPECT_TRUE(!CInst->DbgMarker || CInst->DbgMarker->StoredDPValues.empty()); + EXPECT_TRUE(!CInst->DbgMarker || CInst->DbgMarker->StoredDbgRecords.empty()); ASSERT_TRUE(DInst->DbgMarker); - EXPECT_FALSE(DInst->DbgMarker->StoredDPValues.empty()); + EXPECT_FALSE(DInst->DbgMarker->StoredDbgRecords.empty()); // Wheras if we move D to the start of the block with moveBeforePreserving, // the DPValues should move with it. DInst->moveBeforePreserving(BB, BB.begin()); - EXPECT_FALSE(DInst->DbgMarker->StoredDPValues.empty()); + EXPECT_FALSE(DInst->DbgMarker->StoredDbgRecords.empty()); EXPECT_EQ(&*BB.begin(), DInst); // Similarly, moveAfterPreserving "D" to "C" should move DPValues with "D". DInst->moveAfterPreserving(CInst); - EXPECT_FALSE(DInst->DbgMarker->StoredDPValues.empty()); + EXPECT_FALSE(DInst->DbgMarker->StoredDbgRecords.empty()); // (move back to the start...) DInst->moveBeforePreserving(BB, BB.begin()); @@ -324,8 +324,8 @@ TEST(BasicBlockDbgInfoTest, HeadBitOperations) { // If we move "C" to the beginning of the block, it should go before the // DPValues. They'll stay on "D". CInst->moveBefore(BB, BB.begin()); - EXPECT_TRUE(!CInst->DbgMarker || CInst->DbgMarker->StoredDPValues.empty()); - EXPECT_FALSE(DInst->DbgMarker->StoredDPValues.empty()); + EXPECT_TRUE(!CInst->DbgMarker || CInst->DbgMarker->StoredDbgRecords.empty()); + EXPECT_FALSE(DInst->DbgMarker->StoredDbgRecords.empty()); EXPECT_EQ(&*BB.begin(), CInst); EXPECT_EQ(CInst->getNextNode(), DInst); @@ -341,8 +341,8 @@ TEST(BasicBlockDbgInfoTest, HeadBitOperations) { // run of dbg.values and the next instruction. CInst->moveBefore(BB, DInst->getIterator()); // CInst gains the DPValues. - EXPECT_TRUE(!DInst->DbgMarker || DInst->DbgMarker->StoredDPValues.empty()); - EXPECT_FALSE(CInst->DbgMarker->StoredDPValues.empty()); + EXPECT_TRUE(!DInst->DbgMarker || DInst->DbgMarker->StoredDbgRecords.empty()); + EXPECT_FALSE(CInst->DbgMarker->StoredDbgRecords.empty()); EXPECT_EQ(&*BB.begin(), CInst); UseNewDbgInfoFormat = false; @@ -390,16 +390,16 @@ TEST(BasicBlockDbgInfoTest, InstrDbgAccess) { ASSERT_FALSE(BInst->DbgMarker); ASSERT_TRUE(CInst->DbgMarker); - ASSERT_EQ(CInst->DbgMarker->StoredDPValues.size(), 1u); - DbgRecord *DPV1 = &*CInst->DbgMarker->StoredDPValues.begin(); + ASSERT_EQ(CInst->DbgMarker->StoredDbgRecords.size(), 1u); + DbgRecord *DPV1 = &*CInst->DbgMarker->StoredDbgRecords.begin(); ASSERT_TRUE(DPV1); EXPECT_FALSE(BInst->hasDbgRecords()); // Clone DPValues from one inst to another. Other arguments to clone are // tested in DPMarker test. auto Range1 = BInst->cloneDebugInfoFrom(CInst); - EXPECT_EQ(BInst->DbgMarker->StoredDPValues.size(), 1u); - DbgRecord *DPV2 = &*BInst->DbgMarker->StoredDPValues.begin(); + EXPECT_EQ(BInst->DbgMarker->StoredDbgRecords.size(), 1u); + DbgRecord *DPV2 = &*BInst->DbgMarker->StoredDbgRecords.begin(); EXPECT_EQ(std::distance(Range1.begin(), Range1.end()), 1u); EXPECT_EQ(&*Range1.begin(), DPV2); EXPECT_NE(DPV1, DPV2); @@ -417,12 +417,12 @@ TEST(BasicBlockDbgInfoTest, InstrDbgAccess) { // Dropping should be easy, BInst->dropDbgRecords(); EXPECT_FALSE(BInst->hasDbgRecords()); - EXPECT_EQ(BInst->DbgMarker->StoredDPValues.size(), 0u); + EXPECT_EQ(BInst->DbgMarker->StoredDbgRecords.size(), 0u); // And we should be able to drop individual DPValues. CInst->dropOneDbgRecord(DPV1); EXPECT_FALSE(CInst->hasDbgRecords()); - EXPECT_EQ(CInst->DbgMarker->StoredDPValues.size(), 0u); + EXPECT_EQ(CInst->DbgMarker->StoredDbgRecords.size(), 0u); UseNewDbgInfoFormat = false; } @@ -531,9 +531,9 @@ protected: Branch = &*Last; CInst = &*Dest; - DPVA = cast(&*BInst->DbgMarker->StoredDPValues.begin()); - DPVB = cast(&*Branch->DbgMarker->StoredDPValues.begin()); - DPVConst = cast(&*CInst->DbgMarker->StoredDPValues.begin()); + DPVA = cast(&*BInst->DbgMarker->StoredDbgRecords.begin()); + DPVB = cast(&*Branch->DbgMarker->StoredDbgRecords.begin()); + DPVConst = cast(&*CInst->DbgMarker->StoredDbgRecords.begin()); } void TearDown() override { UseNewDbgInfoFormat = false; } @@ -1171,7 +1171,7 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceTrailing) { // spliced in. Instruction *BInst = &*Entry.begin(); ASSERT_TRUE(BInst->DbgMarker); - EXPECT_EQ(BInst->DbgMarker->StoredDPValues.size(), 1u); + EXPECT_EQ(BInst->DbgMarker->StoredDbgRecords.size(), 1u); UseNewDbgInfoFormat = false; } @@ -1387,7 +1387,7 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty1) { // should be in the correct order of %a, then 0. Instruction *BInst = &*Entry.begin(); ASSERT_TRUE(BInst->hasDbgRecords()); - EXPECT_EQ(BInst->DbgMarker->StoredDPValues.size(), 2u); + EXPECT_EQ(BInst->DbgMarker->StoredDbgRecords.size(), 2u); SmallVector DPValues; for (DbgRecord &DPV : BInst->getDbgRecordRange()) DPValues.push_back(cast(&DPV)); @@ -1457,7 +1457,7 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty2) { // We should now have one dbg.values on the first instruction, %a. Instruction *BInst = &*Entry.begin(); ASSERT_TRUE(BInst->hasDbgRecords()); - EXPECT_EQ(BInst->DbgMarker->StoredDPValues.size(), 1u); + EXPECT_EQ(BInst->DbgMarker->StoredDbgRecords.size(), 1u); SmallVector DPValues; for (DbgRecord &DPV : BInst->getDbgRecordRange()) DPValues.push_back(cast(&DPV)); diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp index 0b019c26148b..4bd11d26071b 100644 --- a/llvm/unittests/IR/DebugInfoTest.cpp +++ b/llvm/unittests/IR/DebugInfoTest.cpp @@ -951,7 +951,7 @@ TEST(MetadataTest, ConvertDbgToDPValue) { ExitBlock->createMarker(FirstInst); ExitBlock->createMarker(RetInst); - // Insert DPValues into markers, order should come out DPV2, DPV1. + // Insert DbgRecords into markers, order should come out DPV2, DPV1. FirstInst->DbgMarker->insertDbgRecord(DPV1, false); FirstInst->DbgMarker->insertDbgRecord(DPV2, true); unsigned int ItCount = 0; @@ -964,7 +964,7 @@ TEST(MetadataTest, ConvertDbgToDPValue) { // Clone them onto the second marker -- should allocate new DPVs. RetInst->DbgMarker->cloneDebugInfoFrom(FirstInst->DbgMarker, std::nullopt, false); - EXPECT_EQ(RetInst->DbgMarker->StoredDPValues.size(), 2u); + EXPECT_EQ(RetInst->DbgMarker->StoredDbgRecords.size(), 2u); ItCount = 0; // Check these things store the same information; but that they're not the same // objects. @@ -980,25 +980,25 @@ TEST(MetadataTest, ConvertDbgToDPValue) { } RetInst->DbgMarker->dropDbgRecords(); - EXPECT_EQ(RetInst->DbgMarker->StoredDPValues.size(), 0u); + EXPECT_EQ(RetInst->DbgMarker->StoredDbgRecords.size(), 0u); // Try cloning one single DPValue. auto DIIt = std::next(FirstInst->DbgMarker->getDbgRecordRange().begin()); RetInst->DbgMarker->cloneDebugInfoFrom(FirstInst->DbgMarker, DIIt, false); - EXPECT_EQ(RetInst->DbgMarker->StoredDPValues.size(), 1u); + EXPECT_EQ(RetInst->DbgMarker->StoredDbgRecords.size(), 1u); // The second DPValue should have been cloned; it should have the same values // as DPV1. - EXPECT_EQ(cast(RetInst->DbgMarker->StoredDPValues.begin()) + EXPECT_EQ(cast(RetInst->DbgMarker->StoredDbgRecords.begin()) ->getRawLocation(), DPV1->getRawLocation()); - // We should be able to drop individual DPValues. + // We should be able to drop individual DbgRecords. RetInst->DbgMarker->dropOneDbgRecord( - &*RetInst->DbgMarker->StoredDPValues.begin()); + &*RetInst->DbgMarker->StoredDbgRecords.begin()); // "Aborb" a DPMarker: this means pretend that the instruction it's attached // to is disappearing so it needs to be transferred into "this" marker. RetInst->DbgMarker->absorbDebugValues(*FirstInst->DbgMarker, true); - EXPECT_EQ(RetInst->DbgMarker->StoredDPValues.size(), 2u); + EXPECT_EQ(RetInst->DbgMarker->StoredDbgRecords.size(), 2u); // Should be the DPV1 and DPV2 objects. ItCount = 0; for (DbgRecord &Item : RetInst->DbgMarker->getDbgRecordRange()) { @@ -1009,7 +1009,7 @@ TEST(MetadataTest, ConvertDbgToDPValue) { } // Finally -- there are two DPValues left over. If we remove evrything in the - // basic block, then they should sink down into the "TrailingDPValues" + // basic block, then they should sink down into the "TrailingDbgRecords" // container for dangling debug-info. Future facilities will restore them // back when a terminator is inserted. FirstInst->DbgMarker->removeMarker(); @@ -1019,7 +1019,7 @@ TEST(MetadataTest, ConvertDbgToDPValue) { DPMarker *EndMarker = ExitBlock->getTrailingDbgRecords(); ASSERT_NE(EndMarker, nullptr); - EXPECT_EQ(EndMarker->StoredDPValues.size(), 2u); + EXPECT_EQ(EndMarker->StoredDbgRecords.size(), 2u); // Test again that it's those two DPValues, DPV1 and DPV2. ItCount = 0; for (DbgRecord &Item : EndMarker->getDbgRecordRange()) { @@ -1115,14 +1115,14 @@ TEST(MetadataTest, DPValueConversionRoutines) { EXPECT_EQ(FirstInst, FirstInst->DbgMarker->MarkedInstr); EXPECT_EQ(SecondInst, SecondInst->DbgMarker->MarkedInstr); - EXPECT_EQ(FirstInst->DbgMarker->StoredDPValues.size(), 1u); + EXPECT_EQ(FirstInst->DbgMarker->StoredDbgRecords.size(), 1u); DPValue *DPV1 = cast(&*FirstInst->DbgMarker->getDbgRecordRange().begin()); EXPECT_EQ(DPV1->getMarker(), FirstInst->DbgMarker); // Should point at %a, an argument. EXPECT_TRUE(isa(DPV1->getVariableLocationOp(0))); - EXPECT_EQ(SecondInst->DbgMarker->StoredDPValues.size(), 1u); + EXPECT_EQ(SecondInst->DbgMarker->StoredDbgRecords.size(), 1u); DPValue *DPV2 = cast(&*SecondInst->DbgMarker->getDbgRecordRange().begin()); EXPECT_EQ(DPV2->getMarker(), SecondInst->DbgMarker); @@ -1135,7 +1135,7 @@ TEST(MetadataTest, DPValueConversionRoutines) { EXPECT_TRUE(BB2->IsNewDbgInfoFormat); for (auto &Inst : *BB2) // Either there should be no marker, or it should be empty. - EXPECT_TRUE(!Inst.DbgMarker || Inst.DbgMarker->StoredDPValues.empty()); + EXPECT_TRUE(!Inst.DbgMarker || Inst.DbgMarker->StoredDbgRecords.empty()); // Validating the first block should continue to not be a problem, Error = verifyModule(*M, &errs(), &BrokenDebugInfo); diff --git a/llvm/unittests/Transforms/Utils/DebugifyTest.cpp b/llvm/unittests/Transforms/Utils/DebugifyTest.cpp index 89fa1334b427..0b00734fc4d7 100644 --- a/llvm/unittests/Transforms/Utils/DebugifyTest.cpp +++ b/llvm/unittests/Transforms/Utils/DebugifyTest.cpp @@ -60,7 +60,7 @@ struct DebugValueDrop : public FunctionPass { for (Instruction &I : BB) { if (auto *DVI = dyn_cast(&I)) Dbgs.push_back(DVI); - // If there are any non-intrinsic records (DPValues), drop those too. + // If there are any non-intrinsic records (DbgRecords), drop those too. I.dropDbgRecords(); } } -- GitLab From e703c735df1257022bcb6ca9de857e20e671392f Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Wed, 13 Mar 2024 09:49:30 -0700 Subject: [PATCH 0153/1407] AMDGPU: Remove incorrect uses of SubtaretPredicate around DS_Reals (#85001) SubtargetPredicate is copied from DS_Pseudo to DS_Real. We should not use another SubtargetPredicate assignment around DS_Real, because doing so will override the predicate from DS_Pseudo. For example, for DS_ADD_RTN_F64, SubtargetPredicate was set to HasLdsAtomicAddF64 in Pseudo. And it will be overridden to isGFX90APlus if we assign isGFX90APlus to SubtargetPredicate in Real definition. --- llvm/lib/Target/AMDGPU/DSInstructions.td | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index 87ace01a6d0e..e944dde15990 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -1735,14 +1735,12 @@ def DS_WRITE_B128_vi : DS_Real_vi<0xdf, DS_WRITE_B128>; def DS_READ_B96_vi : DS_Real_vi<0xfe, DS_READ_B96>; def DS_READ_B128_vi : DS_Real_vi<0xff, DS_READ_B128>; -let SubtargetPredicate = isGFX90APlus in { - def DS_ADD_F64_vi : DS_Real_vi<0x5c, DS_ADD_F64>; - def DS_ADD_RTN_F64_vi : DS_Real_vi<0x7c, DS_ADD_RTN_F64>; -} // End SubtargetPredicate = isGFX90APlus - -let SubtargetPredicate = isGFX940Plus in { - def DS_PK_ADD_F16_vi : DS_Real_vi<0x17, DS_PK_ADD_F16>; - def DS_PK_ADD_RTN_F16_vi : DS_Real_vi<0xb7, DS_PK_ADD_RTN_F16>; - def DS_PK_ADD_BF16_vi : DS_Real_vi<0x18, DS_PK_ADD_BF16>; - def DS_PK_ADD_RTN_BF16_vi : DS_Real_vi<0xb8, DS_PK_ADD_RTN_BF16>; -} // End SubtargetPredicate = isGFX940Plus +// GFX90A+. +def DS_ADD_F64_vi : DS_Real_vi<0x5c, DS_ADD_F64>; +def DS_ADD_RTN_F64_vi : DS_Real_vi<0x7c, DS_ADD_RTN_F64>; + +// GFX940+. +def DS_PK_ADD_F16_vi : DS_Real_vi<0x17, DS_PK_ADD_F16>; +def DS_PK_ADD_RTN_F16_vi : DS_Real_vi<0xb7, DS_PK_ADD_RTN_F16>; +def DS_PK_ADD_BF16_vi : DS_Real_vi<0x18, DS_PK_ADD_BF16>; +def DS_PK_ADD_RTN_BF16_vi : DS_Real_vi<0xb8, DS_PK_ADD_RTN_BF16>; -- GitLab From 3b2694853e361d2221ec8071f815d9f5eef35b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= Date: Wed, 13 Mar 2024 16:53:26 +0000 Subject: [PATCH 0154/1407] [mlir][nfc] Update Linalg matmul -> Vector OP test (#81416) Updates "transform-op-matmul-to-outerproduct.mlir". Summary: * refines TD sequence so that it's easier to reason about the compilation pipeline (e.g. `transform.structured.vectorize_children_and_apply_patterns` is replaced with`transform.structured.vectorize `), * new input dims to be able to distinguish parallel from reduction dims, * updates LIT variable names (makes the output easier to follow), * removes "noise" from the expected LIT output (e.g. types). These Linalg -> Vector tests using Transform Dialect are great reference points for constructing lowering pipelines. This simplification + clean-up will hopefully make it easier to follow. --- .../transform-op-matmul-to-outerproduct.mlir | 67 +++++++++++-------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/mlir/test/Dialect/Linalg/transform-op-matmul-to-outerproduct.mlir b/mlir/test/Dialect/Linalg/transform-op-matmul-to-outerproduct.mlir index ee66073a9a41..a1a0c413da0c 100644 --- a/mlir/test/Dialect/Linalg/transform-op-matmul-to-outerproduct.mlir +++ b/mlir/test/Dialect/Linalg/transform-op-matmul-to-outerproduct.mlir @@ -1,38 +1,51 @@ // RUN: mlir-opt %s -transform-interpreter | FileCheck %s -func.func @outerproduct_matmul(%A: memref<3x3xf32>, %B: memref<3x3xf32>, %C: memref<3x3xf32>) { - linalg.matmul ins(%A, %B: memref<3x3xf32>, memref<3x3xf32>) +func.func @matmul_to_outerproduct(%A: memref<3x4xf32>, %B: memref<4x3xf32>, %C: memref<3x3xf32>) { + linalg.matmul ins(%A, %B: memref<3x4xf32>, memref<4x3xf32>) outs(%C: memref<3x3xf32>) return } -// CHECK-LABEL: func.func @outerproduct_matmul( -// CHECK-SAME: %[[VAL_0:.*]]: memref<3x3xf32>, %[[VAL_1:.*]]: memref<3x3xf32>, %[[VAL_2:.*]]: memref<3x3xf32>) { -// CHECK: %[[VAL_3:.*]] = arith.constant 0 : index -// CHECK: %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[VAL_5:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_3]], %[[VAL_3]]], %[[VAL_4]] {in_bounds = [true, true]} : memref<3x3xf32>, vector<3x3xf32> -// CHECK: %[[VAL_6:.*]] = vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_3]], %[[VAL_3]]], %[[VAL_4]] {in_bounds = [true, true]} : memref<3x3xf32>, vector<3x3xf32> -// CHECK: %[[VAL_7:.*]] = vector.transfer_read %[[VAL_2]]{{\[}}%[[VAL_3]], %[[VAL_3]]], %[[VAL_4]] {in_bounds = [true, true]} : memref<3x3xf32>, vector<3x3xf32> -// CHECK: %[[VAL_8:.*]] = vector.transpose %[[VAL_5]], [1, 0] : vector<3x3xf32> to vector<3x3xf32> -// CHECK: %[[VAL_9:.*]] = vector.extract %[[VAL_8]][0] : vector<3xf32> from vector<3x3xf32> -// CHECK: %[[VAL_10:.*]] = vector.extract %[[VAL_6]][0] : vector<3xf32> from vector<3x3xf32> -// CHECK: %[[VAL_11:.*]] = vector.outerproduct %[[VAL_9]], %[[VAL_10]], %[[VAL_7]] {kind = #vector.kind} : vector<3xf32>, vector<3xf32> -// CHECK: %[[VAL_12:.*]] = vector.extract %[[VAL_8]][1] : vector<3xf32> from vector<3x3xf32> -// CHECK: %[[VAL_13:.*]] = vector.extract %[[VAL_6]][1] : vector<3xf32> from vector<3x3xf32> -// CHECK: %[[VAL_14:.*]] = vector.outerproduct %[[VAL_12]], %[[VAL_13]], %[[VAL_11]] {kind = #vector.kind} : vector<3xf32>, vector<3xf32> -// CHECK: %[[VAL_15:.*]] = vector.extract %[[VAL_8]][2] : vector<3xf32> from vector<3x3xf32> -// CHECK: %[[VAL_16:.*]] = vector.extract %[[VAL_6]][2] : vector<3xf32> from vector<3x3xf32> -// CHECK: %[[VAL_17:.*]] = vector.outerproduct %[[VAL_15]], %[[VAL_16]], %[[VAL_14]] {kind = #vector.kind} : vector<3xf32>, vector<3xf32> -// CHECK: vector.transfer_write %[[VAL_17]], %[[VAL_2]]{{\[}}%[[VAL_3]], %[[VAL_3]]] {in_bounds = [true, true]} : vector<3x3xf32>, memref<3x3xf32> -// CHECK: return -// CHECK: } +// CHECK-LABEL: func.func @matmul_to_outerproduct( +// CHECK-SAME: %[[A:.*]]: memref<3x4xf32>, +// CHECK-SAME: %[[B:.*]]: memref<4x3xf32>, +// CHECK-SAME: %[[C:.*]]: memref<3x3xf32>) { +// CHECK: %[[VEC_A:.*]] = vector.transfer_read %[[A]] +// CHECK: %[[VEC_B:.*]] = vector.transfer_read %[[B]] +// CHECK: %[[VEC_C:.*]] = vector.transfer_read %[[C]] +// CHECK: %[[VEC_A_T:.*]] = vector.transpose %[[VEC_A]], [1, 0] : vector<3x4xf32> to vector<4x3xf32> +// CHECK: %[[A0:.*]] = vector.extract %[[VEC_A_T]][0] : vector<3xf32> from vector<4x3xf32> +// CHECK: %[[B0:.*]] = vector.extract %[[VEC_B]][0] : vector<3xf32> from vector<4x3xf32> +// CHECK: %[[OP_0:.*]] = vector.outerproduct %[[A0]], %[[B0]], %[[VEC_C]] +// CHECK: %[[A1:.*]] = vector.extract %[[VEC_A_T]][1] : vector<3xf32> from vector<4x3xf32> +// CHECK: %[[B1:.*]] = vector.extract %[[VEC_B]][1] : vector<3xf32> from vector<4x3xf32> +// CHECK: %[[OP_1:.*]] = vector.outerproduct %[[A1]], %[[B1]], %[[OP_0]] +// CHECK: %[[A_2:.*]] = vector.extract %[[VEC_A_T]][2] : vector<3xf32> from vector<4x3xf32> +// CHECK: %[[B_2:.*]] = vector.extract %[[VEC_B]][2] : vector<3xf32> from vector<4x3xf32> +// CHECK: %[[OP_2:.*]] = vector.outerproduct %[[A_2]], %[[B_2]], %[[OP_1]] +// CHECK: %[[A_3:.*]] = vector.extract %[[VEC_A_T]][3] : vector<3xf32> from vector<4x3xf32> +// CHECK: %[[B_3:.*]] = vector.extract %[[VEC_B]][3] : vector<3xf32> from vector<4x3xf32> +// CHECK: %[[RES:.*]] = vector.outerproduct %[[A_3]], %[[B_3]], %[[OP_2]] +// CHECK: vector.transfer_write %[[RES]], %[[C]]{{.*}} : vector<3x3xf32>, memref<3x3xf32> module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { - %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op - %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op - %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op - transform.apply_patterns to %2 { + transform.named_sequence @__transform_main(%module: !transform.any_op {transform.readonly}) { + %func = transform.structured.match ops{["func.func"]} in %module : (!transform.any_op) -> !transform.any_op + + // Vectorize: linalg.matmul -> vector.multi_reduction + %matmul = transform.structured.match ops{["linalg.matmul"]} in %func : (!transform.any_op) -> !transform.any_op + transform.structured.vectorize %matmul : !transform.any_op + + // vector.multi_reduction --> vector.contract + transform.apply_patterns to %func { + transform.apply_patterns.vector.reduction_to_contract + // Reduce the rank of xfer ops. This transform vector.contract to be more + // more matmul-like and to enable the lowering to outer product Ops. + transform.apply_patterns.vector.transfer_permutation_patterns + } : !transform.any_op + + // vector.contract --> vector.outerproduct + transform.apply_patterns to %func { transform.apply_patterns.vector.lower_contraction lowering_strategy = "outerproduct" } : !transform.any_op transform.yield -- GitLab From 79cd2c0bb9acb4685094d6b3bf21c758aa51d3df Mon Sep 17 00:00:00 2001 From: "Nadeem, Usman" Date: Wed, 13 Mar 2024 09:54:30 -0700 Subject: [PATCH 0155/1407] [AArch64] Fix tests after PR82457 Change-Id: I44a7e4a10af750b3339d6564c6ce6c2e5c17778e --- llvm/test/CodeGen/AArch64/bitcast.ll | 4 ++-- llvm/test/CodeGen/AArch64/shufflevector.ll | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/bitcast.ll b/llvm/test/CodeGen/AArch64/bitcast.ll index bccfdb93d786..9ebd570e687a 100644 --- a/llvm/test/CodeGen/AArch64/bitcast.ll +++ b/llvm/test/CodeGen/AArch64/bitcast.ll @@ -59,7 +59,7 @@ define i32 @bitcast_v4i8_i32(<4 x i8> %a, <4 x i8> %b){ ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: add v0.4h, v0.4h, v1.4h -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: fmov w0, s0 ; CHECK-NEXT: add sp, sp, #16 ; CHECK-NEXT: ret @@ -388,7 +388,7 @@ define <2 x i16> @bitcast_v4i8_v2i16(<4 x i8> %a, <4 x i8> %b){ ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: add v0.4h, v0.4h, v1.4h ; CHECK-NEXT: add x8, sp, #12 -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: str s0, [sp, #12] ; CHECK-NEXT: ld1 { v0.h }[0], [x8] ; CHECK-NEXT: orr x8, x8, #0x2 diff --git a/llvm/test/CodeGen/AArch64/shufflevector.ll b/llvm/test/CodeGen/AArch64/shufflevector.ll index d79f3ae11167..b1131f287fe9 100644 --- a/llvm/test/CodeGen/AArch64/shufflevector.ll +++ b/llvm/test/CodeGen/AArch64/shufflevector.ll @@ -202,7 +202,7 @@ define i32 @shufflevector_v4i8(<4 x i8> %a, <4 x i8> %b){ ; CHECK-SD-NEXT: ext v0.8b, v1.8b, v0.8b, #6 ; CHECK-SD-NEXT: zip1 v1.4h, v1.4h, v0.4h ; CHECK-SD-NEXT: ext v0.8b, v0.8b, v1.8b, #4 -; CHECK-SD-NEXT: xtn v0.8b, v0.8h +; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-SD-NEXT: fmov w0, s0 ; CHECK-SD-NEXT: add sp, sp, #16 ; CHECK-SD-NEXT: ret @@ -390,7 +390,7 @@ define i32 @shufflevector_v4i8_zeroes(<4 x i8> %a, <4 x i8> %b){ ; CHECK-SD-NEXT: .cfi_def_cfa_offset 16 ; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-SD-NEXT: dup v0.4h, v0.h[0] -; CHECK-SD-NEXT: xtn v0.8b, v0.8h +; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-SD-NEXT: fmov w0, s0 ; CHECK-SD-NEXT: add sp, sp, #16 ; CHECK-SD-NEXT: ret -- GitLab From 122d368b2b120ff233e66658862b90f185f65c6e Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 13 Mar 2024 10:13:21 -0700 Subject: [PATCH 0156/1407] [llvm-objcopy] --[de]compress-debug-sections: don't compress SHF_ALLOC sections, only decompress .debug sections Simplify --[de]compress-debug-sections to make it easier to add custom section [de]compression. Change the following two behaviors to match GNU objcopy. * --compress-debug-sections compresses SHF_ALLOC sections while GNU doesn't. * --decompress-debug-sections decompresses non-debug sections while GNU doesn't. Pull Request: https://github.com/llvm/llvm-project/pull/84885 --- llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp | 63 +++++++------------ llvm/lib/ObjCopy/ELF/ELFObject.h | 1 + .../ELF/Inputs/compress-debug-sections.yaml | 4 ++ .../ELF/compress-debug-sections-zlib.test | 2 + .../ELF/compress-debug-sections-zstd.test | 2 + .../llvm-objcopy/ELF/decompress-sections.test | 36 +++++++++++ 6 files changed, 68 insertions(+), 40 deletions(-) create mode 100644 llvm/test/tools/llvm-objcopy/ELF/decompress-sections.test diff --git a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp index f52bcb74938d..e4d6e02f3aa6 100644 --- a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp +++ b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp @@ -214,33 +214,32 @@ static Error dumpSectionToFile(StringRef SecName, StringRef Filename, SecName.str().c_str()); } -static bool isCompressable(const SectionBase &Sec) { - return !(Sec.Flags & ELF::SHF_COMPRESSED) && - StringRef(Sec.Name).starts_with(".debug"); -} - -static Error replaceDebugSections( - Object &Obj, function_ref ShouldReplace, - function_ref(const SectionBase *)> AddSection) { +Error Object::compressOrDecompressSections(const CommonConfig &Config) { // Build a list of the debug sections we are going to replace. // We can't call `AddSection` while iterating over sections, // because it would mutate the sections array. - SmallVector ToReplace; - for (auto &Sec : Obj.sections()) - if (ShouldReplace(Sec)) - ToReplace.push_back(&Sec); - - // Build a mapping from original section to a new one. - DenseMap FromTo; - for (SectionBase *S : ToReplace) { - Expected NewSection = AddSection(S); - if (!NewSection) - return NewSection.takeError(); - - FromTo[S] = *NewSection; + SmallVector>, 0> + ToReplace; + for (SectionBase &Sec : sections()) { + if ((Sec.Flags & SHF_ALLOC) || !StringRef(Sec.Name).starts_with(".debug")) + continue; + if (auto *CS = dyn_cast(&Sec)) { + if (Config.DecompressDebugSections) { + ToReplace.emplace_back( + &Sec, [=] { return &addSection(*CS); }); + } + } else if (Config.CompressionType != DebugCompressionType::None) { + ToReplace.emplace_back(&Sec, [&, S = &Sec] { + return &addSection( + CompressedSection(*S, Config.CompressionType, Is64Bits)); + }); + } } - return Obj.replaceSections(FromTo); + DenseMap FromTo; + for (auto [S, Func] : ToReplace) + FromTo[S] = Func(); + return replaceSections(FromTo); } static bool isAArch64MappingSymbol(const Symbol &Sym) { @@ -534,24 +533,8 @@ static Error replaceAndRemoveSections(const CommonConfig &Config, if (Error E = Obj.removeSections(ELFConfig.AllowBrokenLinks, RemovePred)) return E; - if (Config.CompressionType != DebugCompressionType::None) { - if (Error Err = replaceDebugSections( - Obj, isCompressable, - [&Config, &Obj](const SectionBase *S) -> Expected { - return &Obj.addSection( - CompressedSection(*S, Config.CompressionType, Obj.Is64Bits)); - })) - return Err; - } else if (Config.DecompressDebugSections) { - if (Error Err = replaceDebugSections( - Obj, - [](const SectionBase &S) { return isa(&S); }, - [&Obj](const SectionBase *S) { - const CompressedSection *CS = cast(S); - return &Obj.addSection(*CS); - })) - return Err; - } + if (Error E = Obj.compressOrDecompressSections(Config)) + return E; return Error::success(); } diff --git a/llvm/lib/ObjCopy/ELF/ELFObject.h b/llvm/lib/ObjCopy/ELF/ELFObject.h index 7a2e20d82d11..f72c109b6009 100644 --- a/llvm/lib/ObjCopy/ELF/ELFObject.h +++ b/llvm/lib/ObjCopy/ELF/ELFObject.h @@ -1210,6 +1210,7 @@ public: Error removeSections(bool AllowBrokenLinks, std::function ToRemove); + Error compressOrDecompressSections(const CommonConfig &Config); Error replaceSections(const DenseMap &FromTo); Error removeSymbols(function_ref ToRemove); template T &addSection(Ts &&...Args) { diff --git a/llvm/test/tools/llvm-objcopy/ELF/Inputs/compress-debug-sections.yaml b/llvm/test/tools/llvm-objcopy/ELF/Inputs/compress-debug-sections.yaml index 67d8435fa486..e2dfee9163a2 100644 --- a/llvm/test/tools/llvm-objcopy/ELF/Inputs/compress-debug-sections.yaml +++ b/llvm/test/tools/llvm-objcopy/ELF/Inputs/compress-debug-sections.yaml @@ -43,6 +43,10 @@ Sections: Type: SHT_PROGBITS Flags: [ SHF_GROUP ] Content: '00' + - Name: .debug_alloc + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Content: 000102030405060708090a0b0c0d0e0f000102030405060708090a0b0c0d0e0f000102030405060708090a0b0c0d0e0f000102030405060708090a0b0c0d0e0f Symbols: - Type: STT_SECTION Section: .debug_foo diff --git a/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test b/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test index e1ebeed8d4fc..056ae84ce491 100644 --- a/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test +++ b/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test @@ -12,8 +12,10 @@ # CHECK: Name Type Address Off Size ES Flg Lk Inf Al # COMPRESSED: .debug_foo PROGBITS 0000000000000000 000040 {{.*}} 00 C 0 0 8 # COMPRESSED-NEXT: .notdebug_foo PROGBITS 0000000000000000 {{.*}} 000008 00 0 0 0 +# COMPRESSED: .debug_alloc PROGBITS 0000000000000000 {{.*}} 000040 00 A 0 0 0 # UNCOMPRESSED: .debug_foo PROGBITS 0000000000000000 000040 000008 00 0 0 0 # UNCOMPRESSED-NEXT: .notdebug_foo PROGBITS 0000000000000000 {{.*}} 000008 00 0 0 0 +# UNCOMPRESSED: .debug_alloc PROGBITS 0000000000000000 {{.*}} 000040 00 A 0 0 0 ## Relocations do not change. # CHECK: Relocation section '.rela.debug_foo' at offset {{.*}} contains 2 entries: diff --git a/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd.test b/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd.test index d763131c4067..bde1c2f311d0 100644 --- a/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd.test +++ b/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd.test @@ -12,8 +12,10 @@ # CHECK: Name Type Address Off Size ES Flg Lk Inf Al # COMPRESSED: .debug_foo PROGBITS 0000000000000000 000040 {{.*}} 00 C 0 0 8 # COMPRESSED-NEXT: .notdebug_foo PROGBITS 0000000000000000 {{.*}} 000008 00 0 0 0 +# COMPRESSED: .debug_alloc PROGBITS 0000000000000000 {{.*}} 000040 00 A 0 0 0 # DECOMPRESSED: .debug_foo PROGBITS 0000000000000000 000040 000008 00 0 0 0 # DECOMPRESSED-NEXT: .notdebug_foo PROGBITS 0000000000000000 {{.*}} 000008 00 0 0 0 +# DECOMPRESSED: .debug_alloc PROGBITS 0000000000000000 {{.*}} 000040 00 A 0 0 0 ## Relocations do not change. # CHECK: Relocation section '.rela.debug_foo' at offset {{.*}} contains 2 entries: diff --git a/llvm/test/tools/llvm-objcopy/ELF/decompress-sections.test b/llvm/test/tools/llvm-objcopy/ELF/decompress-sections.test new file mode 100644 index 000000000000..4258ddbe66a3 --- /dev/null +++ b/llvm/test/tools/llvm-objcopy/ELF/decompress-sections.test @@ -0,0 +1,36 @@ +# REQUIRES: zlib +## Test decompression for different sections. + +# RUN: yaml2obj %s -o %t +# RUN: llvm-objcopy --decompress-debug-sections %t %t.de +# RUN: llvm-readelf -S %t.de | FileCheck %s + +# CHECK: Name Type Address Off Size ES Flg Lk Inf Al +# CHECK: .debug_alloc PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00 AC 0 0 0 +# CHECK-NEXT: .debug_nonalloc PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00 0 0 1 +# CHECK-NEXT: .debugx PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00 0 0 1 +# CHECK-NEXT: nodebug PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00 C 0 0 0 + +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_REL + Machine: EM_X86_64 +Sections: + - Name: .debug_alloc + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_COMPRESSED ] + Content: 010000000000000040000000000000000100000000000000789cd36280002d3269002f800151 + - Name: .debug_nonalloc + Type: SHT_PROGBITS + Flags: [ SHF_COMPRESSED ] + Content: 010000000000000040000000000000000100000000000000789cd36280002d3269002f800151 + - Name: .debugx + Type: SHT_PROGBITS + Flags: [ SHF_COMPRESSED ] + Content: 010000000000000040000000000000000100000000000000789cd36280002d3269002f800151 + - Name: nodebug + Type: SHT_PROGBITS + Flags: [ SHF_COMPRESSED ] + Content: 010000000000000040000000000000000100000000000000789cd36280002d3269002f800151 -- GitLab From 35f5caea5115d7dabf0c1a92c8627069d6dbd556 Mon Sep 17 00:00:00 2001 From: Chen Cheng <110446443+ChengChen002@users.noreply.github.com> Date: Thu, 14 Mar 2024 01:16:42 +0800 Subject: [PATCH 0157/1407] [NFC] Corrected data type (#84880) On windows, "&Method.first" is of type "unsigned long long *", and a type conversion error occurs. --- llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp index d346214d3ae2..57ac991ee37f 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp @@ -87,7 +87,7 @@ static void registerJITLoaderVTuneUnregisterImpl( for (auto &Method : UM) { JITEventWrapper::Wrapper->iJIT_NotifyEvent( iJVM_EVENT_TYPE_METHOD_UNLOAD_START, - const_cast(&Method.first)); + const_cast(&Method.first)); } } -- GitLab From 13ccaf9b9d4400bb128b35ff4ac733e4afc3ad1c Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 13 Mar 2024 10:12:16 -0700 Subject: [PATCH 0158/1407] Revert "Reapply "[analyzer] Accept C library functions from the `std` namespace"" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit e48d5a838f69e0a8e0ae95a8aed1a8809f45465a. Fails to build on x86-64 w/gcc version 11.4.0 (Ubuntu 11.4.0-1ubuntu1~22.04) with the following message: ../llvm-project/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp:41:28: error: declaration of ‘std::unique_ptr IsCLibraryFunctionTest::ASTUnit’ changes meaning of ‘ASTUnit’ [-fpermissive] 41 | std::unique_ptr ASTUnit; | ^~~~~~~ In file included from ../llvm-project/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp:4: ../llvm-project/clang/include/clang/Frontend/ASTUnit.h:89:7: note: ‘ASTUnit’ declared here as ‘class clang::ASTUnit’ 89 | class ASTUnit { | ^~~~~~~ --- .../Core/PathSensitive/CallDescription.h | 8 +- .../StaticAnalyzer/Core/CheckerContext.cpp | 8 +- clang/unittests/StaticAnalyzer/CMakeLists.txt | 1 - .../StaticAnalyzer/IsCLibraryFunctionTest.cpp | 84 ------------------- .../clang/unittests/StaticAnalyzer/BUILD.gn | 1 - 5 files changed, 9 insertions(+), 93 deletions(-) delete mode 100644 clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h index b4e1636130ca..3432d2648633 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h @@ -41,8 +41,12 @@ public: /// - We also accept calls where the number of arguments or parameters is /// greater than the specified value. /// For the exact heuristics, see CheckerContext::isCLibraryFunction(). - /// (This mode only matches functions that are declared either directly - /// within a TU or in the namespace `std`.) + /// Note that functions whose declaration context is not a TU (e.g. + /// methods, functions in namespaces) are not accepted as C library + /// functions. + /// FIXME: If I understand it correctly, this discards calls where C++ code + /// refers a C library function through the namespace `std::` via headers + /// like . CLibrary, /// Matches "simple" functions that are not methods. (Static methods are diff --git a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp index 1a9bff529e9b..d6d4cec9dd3d 100644 --- a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp +++ b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp @@ -87,11 +87,9 @@ bool CheckerContext::isCLibraryFunction(const FunctionDecl *FD, if (!II) return false; - // C library functions are either declared directly within a TU (the common - // case) or they are accessed through the namespace `std` (when they are used - // in C++ via headers like ). - const DeclContext *DC = FD->getDeclContext()->getRedeclContext(); - if (!(DC->isTranslationUnit() || DC->isStdNamespace())) + // Look through 'extern "C"' and anything similar invented in the future. + // If this function is not in TU directly, it is not a C library function. + if (!FD->getDeclContext()->getRedeclContext()->isTranslationUnit()) return false; // If this function is not externally visible, it is not a C library function. diff --git a/clang/unittests/StaticAnalyzer/CMakeLists.txt b/clang/unittests/StaticAnalyzer/CMakeLists.txt index db56e77331b8..775f0f8486b8 100644 --- a/clang/unittests/StaticAnalyzer/CMakeLists.txt +++ b/clang/unittests/StaticAnalyzer/CMakeLists.txt @@ -11,7 +11,6 @@ add_clang_unittest(StaticAnalysisTests CallEventTest.cpp ConflictingEvalCallsTest.cpp FalsePositiveRefutationBRVisitorTest.cpp - IsCLibraryFunctionTest.cpp NoStateChangeFuncVisitorTest.cpp ParamRegionTest.cpp RangeSetTest.cpp diff --git a/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp b/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp deleted file mode 100644 index 31ff13f428da..000000000000 --- a/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include "clang/ASTMatchers/ASTMatchFinder.h" -#include "clang/ASTMatchers/ASTMatchers.h" -#include "clang/Analysis/AnalysisDeclContext.h" -#include "clang/Frontend/ASTUnit.h" -#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" -#include "clang/Tooling/Tooling.h" -#include "gtest/gtest.h" - -#include - -using namespace clang; -using namespace ento; -using namespace ast_matchers; - -class IsCLibraryFunctionTest : public testing::Test { -public: - const FunctionDecl *getFunctionDecl() const { return Result; } - - testing::AssertionResult buildAST(StringRef Code) { - ASTUnit = tooling::buildASTFromCode(Code); - if (!ASTUnit) - return testing::AssertionFailure() << "AST construction failed"; - - ASTContext &Context = ASTUnit->getASTContext(); - if (Context.getDiagnostics().hasErrorOccurred()) - return testing::AssertionFailure() << "Compilation error"; - - auto Matches = ast_matchers::match(functionDecl().bind("fn"), Context); - if (Matches.empty()) - return testing::AssertionFailure() << "No function declaration found"; - - if (Matches.size() > 1) - return testing::AssertionFailure() - << "Multiple function declarations found"; - - Result = Matches[0].getNodeAs("fn"); - return testing::AssertionSuccess(); - } - -private: - std::unique_ptr ASTUnit; - const FunctionDecl *Result = nullptr; -}; - -TEST_F(IsCLibraryFunctionTest, AcceptsGlobal) { - ASSERT_TRUE(buildAST(R"cpp(void fun();)cpp")); - EXPECT_TRUE(CheckerContext::isCLibraryFunction(getFunctionDecl())); -} - -TEST_F(IsCLibraryFunctionTest, AcceptsExternCGlobal) { - ASSERT_TRUE(buildAST(R"cpp(extern "C" { void fun(); })cpp")); - EXPECT_TRUE(CheckerContext::isCLibraryFunction(getFunctionDecl())); -} - -TEST_F(IsCLibraryFunctionTest, RejectsNoInlineNoExternalLinkage) { - // Functions that are neither inlined nor externally visible cannot be C library functions. - ASSERT_TRUE(buildAST(R"cpp(static void fun();)cpp")); - EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl())); -} - -TEST_F(IsCLibraryFunctionTest, RejectsAnonymousNamespace) { - ASSERT_TRUE(buildAST(R"cpp(namespace { void fun(); })cpp")); - EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl())); -} - -TEST_F(IsCLibraryFunctionTest, AcceptsStdNamespace) { - ASSERT_TRUE(buildAST(R"cpp(namespace std { void fun(); })cpp")); - EXPECT_TRUE(CheckerContext::isCLibraryFunction(getFunctionDecl())); -} - -TEST_F(IsCLibraryFunctionTest, RejectsOtherNamespaces) { - ASSERT_TRUE(buildAST(R"cpp(namespace stdx { void fun(); })cpp")); - EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl())); -} - -TEST_F(IsCLibraryFunctionTest, RejectsClassStatic) { - ASSERT_TRUE(buildAST(R"cpp(class A { static void fun(); };)cpp")); - EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl())); -} - -TEST_F(IsCLibraryFunctionTest, RejectsClassMember) { - ASSERT_TRUE(buildAST(R"cpp(class A { void fun(); };)cpp")); - EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl())); -} diff --git a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn index 9c240cff1816..01c2b6ced336 100644 --- a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn @@ -19,7 +19,6 @@ unittest("StaticAnalysisTests") { "CallEventTest.cpp", "ConflictingEvalCallsTest.cpp", "FalsePositiveRefutationBRVisitorTest.cpp", - "IsCLibraryFunctionTest.cpp", "NoStateChangeFuncVisitorTest.cpp", "ParamRegionTest.cpp", "RangeSetTest.cpp", -- GitLab From cd20600767409b183a6d213d56f85f8041a21487 Mon Sep 17 00:00:00 2001 From: Aleksandr Popov <42888396+aleks-tmb@users.noreply.github.com> Date: Wed, 13 Mar 2024 18:30:03 +0100 Subject: [PATCH 0159/1407] [LoopConstrainer] Apply loop gurads to check that loop bounds are safe (#71531) Loop guards that apply to loop SCEV bounds allow IRCE for cases with compound loop bounds such as: if (K > 0 && M > 0) for (i = 0; i < min(K, M); i++) {...} if (K > 0 && M > 0) for (i = min(K, M); i >= 0; i--) {...} Otherwise SCEV couldn't prove that loops have safe bounds in these cases. Co-authored-by: Aleksander Popov --- llvm/lib/Transforms/Utils/LoopConstrainer.cpp | 22 +++-- .../Transforms/IRCE/compound-loop-bound.ll | 85 +++++++++++++++++-- 2 files changed, 90 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Transforms/Utils/LoopConstrainer.cpp b/llvm/lib/Transforms/Utils/LoopConstrainer.cpp index 81545ef37521..d9832eeb0697 100644 --- a/llvm/lib/Transforms/Utils/LoopConstrainer.cpp +++ b/llvm/lib/Transforms/Utils/LoopConstrainer.cpp @@ -42,8 +42,11 @@ static bool isSafeDecreasingBound(const SCEV *Start, const SCEV *BoundSCEV, ICmpInst::Predicate BoundPred = IsSigned ? CmpInst::ICMP_SGT : CmpInst::ICMP_UGT; + auto StartLG = SE.applyLoopGuards(Start, L); + auto BoundLG = SE.applyLoopGuards(BoundSCEV, L); + if (LatchBrExitIdx == 1) - return SE.isLoopEntryGuardedByCond(L, BoundPred, Start, BoundSCEV); + return SE.isLoopEntryGuardedByCond(L, BoundPred, StartLG, BoundLG); assert(LatchBrExitIdx == 0 && "LatchBrExitIdx should be either 0 or 1"); @@ -54,10 +57,10 @@ static bool isSafeDecreasingBound(const SCEV *Start, const SCEV *BoundSCEV, const SCEV *Limit = SE.getMinusSCEV(SE.getConstant(Min), StepPlusOne); const SCEV *MinusOne = - SE.getMinusSCEV(BoundSCEV, SE.getOne(BoundSCEV->getType())); + SE.getMinusSCEV(BoundLG, SE.getOne(BoundLG->getType())); - return SE.isLoopEntryGuardedByCond(L, BoundPred, Start, MinusOne) && - SE.isLoopEntryGuardedByCond(L, BoundPred, BoundSCEV, Limit); + return SE.isLoopEntryGuardedByCond(L, BoundPred, StartLG, MinusOne) && + SE.isLoopEntryGuardedByCond(L, BoundPred, BoundLG, Limit); } /// Given a loop with an increasing induction variable, is it possible to @@ -86,8 +89,11 @@ static bool isSafeIncreasingBound(const SCEV *Start, const SCEV *BoundSCEV, ICmpInst::Predicate BoundPred = IsSigned ? CmpInst::ICMP_SLT : CmpInst::ICMP_ULT; + auto StartLG = SE.applyLoopGuards(Start, L); + auto BoundLG = SE.applyLoopGuards(BoundSCEV, L); + if (LatchBrExitIdx == 1) - return SE.isLoopEntryGuardedByCond(L, BoundPred, Start, BoundSCEV); + return SE.isLoopEntryGuardedByCond(L, BoundPred, StartLG, BoundLG); assert(LatchBrExitIdx == 0 && "LatchBrExitIdx should be 0 or 1"); @@ -97,9 +103,9 @@ static bool isSafeIncreasingBound(const SCEV *Start, const SCEV *BoundSCEV, : APInt::getMaxValue(BitWidth); const SCEV *Limit = SE.getMinusSCEV(SE.getConstant(Max), StepMinusOne); - return (SE.isLoopEntryGuardedByCond(L, BoundPred, Start, - SE.getAddExpr(BoundSCEV, Step)) && - SE.isLoopEntryGuardedByCond(L, BoundPred, BoundSCEV, Limit)); + return (SE.isLoopEntryGuardedByCond(L, BoundPred, StartLG, + SE.getAddExpr(BoundLG, Step)) && + SE.isLoopEntryGuardedByCond(L, BoundPred, BoundLG, Limit)); } /// Returns estimate for max latch taken count of the loop of the narrowest diff --git a/llvm/test/Transforms/IRCE/compound-loop-bound.ll b/llvm/test/Transforms/IRCE/compound-loop-bound.ll index 0930d19e2215..e50d8c6127f4 100644 --- a/llvm/test/Transforms/IRCE/compound-loop-bound.ll +++ b/llvm/test/Transforms/IRCE/compound-loop-bound.ll @@ -16,23 +16,56 @@ define void @incrementing_loop(ptr %arr, ptr %len_ptr, i32 %K, i32 %M) { ; CHECK-NEXT: br i1 [[AND]], label [[PREHEADER:%.*]], label [[EXIT:%.*]] ; CHECK: preheader: ; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[K]], i32 [[M]]) +; CHECK-NEXT: [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[LEN]], i32 [[M]]) +; CHECK-NEXT: [[SMIN2:%.*]] = call i32 @llvm.smin.i32(i32 [[SMIN1]], i32 [[K]]) +; CHECK-NEXT: [[EXIT_MAINLOOP_AT:%.*]] = call i32 @llvm.smax.i32(i32 [[SMIN2]], i32 0) +; CHECK-NEXT: [[TMP0:%.*]] = icmp slt i32 0, [[EXIT_MAINLOOP_AT]] +; CHECK-NEXT: br i1 [[TMP0]], label [[LOOP_PREHEADER:%.*]], label [[MAIN_PSEUDO_EXIT:%.*]] +; CHECK: loop.preheader: ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[IDX:%.*]] = phi i32 [ 0, [[PREHEADER]] ], [ [[IDX_NEXT:%.*]], [[IN_BOUNDS:%.*]] ] -; CHECK-NEXT: [[IDX_NEXT]] = add i32 [[IDX]], 1 +; CHECK-NEXT: [[IDX:%.*]] = phi i32 [ [[IDX_NEXT:%.*]], [[IN_BOUNDS:%.*]] ], [ 0, [[LOOP_PREHEADER]] ] +; CHECK-NEXT: [[IDX_NEXT]] = add nsw i32 [[IDX]], 1 ; CHECK-NEXT: [[GUARD:%.*]] = icmp slt i32 [[IDX]], [[LEN]] -; CHECK-NEXT: br i1 [[GUARD]], label [[IN_BOUNDS]], label [[OUT_OF_BOUNDS:%.*]] +; CHECK-NEXT: br i1 true, label [[IN_BOUNDS]], label [[OUT_OF_BOUNDS_LOOPEXIT3:%.*]] ; CHECK: in.bounds: ; CHECK-NEXT: [[ADDR:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX]] ; CHECK-NEXT: store i32 0, ptr [[ADDR]], align 4 ; CHECK-NEXT: [[NEXT:%.*]] = icmp slt i32 [[IDX_NEXT]], [[SMIN]] -; CHECK-NEXT: br i1 [[NEXT]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 [[IDX_NEXT]], [[EXIT_MAINLOOP_AT]] +; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP]], label [[MAIN_EXIT_SELECTOR:%.*]] +; CHECK: main.exit.selector: +; CHECK-NEXT: [[IDX_NEXT_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT]], [[IN_BOUNDS]] ] +; CHECK-NEXT: [[TMP2:%.*]] = icmp slt i32 [[IDX_NEXT_LCSSA]], [[SMIN]] +; CHECK-NEXT: br i1 [[TMP2]], label [[MAIN_PSEUDO_EXIT]], label [[EXIT_LOOPEXIT:%.*]] +; CHECK: main.pseudo.exit: +; CHECK-NEXT: [[IDX_COPY:%.*]] = phi i32 [ 0, [[PREHEADER]] ], [ [[IDX_NEXT_LCSSA]], [[MAIN_EXIT_SELECTOR]] ] +; CHECK-NEXT: [[INDVAR_END:%.*]] = phi i32 [ 0, [[PREHEADER]] ], [ [[IDX_NEXT_LCSSA]], [[MAIN_EXIT_SELECTOR]] ] +; CHECK-NEXT: br label [[POSTLOOP:%.*]] +; CHECK: out.of.bounds.loopexit: +; CHECK-NEXT: br label [[OUT_OF_BOUNDS:%.*]] +; CHECK: out.of.bounds.loopexit3: +; CHECK-NEXT: br label [[OUT_OF_BOUNDS]] ; CHECK: out.of.bounds: ; CHECK-NEXT: ret void +; CHECK: exit.loopexit.loopexit: +; CHECK-NEXT: br label [[EXIT_LOOPEXIT]] ; CHECK: exit.loopexit: ; CHECK-NEXT: br label [[EXIT]] ; CHECK: exit: ; CHECK-NEXT: ret void +; CHECK: postloop: +; CHECK-NEXT: br label [[LOOP_POSTLOOP:%.*]] +; CHECK: loop.postloop: +; CHECK-NEXT: [[IDX_POSTLOOP:%.*]] = phi i32 [ [[IDX_COPY]], [[POSTLOOP]] ], [ [[IDX_NEXT_POSTLOOP:%.*]], [[IN_BOUNDS_POSTLOOP:%.*]] ] +; CHECK-NEXT: [[IDX_NEXT_POSTLOOP]] = add i32 [[IDX_POSTLOOP]], 1 +; CHECK-NEXT: [[GUARD_POSTLOOP:%.*]] = icmp slt i32 [[IDX_POSTLOOP]], [[LEN]] +; CHECK-NEXT: br i1 [[GUARD_POSTLOOP]], label [[IN_BOUNDS_POSTLOOP]], label [[OUT_OF_BOUNDS_LOOPEXIT:%.*]] +; CHECK: in.bounds.postloop: +; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]] +; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4 +; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], [[SMIN]] +; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP1:![0-9]+]], !loop_constrainer.loop.clone !6 ; entry: %len = load i32, ptr %len_ptr, !range !0 @@ -78,24 +111,58 @@ define void @decrementing_loop(ptr %arr, ptr %len_ptr, i32 %K, i32 %M) { ; CHECK-NEXT: [[AND:%.*]] = and i1 [[CHECK0]], [[CHECK1]] ; CHECK-NEXT: br i1 [[AND]], label [[PREHEADER:%.*]], label [[EXIT:%.*]] ; CHECK: preheader: -; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[K]], i32 [[M]]) +; CHECK-NEXT: [[INDVAR_START:%.*]] = call i32 @llvm.smin.i32(i32 [[K]], i32 [[M]]) +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDVAR_START]], 1 +; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[LEN]], i32 [[TMP0]]) +; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[SMIN]], i32 0) +; CHECK-NEXT: [[EXIT_PRELOOP_AT:%.*]] = add nsw i32 [[SMAX]], -1 +; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i32 [[INDVAR_START]], [[EXIT_PRELOOP_AT]] +; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_PRELOOP_PREHEADER:%.*]], label [[PRELOOP_PSEUDO_EXIT:%.*]] +; CHECK: loop.preloop.preheader: +; CHECK-NEXT: br label [[LOOP_PRELOOP:%.*]] +; CHECK: mainloop: ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[IDX:%.*]] = phi i32 [ [[SMIN]], [[PREHEADER]] ], [ [[IDX_DEC:%.*]], [[IN_BOUNDS:%.*]] ] -; CHECK-NEXT: [[IDX_DEC]] = sub i32 [[IDX]], 1 +; CHECK-NEXT: [[IDX:%.*]] = phi i32 [ [[IDX_PRELOOP_COPY:%.*]], [[MAINLOOP:%.*]] ], [ [[IDX_DEC:%.*]], [[IN_BOUNDS:%.*]] ] +; CHECK-NEXT: [[IDX_DEC]] = sub nsw i32 [[IDX]], 1 ; CHECK-NEXT: [[GUARD:%.*]] = icmp slt i32 [[IDX]], [[LEN]] -; CHECK-NEXT: br i1 [[GUARD]], label [[IN_BOUNDS]], label [[OUT_OF_BOUNDS:%.*]] +; CHECK-NEXT: br i1 true, label [[IN_BOUNDS]], label [[OUT_OF_BOUNDS_LOOPEXIT1:%.*]] ; CHECK: in.bounds: ; CHECK-NEXT: [[ADDR:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX]] ; CHECK-NEXT: store i32 0, ptr [[ADDR]], align 4 ; CHECK-NEXT: [[NEXT:%.*]] = icmp sgt i32 [[IDX_DEC]], -1 -; CHECK-NEXT: br i1 [[NEXT]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]] +; CHECK-NEXT: br i1 [[NEXT]], label [[LOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]] +; CHECK: out.of.bounds.loopexit: +; CHECK-NEXT: br label [[OUT_OF_BOUNDS:%.*]] +; CHECK: out.of.bounds.loopexit1: +; CHECK-NEXT: br label [[OUT_OF_BOUNDS]] ; CHECK: out.of.bounds: ; CHECK-NEXT: ret void +; CHECK: exit.loopexit.loopexit: +; CHECK-NEXT: br label [[EXIT_LOOPEXIT:%.*]] ; CHECK: exit.loopexit: ; CHECK-NEXT: br label [[EXIT]] ; CHECK: exit: ; CHECK-NEXT: ret void +; CHECK: loop.preloop: +; CHECK-NEXT: [[IDX_PRELOOP:%.*]] = phi i32 [ [[IDX_DEC_PRELOOP:%.*]], [[IN_BOUNDS_PRELOOP:%.*]] ], [ [[INDVAR_START]], [[LOOP_PRELOOP_PREHEADER]] ] +; CHECK-NEXT: [[IDX_DEC_PRELOOP]] = sub i32 [[IDX_PRELOOP]], 1 +; CHECK-NEXT: [[GUARD_PRELOOP:%.*]] = icmp slt i32 [[IDX_PRELOOP]], [[LEN]] +; CHECK-NEXT: br i1 [[GUARD_PRELOOP]], label [[IN_BOUNDS_PRELOOP]], label [[OUT_OF_BOUNDS_LOOPEXIT:%.*]] +; CHECK: in.bounds.preloop: +; CHECK-NEXT: [[ADDR_PRELOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_PRELOOP]] +; CHECK-NEXT: store i32 0, ptr [[ADDR_PRELOOP]], align 4 +; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp sgt i32 [[IDX_DEC_PRELOOP]], -1 +; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt i32 [[IDX_DEC_PRELOOP]], [[EXIT_PRELOOP_AT]] +; CHECK-NEXT: br i1 [[TMP2]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP7:![0-9]+]], !loop_constrainer.loop.clone !6 +; CHECK: preloop.exit.selector: +; CHECK-NEXT: [[IDX_DEC_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_DEC_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ] +; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt i32 [[IDX_DEC_PRELOOP_LCSSA]], -1 +; CHECK-NEXT: br i1 [[TMP3]], label [[PRELOOP_PSEUDO_EXIT]], label [[EXIT_LOOPEXIT]] +; CHECK: preloop.pseudo.exit: +; CHECK-NEXT: [[IDX_PRELOOP_COPY]] = phi i32 [ [[INDVAR_START]], [[PREHEADER]] ], [ [[IDX_DEC_PRELOOP_LCSSA]], [[PRELOOP_EXIT_SELECTOR]] ] +; CHECK-NEXT: [[INDVAR_END:%.*]] = phi i32 [ [[INDVAR_START]], [[PREHEADER]] ], [ [[IDX_DEC_PRELOOP_LCSSA]], [[PRELOOP_EXIT_SELECTOR]] ] +; CHECK-NEXT: br label [[MAINLOOP]] ; entry: %len = load i32, ptr %len_ptr, !range !0 -- GitLab From 175b533720956017bb18d1280362f6890ee15b05 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Wed, 13 Mar 2024 10:44:12 -0700 Subject: [PATCH 0160/1407] workflows: Add workaround for lld failures on MacOS (#85021) See #81967 --- .github/workflows/llvm-project-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/llvm-project-tests.yml b/.github/workflows/llvm-project-tests.yml index 43b90193406f..a52dd2db8035 100644 --- a/.github/workflows/llvm-project-tests.yml +++ b/.github/workflows/llvm-project-tests.yml @@ -118,6 +118,11 @@ jobs: else builddir="$(pwd)"/build fi + if [ "${{ runner.os }}" == "macOS" ]; then + # Workaround test failure on some lld tests on MacOS + # https://github.com/llvm/llvm-project/issues/81967 + extra_cmake_args="-DLLVM_DISABLE_ASSEMBLY_FILES=ON" + fi echo "llvm-builddir=$builddir" >> "$GITHUB_OUTPUT" cmake -G Ninja \ -B "$builddir" \ -- GitLab From bd77a26e9a15981114e9802d83047f42631125a2 Mon Sep 17 00:00:00 2001 From: Sirraide Date: Wed, 13 Mar 2024 18:49:44 +0100 Subject: [PATCH 0161/1407] [Clang][Sema] Properly get captured 'this' pointer in lambdas with an explicit object parameter in constant evaluator (#81102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There were some bugs wrt explicit object parameters in lambdas in the constant evaluator: - The code evaluating a `CXXThisExpr` wasn’t checking for explicit object parameters at all and thus assumed that there was no `this` in the current context because the lambda didn’t have one, even though we were in a member function and had captured its `this`. - The code retrieving captures as lvalues *did* account for explicit object parameters, but it did not handle the case of the explicit object parameter being passed by value rather than by reference. This fixes #80997. --------- Co-authored-by: cor3ntin Co-authored-by: Aaron Ballman --- clang/docs/ReleaseNotes.rst | 3 + clang/lib/AST/ExprConstant.cpp | 129 ++++++++++-------- .../constexpr-explicit-object-lambda.cpp | 34 +++++ 3 files changed, 111 insertions(+), 55 deletions(-) create mode 100644 clang/test/SemaCXX/constexpr-explicit-object-lambda.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index c5488e8742f6..5fe3fd066df2 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -373,6 +373,9 @@ Bug Fixes to C++ Support and (`#74494 `_) - Allow access to a public template alias declaration that refers to friend's private nested type. (#GH25708). +- Fixed a crash in constant evaluation when trying to access a + captured ``this`` pointer in a lambda with an explicit object parameter. + Fixes (#GH80997) Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index 726415cfbde0..b154a196e11c 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -8517,6 +8517,53 @@ public: }; } // end anonymous namespace +/// Get an lvalue to a field of a lambda's closure type. +static bool HandleLambdaCapture(EvalInfo &Info, const Expr *E, LValue &Result, + const CXXMethodDecl *MD, const FieldDecl *FD, + bool LValueToRValueConversion) { + // Static lambda function call operators can't have captures. We already + // diagnosed this, so bail out here. + if (MD->isStatic()) { + assert(Info.CurrentCall->This == nullptr && + "This should not be set for a static call operator"); + return false; + } + + // Start with 'Result' referring to the complete closure object... + if (MD->isExplicitObjectMemberFunction()) { + // Self may be passed by reference or by value. + const ParmVarDecl *Self = MD->getParamDecl(0); + if (Self->getType()->isReferenceType()) { + APValue *RefValue = Info.getParamSlot(Info.CurrentCall->Arguments, Self); + Result.setFrom(Info.Ctx, *RefValue); + } else { + const ParmVarDecl *VD = Info.CurrentCall->Arguments.getOrigParam(Self); + CallStackFrame *Frame = + Info.getCallFrameAndDepth(Info.CurrentCall->Arguments.CallIndex) + .first; + unsigned Version = Info.CurrentCall->Arguments.Version; + Result.set({VD, Frame->Index, Version}); + } + } else + Result = *Info.CurrentCall->This; + + // ... then update it to refer to the field of the closure object + // that represents the capture. + if (!HandleLValueMember(Info, E, Result, FD)) + return false; + + // And if the field is of reference type (or if we captured '*this' by + // reference), update 'Result' to refer to what + // the field refers to. + if (LValueToRValueConversion) { + APValue RVal; + if (!handleLValueToRValueConversion(Info, E, FD->getType(), Result, RVal)) + return false; + Result.setFrom(Info.Ctx, RVal); + } + return true; +} + /// Evaluate an expression as an lvalue. This can be legitimately called on /// expressions which are not glvalues, in three cases: /// * function designators in C, and @@ -8561,37 +8608,8 @@ bool LValueExprEvaluator::VisitVarDecl(const Expr *E, const VarDecl *VD) { if (auto *FD = Info.CurrentCall->LambdaCaptureFields.lookup(VD)) { const auto *MD = cast(Info.CurrentCall->Callee); - - // Static lambda function call operators can't have captures. We already - // diagnosed this, so bail out here. - if (MD->isStatic()) { - assert(Info.CurrentCall->This == nullptr && - "This should not be set for a static call operator"); - return false; - } - - // Start with 'Result' referring to the complete closure object... - if (MD->isExplicitObjectMemberFunction()) { - APValue *RefValue = - Info.getParamSlot(Info.CurrentCall->Arguments, MD->getParamDecl(0)); - Result.setFrom(Info.Ctx, *RefValue); - } else - Result = *Info.CurrentCall->This; - - // ... then update it to refer to the field of the closure object - // that represents the capture. - if (!HandleLValueMember(Info, E, Result, FD)) - return false; - // And if the field is of reference type, update 'Result' to refer to what - // the field refers to. - if (FD->getType()->isReferenceType()) { - APValue RVal; - if (!handleLValueToRValueConversion(Info, E, FD->getType(), Result, - RVal)) - return false; - Result.setFrom(Info.Ctx, RVal); - } - return true; + return HandleLambdaCapture(Info, E, Result, MD, FD, + FD->getType()->isReferenceType()); } } @@ -9069,45 +9087,46 @@ public: return Error(E); } bool VisitCXXThisExpr(const CXXThisExpr *E) { - // Can't look at 'this' when checking a potential constant expression. - if (Info.checkingPotentialConstantExpression()) - return false; - if (!Info.CurrentCall->This) { + auto DiagnoseInvalidUseOfThis = [&] { if (Info.getLangOpts().CPlusPlus11) Info.FFDiag(E, diag::note_constexpr_this) << E->isImplicit(); else Info.FFDiag(E); + }; + + // Can't look at 'this' when checking a potential constant expression. + if (Info.checkingPotentialConstantExpression()) return false; + + bool IsExplicitLambda = + isLambdaCallWithExplicitObjectParameter(Info.CurrentCall->Callee); + if (!IsExplicitLambda) { + if (!Info.CurrentCall->This) { + DiagnoseInvalidUseOfThis(); + return false; + } + + Result = *Info.CurrentCall->This; } - Result = *Info.CurrentCall->This; if (isLambdaCallOperator(Info.CurrentCall->Callee)) { // Ensure we actually have captured 'this'. If something was wrong with // 'this' capture, the error would have been previously reported. // Otherwise we can be inside of a default initialization of an object // declared by lambda's body, so no need to return false. - if (!Info.CurrentCall->LambdaThisCaptureField) - return true; - - // If we have captured 'this', the 'this' expression refers - // to the enclosing '*this' object (either by value or reference) which is - // either copied into the closure object's field that represents the - // '*this' or refers to '*this'. - // Update 'Result' to refer to the data member/field of the closure object - // that represents the '*this' capture. - if (!HandleLValueMember(Info, E, Result, - Info.CurrentCall->LambdaThisCaptureField)) - return false; - // If we captured '*this' by reference, replace the field with its referent. - if (Info.CurrentCall->LambdaThisCaptureField->getType() - ->isPointerType()) { - APValue RVal; - if (!handleLValueToRValueConversion(Info, E, E->getType(), Result, - RVal)) + if (!Info.CurrentCall->LambdaThisCaptureField) { + if (IsExplicitLambda && !Info.CurrentCall->This) { + DiagnoseInvalidUseOfThis(); return false; + } - Result.setFrom(Info.Ctx, RVal); + return true; } + + const auto *MD = cast(Info.CurrentCall->Callee); + return HandleLambdaCapture( + Info, E, Result, MD, Info.CurrentCall->LambdaThisCaptureField, + Info.CurrentCall->LambdaThisCaptureField->getType()->isPointerType()); } return true; } diff --git a/clang/test/SemaCXX/constexpr-explicit-object-lambda.cpp b/clang/test/SemaCXX/constexpr-explicit-object-lambda.cpp new file mode 100644 index 000000000000..4e8e94d428d0 --- /dev/null +++ b/clang/test/SemaCXX/constexpr-explicit-object-lambda.cpp @@ -0,0 +1,34 @@ +// RUN: %clang_cc1 -std=c++23 -verify %s +// expected-no-diagnostics + +struct S { + int i = 42; + constexpr auto f1() { + return [this](this auto) { + return this->i; + }(); + }; + + constexpr auto f2() { + return [this](this auto&&) { + return this->i; + }(); + }; + + constexpr auto f3() { + return [i = this->i](this auto) { + return i; + }(); + }; + + constexpr auto f4() { + return [i = this->i](this auto&&) { + return i; + }(); + }; +}; + +static_assert(S().f1() == 42); +static_assert(S().f2() == 42); +static_assert(S().f3() == 42); +static_assert(S().f4() == 42); -- GitLab From ab9564c315c5111f73788aec9715b488db68d895 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 13 Mar 2024 10:51:48 -0700 Subject: [PATCH 0162/1407] [RISCV] Add SMLoc to expanded vector pseudoinstructions in AsmParser. (#84875) This is needed for llvm-mca to correctly apply vsetvli instruments to these instructions. Fixes #84799. --- llvm/include/llvm/MC/MCInstBuilder.h | 6 ++ .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 39 +++++++---- .../RISCV/SiFive7/vector-integer-arithmetic.s | 70 +++++++++++++++++-- 3 files changed, 97 insertions(+), 18 deletions(-) diff --git a/llvm/include/llvm/MC/MCInstBuilder.h b/llvm/include/llvm/MC/MCInstBuilder.h index 6e5e9dd69018..d06ed4c6c840 100644 --- a/llvm/include/llvm/MC/MCInstBuilder.h +++ b/llvm/include/llvm/MC/MCInstBuilder.h @@ -27,6 +27,12 @@ public: Inst.setOpcode(Opcode); } + /// Set the location. + MCInstBuilder &setLoc(SMLoc SM) { + Inst.setLoc(SM); + return *this; + } + /// Add a new register operand. MCInstBuilder &addReg(unsigned Reg) { Inst.addOperand(MCOperand::createReg(Reg)); diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index d83979a873f2..caff0e8fcefe 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -3271,11 +3271,13 @@ void RISCVAsmParser::emitVMSGE(MCInst &Inst, unsigned Opcode, SMLoc IDLoc, .addOperand(Inst.getOperand(0)) .addOperand(Inst.getOperand(1)) .addOperand(Inst.getOperand(2)) - .addReg(RISCV::NoRegister)); + .addReg(RISCV::NoRegister) + .setLoc(IDLoc)); emitToStreamer(Out, MCInstBuilder(RISCV::VMNAND_MM) .addOperand(Inst.getOperand(0)) .addOperand(Inst.getOperand(0)) - .addOperand(Inst.getOperand(0))); + .addOperand(Inst.getOperand(0)) + .setLoc(IDLoc)); } else if (Inst.getNumOperands() == 4) { // masked va >= x, vd != v0 // @@ -3287,11 +3289,13 @@ void RISCVAsmParser::emitVMSGE(MCInst &Inst, unsigned Opcode, SMLoc IDLoc, .addOperand(Inst.getOperand(0)) .addOperand(Inst.getOperand(1)) .addOperand(Inst.getOperand(2)) - .addOperand(Inst.getOperand(3))); + .addOperand(Inst.getOperand(3)) + .setLoc(IDLoc)); emitToStreamer(Out, MCInstBuilder(RISCV::VMXOR_MM) .addOperand(Inst.getOperand(0)) .addOperand(Inst.getOperand(0)) - .addReg(RISCV::V0)); + .addReg(RISCV::V0) + .setLoc(IDLoc)); } else if (Inst.getNumOperands() == 5 && Inst.getOperand(0).getReg() == RISCV::V0) { // masked va >= x, vd == v0 @@ -3306,11 +3310,13 @@ void RISCVAsmParser::emitVMSGE(MCInst &Inst, unsigned Opcode, SMLoc IDLoc, .addOperand(Inst.getOperand(1)) .addOperand(Inst.getOperand(2)) .addOperand(Inst.getOperand(3)) - .addReg(RISCV::NoRegister)); + .addReg(RISCV::NoRegister) + .setLoc(IDLoc)); emitToStreamer(Out, MCInstBuilder(RISCV::VMANDN_MM) .addOperand(Inst.getOperand(0)) .addOperand(Inst.getOperand(0)) - .addOperand(Inst.getOperand(1))); + .addOperand(Inst.getOperand(1)) + .setLoc(IDLoc)); } else if (Inst.getNumOperands() == 5) { // masked va >= x, any vd // @@ -3323,19 +3329,23 @@ void RISCVAsmParser::emitVMSGE(MCInst &Inst, unsigned Opcode, SMLoc IDLoc, .addOperand(Inst.getOperand(1)) .addOperand(Inst.getOperand(2)) .addOperand(Inst.getOperand(3)) - .addReg(RISCV::NoRegister)); + .addReg(RISCV::NoRegister) + .setLoc(IDLoc)); emitToStreamer(Out, MCInstBuilder(RISCV::VMANDN_MM) .addOperand(Inst.getOperand(1)) .addReg(RISCV::V0) - .addOperand(Inst.getOperand(1))); + .addOperand(Inst.getOperand(1)) + .setLoc(IDLoc)); emitToStreamer(Out, MCInstBuilder(RISCV::VMANDN_MM) .addOperand(Inst.getOperand(0)) .addOperand(Inst.getOperand(0)) - .addReg(RISCV::V0)); + .addReg(RISCV::V0) + .setLoc(IDLoc)); emitToStreamer(Out, MCInstBuilder(RISCV::VMOR_MM) .addOperand(Inst.getOperand(0)) .addOperand(Inst.getOperand(1)) - .addOperand(Inst.getOperand(0))); + .addOperand(Inst.getOperand(0)) + .setLoc(IDLoc)); } } @@ -3637,7 +3647,8 @@ bool RISCVAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc, .addOperand(Inst.getOperand(0)) .addOperand(Inst.getOperand(1)) .addImm(Imm - 1) - .addOperand(Inst.getOperand(3))); + .addOperand(Inst.getOperand(3)) + .setLoc(IDLoc)); return false; } case RISCV::PseudoVMSGEU_VI: @@ -3655,7 +3666,8 @@ bool RISCVAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc, .addOperand(Inst.getOperand(0)) .addOperand(Inst.getOperand(1)) .addOperand(Inst.getOperand(1)) - .addOperand(Inst.getOperand(3))); + .addOperand(Inst.getOperand(3)) + .setLoc(IDLoc)); } else { // Other immediate values can subtract one like signed. unsigned Opc = Inst.getOpcode() == RISCV::PseudoVMSGEU_VI @@ -3665,7 +3677,8 @@ bool RISCVAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc, .addOperand(Inst.getOperand(0)) .addOperand(Inst.getOperand(1)) .addImm(Imm - 1) - .addOperand(Inst.getOperand(3))); + .addOperand(Inst.getOperand(3)) + .setLoc(IDLoc)); } return false; diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFive7/vector-integer-arithmetic.s b/llvm/test/tools/llvm-mca/RISCV/SiFive7/vector-integer-arithmetic.s index 21459bc45d45..3b6fd7e15013 100644 --- a/llvm/test/tools/llvm-mca/RISCV/SiFive7/vector-integer-arithmetic.s +++ b/llvm/test/tools/llvm-mca/RISCV/SiFive7/vector-integer-arithmetic.s @@ -399,6 +399,26 @@ vmseq.vv v4, v8, v12 vsetvli zero, zero, e64, m8, tu, mu vmseq.vx v4, v8, x10 +# Pseudo instructions +vsetvli zero, zero, e8, mf8, tu, mu +vmslt.vi v4, v8, 1 +vsetvli zero, zero, e8, mf4, tu, mu +vmsltu.vi v4, v8, 1 +vsetvli zero, zero, e8, mf2, tu, mu +vmsltu.vi v4, v8, 0 +vsetvli zero, zero, e8, m1, tu, mu +vmsgeu.vi v4, v8, 1 +vsetvli zero, zero, e8, m2, tu, mu +vmsge.vi v4, v8, 1 +vsetvli zero, zero, e8, m4, tu, mu +vmsgeu.vi v4, v8, 0 +vsetvli zero, zero, e16, mf4, tu, mu +vmsge.vi v4, v8, 0 +vsetvli zero, zero, e16, mf2, tu, mu +vmsge.vx v4, v8, x10 +vsetvli zero, zero, e16, m1, tu, mu +vmsgeu.vx v4, v8, x11 + # Vector Integer Min/Max Instructions vsetvli zero, zero, e8, mf8, tu, mu vminu.vv v4, v8, v12 @@ -754,14 +774,14 @@ vsetvli zero, zero, e64, m8, tu, mu vmv.v.v v4, v12 # CHECK: Iterations: 1 -# CHECK-NEXT: Instructions: 707 -# CHECK-NEXT: Total Cycles: 11962 -# CHECK-NEXT: Total uOps: 707 +# CHECK-NEXT: Instructions: 727 +# CHECK-NEXT: Total Cycles: 12018 +# CHECK-NEXT: Total uOps: 727 # CHECK: Dispatch Width: 2 # CHECK-NEXT: uOps Per Cycle: 0.06 # CHECK-NEXT: IPC: 0.06 -# CHECK-NEXT: Block RThroughput: 11549.0 +# CHECK-NEXT: Block RThroughput: 11583.0 # CHECK: Instruction Info: # CHECK-NEXT: [1]: #uOps @@ -1144,6 +1164,26 @@ vmv.v.v v4, v12 # CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e64, m8, tu, mu # CHECK-NEXT: 1 19 17.00 vmseq.vx v4, v8, a0 # CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, mf8, tu, mu +# CHECK-NEXT: 1 4 2.00 vmsle.vi v4, v8, 0 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, mf4, tu, mu +# CHECK-NEXT: 1 4 2.00 vmsleu.vi v4, v8, 0 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, mf2, tu, mu +# CHECK-NEXT: 1 4 2.00 vmsne.vv v4, v8, v8 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, m1, tu, mu +# CHECK-NEXT: 1 5 3.00 vmsgtu.vi v4, v8, 0 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, m2, tu, mu +# CHECK-NEXT: 1 7 5.00 vmsgt.vi v4, v8, 0 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, m4, tu, mu +# CHECK-NEXT: 1 11 9.00 vmseq.vv v4, v8, v8 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e16, mf4, tu, mu +# CHECK-NEXT: 1 4 2.00 vmsgt.vi v4, v8, -1 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e16, mf2, tu, mu +# CHECK-NEXT: 1 4 2.00 vmslt.vx v4, v8, a0 +# CHECK-NEXT: 1 4 2.00 vmnot.m v4, v4 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e16, m1, tu, mu +# CHECK-NEXT: 1 5 3.00 vmsltu.vx v4, v8, a1 +# CHECK-NEXT: 1 4 2.00 vmnot.m v4, v4 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, mf8, tu, mu # CHECK-NEXT: 1 4 2.00 vminu.vv v4, v8, v12 # CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, mf4, tu, mu # CHECK-NEXT: 1 4 2.00 vminu.vx v4, v8, a0 @@ -1492,7 +1532,7 @@ vmv.v.v v4, v12 # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] -# CHECK-NEXT: - - 333.00 - 11549.00 374.00 - - +# CHECK-NEXT: - - 342.00 - 11583.00 385.00 - - # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] Instructions: @@ -1868,6 +1908,26 @@ vmv.v.v v4, v12 # CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e64, m8, tu, mu # CHECK-NEXT: - - - - 17.00 1.00 - - vmseq.vx v4, v8, a0 # CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, mf8, tu, mu +# CHECK-NEXT: - - - - 2.00 1.00 - - vmsle.vi v4, v8, 0 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, mf4, tu, mu +# CHECK-NEXT: - - - - 2.00 1.00 - - vmsleu.vi v4, v8, 0 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, mf2, tu, mu +# CHECK-NEXT: - - - - 2.00 1.00 - - vmsne.vv v4, v8, v8 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, m1, tu, mu +# CHECK-NEXT: - - - - 3.00 1.00 - - vmsgtu.vi v4, v8, 0 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, m2, tu, mu +# CHECK-NEXT: - - - - 5.00 1.00 - - vmsgt.vi v4, v8, 0 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, m4, tu, mu +# CHECK-NEXT: - - - - 9.00 1.00 - - vmseq.vv v4, v8, v8 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e16, mf4, tu, mu +# CHECK-NEXT: - - - - 2.00 1.00 - - vmsgt.vi v4, v8, -1 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e16, mf2, tu, mu +# CHECK-NEXT: - - - - 2.00 1.00 - - vmslt.vx v4, v8, a0 +# CHECK-NEXT: - - - - 2.00 1.00 - - vmnot.m v4, v4 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e16, m1, tu, mu +# CHECK-NEXT: - - - - 3.00 1.00 - - vmsltu.vx v4, v8, a1 +# CHECK-NEXT: - - - - 2.00 1.00 - - vmnot.m v4, v4 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, mf8, tu, mu # CHECK-NEXT: - - - - 2.00 1.00 - - vminu.vv v4, v8, v12 # CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, mf4, tu, mu # CHECK-NEXT: - - - - 2.00 1.00 - - vminu.vx v4, v8, a0 -- GitLab From 0bb30f9896d9cdd92514e0a2bfdc03811831f21c Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Wed, 13 Mar 2024 10:56:26 -0700 Subject: [PATCH 0163/1407] [NFC] [hwasan] factor out some opt handling (#84414) --- .../Instrumentation/HWAddressSanitizer.cpp | 47 +++++++------------ 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index 11a5c29c35f7..87584dace32d 100644 --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -260,6 +260,10 @@ static cl::opt ClUsePageAliases("hwasan-experimental-use-page-aliases", namespace { +template T optOr(cl::opt &Opt, T Other) { + return Opt.getNumOccurrences() ? Opt : Other; +} + bool shouldUsePageAliases(const Triple &TargetTriple) { return ClUsePageAliases && TargetTriple.getArch() == Triple::x86_64; } @@ -269,14 +273,11 @@ bool shouldInstrumentStack(const Triple &TargetTriple) { } bool shouldInstrumentWithCalls(const Triple &TargetTriple) { - return ClInstrumentWithCalls.getNumOccurrences() - ? ClInstrumentWithCalls - : TargetTriple.getArch() == Triple::x86_64; + return optOr(ClInstrumentWithCalls, TargetTriple.getArch() == Triple::x86_64); } bool mightUseStackSafetyAnalysis(bool DisableOptimization) { - return ClUseStackSafety.getNumOccurrences() ? ClUseStackSafety - : !DisableOptimization; + return optOr(ClUseStackSafety, !DisableOptimization); } bool shouldUseStackSafetyAnalysis(const Triple &TargetTriple, @@ -296,10 +297,8 @@ public: HWAddressSanitizer(Module &M, bool CompileKernel, bool Recover, const StackSafetyGlobalInfo *SSI) : M(M), SSI(SSI) { - this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover; - this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 - ? ClEnableKhwasan - : CompileKernel; + this->Recover = optOr(ClRecover, Recover); + this->CompileKernel = optOr(ClEnableKhwasan, CompileKernel); this->Rng = ClRandomSkipRate.getNumOccurrences() ? M.createRNG("hwasan") : nullptr; @@ -625,19 +624,14 @@ void HWAddressSanitizer::initializeModule() { bool NewRuntime = !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30); - UseShortGranules = - ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime; - OutlinedChecks = - (TargetTriple.isAArch64() || TargetTriple.isRISCV64()) && - TargetTriple.isOSBinFormatELF() && - (ClInlineAllChecks.getNumOccurrences() ? !ClInlineAllChecks : !Recover); + UseShortGranules = optOr(ClUseShortGranules, NewRuntime); + OutlinedChecks = (TargetTriple.isAArch64() || TargetTriple.isRISCV64()) && + TargetTriple.isOSBinFormatELF() && + !optOr(ClInlineAllChecks, Recover); - InlineFastPath = - (ClInlineFastPathChecks.getNumOccurrences() - ? ClInlineFastPathChecks - : !(TargetTriple.isAndroid() || - TargetTriple.isOSFuchsia())); // These platforms may prefer less - // inlining to reduce binary size. + // These platforms may prefer less inlining to reduce binary size. + InlineFastPath = optOr(ClInlineFastPathChecks, !(TargetTriple.isAndroid() || + TargetTriple.isOSFuchsia())); if (ClMatchAllTag.getNumOccurrences()) { if (ClMatchAllTag != -1) { @@ -649,22 +643,17 @@ void HWAddressSanitizer::initializeModule() { UseMatchAllCallback = !CompileKernel && MatchAllTag.has_value(); // If we don't have personality function support, fall back to landing pads. - InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences() - ? ClInstrumentLandingPads - : !NewRuntime; + InstrumentLandingPads = optOr(ClInstrumentLandingPads, !NewRuntime); if (!CompileKernel) { createHwasanCtorComdat(); - bool InstrumentGlobals = - ClGlobals.getNumOccurrences() ? ClGlobals : NewRuntime; + bool InstrumentGlobals = optOr(ClGlobals, NewRuntime); if (InstrumentGlobals && !UsePageAliases) instrumentGlobals(); bool InstrumentPersonalityFunctions = - ClInstrumentPersonalityFunctions.getNumOccurrences() - ? ClInstrumentPersonalityFunctions - : NewRuntime; + optOr(ClInstrumentPersonalityFunctions, NewRuntime); if (InstrumentPersonalityFunctions) instrumentPersonalityFunctions(); } -- GitLab From c41966161fffea6ef280fbd341ef1751f70379dd Mon Sep 17 00:00:00 2001 From: Jacek Caban Date: Wed, 13 Mar 2024 18:56:46 +0100 Subject: [PATCH 0164/1407] [llvm-ar] Be explicit about archive format in coff-symtab.test tests. (#85112) Fixes test failures on AIX after #82898. --- llvm/test/tools/llvm-ar/coff-symtab.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/tools/llvm-ar/coff-symtab.test b/llvm/test/tools/llvm-ar/coff-symtab.test index 4f7270d9e2c6..4a574723a9be 100644 --- a/llvm/test/tools/llvm-ar/coff-symtab.test +++ b/llvm/test/tools/llvm-ar/coff-symtab.test @@ -16,7 +16,7 @@ RUN: llvm-nm --print-armap out3.a | FileCheck %s Create an empty archive with no symbol map, add a COFF file to it and check that the output archive is a COFF archive. -RUN: llvm-ar rcS out4.a +RUN: llvm-ar --format coff rcS out4.a RUN: llvm-ar rs out4.a coff-symtab.obj RUN: llvm-nm --print-armap out4.a | FileCheck %s -- GitLab From 3e6d56617f43f86d65dba04c94277dc4a40c2a86 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 13 Mar 2024 11:01:37 -0700 Subject: [PATCH 0165/1407] [SLP][NFC]Add a test with reused buildvector node, being resized after minbitwidth analysis. --- .../AArch64/gather-with-minbith-user.ll | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll new file mode 100644 index 000000000000..9566c00dd630 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll @@ -0,0 +1,89 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s + +define void @h() { +; CHECK-LABEL: define void @h() { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr i8, ptr null, i64 16 +; CHECK-NEXT: [[TMP0:%.*]] = sub <8 x i32> zeroinitializer, zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = add <8 x i32> zeroinitializer, zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> [[TMP1]], <8 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = or <8 x i32> [[TMP2]], zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = trunc <8 x i32> [[TMP3]] to <8 x i16> +; CHECK-NEXT: store <8 x i16> [[TMP4]], ptr [[ARRAYIDX2]], align 2 +; CHECK-NEXT: ret void +; +entry: + %conv9 = zext i16 0 to i32 + %arrayidx2 = getelementptr i8, ptr null, i64 16 + %conv310 = zext i16 0 to i32 + %add4 = add i32 %conv310, %conv9 + %sub = sub i32 0, %conv310 + %conv15 = sext i16 0 to i32 + %shr = ashr i32 0, 0 + %arrayidx18 = getelementptr i8, ptr null, i64 24 + %conv19 = sext i16 0 to i32 + %sub20 = sub i32 %shr, %conv19 + %shr29 = ashr i32 0, 0 + %add30 = add i32 %shr29, %conv15 + %sub39 = or i32 %sub, %sub20 + %conv40 = trunc i32 %sub39 to i16 + store i16 %conv40, ptr %arrayidx2, align 2 + %sub44 = or i32 %add4, %add30 + %conv45 = trunc i32 %sub44 to i16 + store i16 %conv45, ptr %arrayidx18, align 2 + %arrayidx2.1 = getelementptr i8, ptr null, i64 18 + %conv3.112 = zext i16 0 to i32 + %add4.1 = add i32 %conv3.112, 0 + %sub.1 = sub i32 0, %conv3.112 + %conv15.1 = sext i16 0 to i32 + %shr.1 = ashr i32 0, 0 + %arrayidx18.1 = getelementptr i8, ptr null, i64 26 + %conv19.1 = sext i16 0 to i32 + %sub20.1 = sub i32 %shr.1, %conv19.1 + %shr29.1 = ashr i32 0, 0 + %add30.1 = add i32 %shr29.1, %conv15.1 + %sub39.1 = or i32 %sub.1, %sub20.1 + %conv40.1 = trunc i32 %sub39.1 to i16 + store i16 %conv40.1, ptr %arrayidx2.1, align 2 + %sub44.1 = or i32 %add4.1, %add30.1 + %conv45.1 = trunc i32 %sub44.1 to i16 + store i16 %conv45.1, ptr %arrayidx18.1, align 2 + %conv.213 = zext i16 0 to i32 + %arrayidx2.2 = getelementptr i8, ptr null, i64 20 + %conv3.214 = zext i16 0 to i32 + %add4.2 = add i32 0, %conv.213 + %sub.2 = sub i32 0, %conv3.214 + %conv15.2 = sext i16 0 to i32 + %shr.2 = ashr i32 0, 0 + %arrayidx18.2 = getelementptr i8, ptr null, i64 28 + %conv19.2 = sext i16 0 to i32 + %sub20.2 = sub i32 %shr.2, %conv19.2 + %shr29.2 = ashr i32 0, 0 + %add30.2 = add i32 %shr29.2, %conv15.2 + %sub39.2 = or i32 %sub.2, %sub20.2 + %conv40.2 = trunc i32 %sub39.2 to i16 + store i16 %conv40.2, ptr %arrayidx2.2, align 2 + %sub44.2 = or i32 %add4.2, %add30.2 + %conv45.2 = trunc i32 %sub44.2 to i16 + store i16 %conv45.2, ptr %arrayidx18.2, align 2 + %conv.315 = zext i16 0 to i32 + %arrayidx2.3 = getelementptr i8, ptr null, i64 22 + %conv3.316 = zext i16 0 to i32 + %add4.3 = add i32 0, %conv.315 + %sub.3 = sub i32 0, %conv3.316 + %conv15.3 = sext i16 0 to i32 + %shr.3 = ashr i32 0, 0 + %arrayidx18.3 = getelementptr i8, ptr null, i64 30 + %conv19.3 = sext i16 0 to i32 + %sub20.3 = sub i32 %shr.3, %conv19.3 + %shr29.3 = ashr i32 0, 0 + %add30.3 = add i32 %shr29.3, %conv15.3 + %sub39.3 = or i32 %sub.3, %sub20.3 + %conv40.3 = trunc i32 %sub39.3 to i16 + store i16 %conv40.3, ptr %arrayidx2.3, align 2 + %sub44.3 = or i32 %add4.3, %add30.3 + %conv45.3 = trunc i32 %sub44.3 to i16 + store i16 %conv45.3, ptr %arrayidx18.3, align 2 + ret void +} -- GitLab From f50d3582b4844b86ad86372028e44b52c560ec7d Mon Sep 17 00:00:00 2001 From: Ian Anderson Date: Wed, 13 Mar 2024 11:15:41 -0700 Subject: [PATCH 0166/1407] [clang][modules] giving the __stddef_ headers their own modules can cause redeclaration errors with -fbuiltin-headers-in-system-modules (#84127) On Apple platforms, some of the stddef.h types are also declared in system headers. In particular NULL has a conflicting declaration in . When that's in a different module from <__stddef_null.h>, redeclaration errors can occur. Make the \_\_stddef_ headers be non-modular in -fbuiltin-headers-in-system-modules and restore them back to not respecting their header guards. Still define the header guards though. __stddef_max_align_t.h was in _Builtin_stddef_max_align_t prior to the addition of _Builtin_stddef, and it needs to stay in a module because struct's can't be type merged. __stddef_wint_t.h didn't used to have a module, but leave it in it current module since it doesn't really belong to stddef.h. --- clang/lib/Basic/Module.cpp | 7 ++-- clang/lib/Headers/__stddef_null.h | 2 +- clang/lib/Headers/__stddef_nullptr_t.h | 7 +++- clang/lib/Headers/__stddef_offsetof.h | 7 +++- clang/lib/Headers/__stddef_ptrdiff_t.h | 7 +++- clang/lib/Headers/__stddef_rsize_t.h | 7 +++- clang/lib/Headers/__stddef_size_t.h | 7 +++- clang/lib/Headers/__stddef_unreachable.h | 7 +++- clang/lib/Headers/__stddef_wchar_t.h | 7 +++- clang/lib/Headers/module.modulemap | 20 ++++++------ clang/lib/Lex/ModuleMap.cpp | 9 ++++-- .../no-undeclared-includes-builtins.cpp | 2 +- clang/test/Modules/stddef.c | 32 +++++++++++-------- 13 files changed, 80 insertions(+), 41 deletions(-) diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp index 9f597dcf8b0f..256365d66bb9 100644 --- a/clang/lib/Basic/Module.cpp +++ b/clang/lib/Basic/Module.cpp @@ -301,10 +301,9 @@ bool Module::directlyUses(const Module *Requested) { if (Requested->isSubModuleOf(Use)) return true; - // Anyone is allowed to use our builtin stdarg.h and stddef.h and their - // accompanying modules. - if (Requested->getTopLevelModuleName() == "_Builtin_stdarg" || - Requested->getTopLevelModuleName() == "_Builtin_stddef") + // Anyone is allowed to use our builtin stddef.h and its accompanying modules. + if (Requested->fullModuleNameIs({"_Builtin_stddef", "max_align_t"}) || + Requested->fullModuleNameIs({"_Builtin_stddef_wint_t"})) return true; if (NoUndeclaredIncludes) diff --git a/clang/lib/Headers/__stddef_null.h b/clang/lib/Headers/__stddef_null.h index 7336fdab3897..c10bd2d7d988 100644 --- a/clang/lib/Headers/__stddef_null.h +++ b/clang/lib/Headers/__stddef_null.h @@ -7,7 +7,7 @@ *===-----------------------------------------------------------------------=== */ -#if !defined(NULL) || !__has_feature(modules) +#if !defined(NULL) || !__building_module(_Builtin_stddef) /* linux/stddef.h will define NULL to 0. glibc (and other) headers then define * __need_NULL and rely on stddef.h to redefine NULL to the correct value again. diff --git a/clang/lib/Headers/__stddef_nullptr_t.h b/clang/lib/Headers/__stddef_nullptr_t.h index 183d394d56c1..7f3fbe6fe0d3 100644 --- a/clang/lib/Headers/__stddef_nullptr_t.h +++ b/clang/lib/Headers/__stddef_nullptr_t.h @@ -7,7 +7,12 @@ *===-----------------------------------------------------------------------=== */ -#ifndef _NULLPTR_T +/* + * When -fbuiltin-headers-in-system-modules is set this is a non-modular header + * and needs to behave as if it was textual. + */ +#if !defined(_NULLPTR_T) || \ + (__has_feature(modules) && !__building_module(_Builtin_stddef)) #define _NULLPTR_T #ifdef __cplusplus diff --git a/clang/lib/Headers/__stddef_offsetof.h b/clang/lib/Headers/__stddef_offsetof.h index 3b347b3b92f6..84172c6cd273 100644 --- a/clang/lib/Headers/__stddef_offsetof.h +++ b/clang/lib/Headers/__stddef_offsetof.h @@ -7,6 +7,11 @@ *===-----------------------------------------------------------------------=== */ -#ifndef offsetof +/* + * When -fbuiltin-headers-in-system-modules is set this is a non-modular header + * and needs to behave as if it was textual. + */ +#if !defined(offsetof) || \ + (__has_feature(modules) && !__building_module(_Builtin_stddef)) #define offsetof(t, d) __builtin_offsetof(t, d) #endif diff --git a/clang/lib/Headers/__stddef_ptrdiff_t.h b/clang/lib/Headers/__stddef_ptrdiff_t.h index 3ea6d7d2852e..fd3c893c66c9 100644 --- a/clang/lib/Headers/__stddef_ptrdiff_t.h +++ b/clang/lib/Headers/__stddef_ptrdiff_t.h @@ -7,7 +7,12 @@ *===-----------------------------------------------------------------------=== */ -#ifndef _PTRDIFF_T +/* + * When -fbuiltin-headers-in-system-modules is set this is a non-modular header + * and needs to behave as if it was textual. + */ +#if !defined(_PTRDIFF_T) || \ + (__has_feature(modules) && !__building_module(_Builtin_stddef)) #define _PTRDIFF_T typedef __PTRDIFF_TYPE__ ptrdiff_t; diff --git a/clang/lib/Headers/__stddef_rsize_t.h b/clang/lib/Headers/__stddef_rsize_t.h index b6428d0c12b6..dd433d40d973 100644 --- a/clang/lib/Headers/__stddef_rsize_t.h +++ b/clang/lib/Headers/__stddef_rsize_t.h @@ -7,7 +7,12 @@ *===-----------------------------------------------------------------------=== */ -#ifndef _RSIZE_T +/* + * When -fbuiltin-headers-in-system-modules is set this is a non-modular header + * and needs to behave as if it was textual. + */ +#if !defined(_RSIZE_T) || \ + (__has_feature(modules) && !__building_module(_Builtin_stddef)) #define _RSIZE_T typedef __SIZE_TYPE__ rsize_t; diff --git a/clang/lib/Headers/__stddef_size_t.h b/clang/lib/Headers/__stddef_size_t.h index e4a389510bcd..3dd7b1f37929 100644 --- a/clang/lib/Headers/__stddef_size_t.h +++ b/clang/lib/Headers/__stddef_size_t.h @@ -7,7 +7,12 @@ *===-----------------------------------------------------------------------=== */ -#ifndef _SIZE_T +/* + * When -fbuiltin-headers-in-system-modules is set this is a non-modular header + * and needs to behave as if it was textual. + */ +#if !defined(_SIZE_T) || \ + (__has_feature(modules) && !__building_module(_Builtin_stddef)) #define _SIZE_T typedef __SIZE_TYPE__ size_t; diff --git a/clang/lib/Headers/__stddef_unreachable.h b/clang/lib/Headers/__stddef_unreachable.h index 3e7fe0197966..518580c92d3f 100644 --- a/clang/lib/Headers/__stddef_unreachable.h +++ b/clang/lib/Headers/__stddef_unreachable.h @@ -7,6 +7,11 @@ *===-----------------------------------------------------------------------=== */ -#ifndef unreachable +/* + * When -fbuiltin-headers-in-system-modules is set this is a non-modular header + * and needs to behave as if it was textual. + */ +#if !defined(unreachable) || \ + (__has_feature(modules) && !__building_module(_Builtin_stddef)) #define unreachable() __builtin_unreachable() #endif diff --git a/clang/lib/Headers/__stddef_wchar_t.h b/clang/lib/Headers/__stddef_wchar_t.h index 16a6186512c0..bd69f6322541 100644 --- a/clang/lib/Headers/__stddef_wchar_t.h +++ b/clang/lib/Headers/__stddef_wchar_t.h @@ -9,7 +9,12 @@ #if !defined(__cplusplus) || (defined(_MSC_VER) && !_NATIVE_WCHAR_T_DEFINED) -#ifndef _WCHAR_T +/* + * When -fbuiltin-headers-in-system-modules is set this is a non-modular header + * and needs to behave as if it was textual. + */ +#if !defined(_WCHAR_T) || \ + (__has_feature(modules) && !__building_module(_Builtin_stddef)) #define _WCHAR_T #ifdef _MSC_EXTENSIONS diff --git a/clang/lib/Headers/module.modulemap b/clang/lib/Headers/module.modulemap index a786689d3917..56a13f69bc05 100644 --- a/clang/lib/Headers/module.modulemap +++ b/clang/lib/Headers/module.modulemap @@ -155,9 +155,9 @@ module _Builtin_intrinsics [system] [extern_c] { // Start -fbuiltin-headers-in-system-modules affected modules -// The following modules all ignore their top level headers -// when -fbuiltin-headers-in-system-modules is passed, and -// most of those headers join system modules when present. +// The following modules all ignore their headers when +// -fbuiltin-headers-in-system-modules is passed, and many of +// those headers join system modules when present. // e.g. if -fbuiltin-headers-in-system-modules is passed, then // float.h will not be in the _Builtin_float module (that module @@ -190,11 +190,6 @@ module _Builtin_stdalign [system] { export * } -// When -fbuiltin-headers-in-system-modules is passed, only -// the top level headers are removed, the implementation headers -// will always be in their submodules. That means when stdarg.h -// is included, it will still import this module and make the -// appropriate submodules visible. module _Builtin_stdarg [system] { textual header "stdarg.h" @@ -237,6 +232,8 @@ module _Builtin_stdbool [system] { module _Builtin_stddef [system] { textual header "stddef.h" + // __stddef_max_align_t.h is always in this module, even if + // -fbuiltin-headers-in-system-modules is passed. explicit module max_align_t { header "__stddef_max_align_t.h" export * @@ -283,9 +280,10 @@ module _Builtin_stddef [system] { } } -/* wint_t is provided by and not . It's here - * for compatibility, but must be explicitly requested. Therefore - * __stddef_wint_t.h is not part of _Builtin_stddef. */ +// wint_t is provided by and not . It's here +// for compatibility, but must be explicitly requested. Therefore +// __stddef_wint_t.h is not part of _Builtin_stddef. It is always in +// this module even if -fbuiltin-headers-in-system-modules is passed. module _Builtin_stddef_wint_t [system] { header "__stddef_wint_t.h" export * diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp index afb2948f05ae..10c475f617d4 100644 --- a/clang/lib/Lex/ModuleMap.cpp +++ b/clang/lib/Lex/ModuleMap.cpp @@ -2498,9 +2498,12 @@ void ModuleMapParser::parseHeaderDecl(MMToken::TokenKind LeadingToken, } bool NeedsFramework = false; - // Don't add the top level headers to the builtin modules if the builtin headers - // belong to the system modules. - if (!Map.LangOpts.BuiltinHeadersInSystemModules || ActiveModule->isSubModule() || !isBuiltInModuleName(ActiveModule->Name)) + // Don't add headers to the builtin modules if the builtin headers belong to + // the system modules, with the exception of __stddef_max_align_t.h which + // always had its own module. + if (!Map.LangOpts.BuiltinHeadersInSystemModules || + !isBuiltInModuleName(ActiveModule->getTopLevelModuleName()) || + ActiveModule->fullModuleNameIs({"_Builtin_stddef", "max_align_t"})) Map.addUnresolvedHeader(ActiveModule, std::move(Header), NeedsFramework); if (NeedsFramework) diff --git a/clang/test/Modules/no-undeclared-includes-builtins.cpp b/clang/test/Modules/no-undeclared-includes-builtins.cpp index c9bffc556199..f9eefd24a33c 100644 --- a/clang/test/Modules/no-undeclared-includes-builtins.cpp +++ b/clang/test/Modules/no-undeclared-includes-builtins.cpp @@ -8,7 +8,7 @@ // headers. // RUN: rm -rf %t -// RUN: %clang_cc1 -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -I %S/Inputs/no-undeclared-includes-builtins/libcxx -I %S/Inputs/no-undeclared-includes-builtins/glibc %s +// RUN: %clang_cc1 -fmodules-cache-path=%t -fmodules -fbuiltin-headers-in-system-modules -fimplicit-module-maps -I %S/Inputs/no-undeclared-includes-builtins/libcxx -I %S/Inputs/no-undeclared-includes-builtins/glibc %s // expected-no-diagnostics #include diff --git a/clang/test/Modules/stddef.c b/clang/test/Modules/stddef.c index 5bc0d1e44c85..762398261468 100644 --- a/clang/test/Modules/stddef.c +++ b/clang/test/Modules/stddef.c @@ -1,29 +1,33 @@ // RUN: rm -rf %t -// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fbuiltin-headers-in-system-modules -fmodules-cache-path=%t -I%S/Inputs/StdDef %s -verify -fno-modules-error-recovery +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fbuiltin-headers-in-system-modules -fmodules-cache-path=%t -I%S/Inputs/StdDef %s -verify=builtin-headers-in-system-modules -fno-modules-error-recovery // RUN: rm -rf %t -// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -I%S/Inputs/StdDef %s -verify -fno-modules-error-recovery +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -I%S/Inputs/StdDef %s -verify=no-builtin-headers-in-system-modules -fno-modules-error-recovery #include "ptrdiff_t.h" ptrdiff_t pdt; -// size_t is declared in both size_t.h and __stddef_size_t.h, both of which are -// modular headers. Regardless of whether stddef.h joins the StdDef test module -// or is in its _Builtin_stddef module, __stddef_size_t.h will be in -// _Builtin_stddef.size_t. It's not defined which module will win as the expected -// provider of size_t. For the purposes of this test it doesn't matter which header -// gets reported, just as long as it isn't other.h or include_again.h. -size_t st; // expected-error-re {{missing '#include "{{size_t|__stddef_size_t}}.h"'; 'size_t' must be declared before it is used}} -// expected-note@size_t.h:* 0+ {{here}} -// expected-note@__stddef_size_t.h:* 0+ {{here}} +// size_t is declared in both size_t.h and __stddef_size_t.h. If +// -fbuiltin-headers-in-system-modules is set, then __stddef_size_t.h is a +// non-modular header that will be transitively pulled in the StdDef test module +// by include_again.h. Otherwise it will be in the _Builtin_stddef module. In +// any case it's not defined which module will win as the expected provider of +// size_t. For the purposes of this test it doesn't matter which of the two +// providing headers get reported. +size_t st; // builtin-headers-in-system-modules-error-re {{missing '#include "{{size_t|include_again}}.h"'; 'size_t' must be declared before it is used}} \ + no-builtin-headers-in-system-modules-error-re {{missing '#include "{{size_t|__stddef_size_t}}.h"'; 'size_t' must be declared before it is used}} +// builtin-headers-in-system-modules-note@size_t.h:* 0+ {{here}} \ + no-builtin-headers-in-system-modules-note@size_t.h:* 0+ {{here}} +// builtin-headers-in-system-modules-note@__stddef_size_t.h:* 0+ {{here}} \ + no-builtin-headers-in-system-modules-note@__stddef_size_t.h:* 0+ {{here}} #include "include_again.h" -// Includes which includes <__stddef_size_t.h> which imports the -// _Builtin_stddef.size_t module. +// Includes which includes <__stddef_size_t.h>. size_t st2; #include "size_t.h" -// Redeclares size_t, but the type merger should figure it out. +// Redeclares size_t when -fbuiltin-headers-in-system-modules is not passed, but +// the type merger should figure it out. size_t st3; -- GitLab From 55b90b5140a2fe5f625a1dfe9dbb4ed4df968ce0 Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Wed, 13 Mar 2024 11:28:44 -0700 Subject: [PATCH 0167/1407] [compiler-rt] Remove llvm_gtest dependency from unit tests All these unit tests already include ${COMPILER_RT_GTEST_SOURCE} as an input source file and the target llvm_gtest does not exist for standalone builds. Currently the DEPS argument is ignored for standalone builds so the missing target is not a problem, but as part of fixing a build race for standalone builds I am planning to include those dependencies in COMPILER_RT_TEST_STANDALONE_BUILD_LIBS configurations. Reviewed By: vitalybuka Pull Request: https://github.com/llvm/llvm-project/pull/83649 --- compiler-rt/lib/asan/tests/CMakeLists.txt | 2 +- compiler-rt/lib/fuzzer/tests/CMakeLists.txt | 4 ++-- compiler-rt/lib/gwp_asan/tests/CMakeLists.txt | 2 +- compiler-rt/lib/interception/tests/CMakeLists.txt | 1 - compiler-rt/lib/msan/tests/CMakeLists.txt | 2 +- compiler-rt/lib/orc/tests/CMakeLists.txt | 2 +- compiler-rt/lib/sanitizer_common/tests/CMakeLists.txt | 1 - compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt | 2 +- compiler-rt/lib/tsan/tests/CMakeLists.txt | 2 +- compiler-rt/lib/xray/tests/CMakeLists.txt | 2 +- 10 files changed, 9 insertions(+), 11 deletions(-) diff --git a/compiler-rt/lib/asan/tests/CMakeLists.txt b/compiler-rt/lib/asan/tests/CMakeLists.txt index 6ee2fb01c0df..bda47bd7fd6a 100644 --- a/compiler-rt/lib/asan/tests/CMakeLists.txt +++ b/compiler-rt/lib/asan/tests/CMakeLists.txt @@ -172,7 +172,7 @@ function(add_asan_tests arch test_runtime) function(generate_asan_tests test_objects test_suite testname) generate_compiler_rt_tests(${test_objects} ${test_suite} ${testname} ${arch} COMPILE_DEPS ${ASAN_UNITTEST_HEADERS} ${ASAN_IGNORELIST_FILE} - DEPS llvm_gtest asan + DEPS asan KIND ${TEST_KIND} ${ARGN} ) diff --git a/compiler-rt/lib/fuzzer/tests/CMakeLists.txt b/compiler-rt/lib/fuzzer/tests/CMakeLists.txt index dd82c492e83a..8f5707c687ac 100644 --- a/compiler-rt/lib/fuzzer/tests/CMakeLists.txt +++ b/compiler-rt/lib/fuzzer/tests/CMakeLists.txt @@ -74,7 +74,7 @@ if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST FUZZER_SUPPORTED_ARCH) FuzzerUnitTests "Fuzzer-${arch}-Test" ${arch} SOURCES FuzzerUnittest.cpp ${COMPILER_RT_GTEST_SOURCE} RUNTIME ${LIBFUZZER_TEST_RUNTIME} - DEPS llvm_gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} + DEPS ${LIBFUZZER_TEST_RUNTIME_DEPS} CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS} LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS}) set_target_properties(FuzzerUnitTests PROPERTIES @@ -84,7 +84,7 @@ if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST FUZZER_SUPPORTED_ARCH) generate_compiler_rt_tests(FuzzedDataProviderTestObjects FuzzedDataProviderUnitTests "FuzzerUtils-${arch}-Test" ${arch} SOURCES FuzzedDataProviderUnittest.cpp ${COMPILER_RT_GTEST_SOURCE} - DEPS llvm_gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} ${COMPILER_RT_SOURCE_DIR}/include/fuzzer/FuzzedDataProvider.h + DEPS ${LIBFUZZER_TEST_RUNTIME_DEPS} ${COMPILER_RT_SOURCE_DIR}/include/fuzzer/FuzzedDataProvider.h CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS} LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS}) set_target_properties(FuzzedDataProviderUnitTests PROPERTIES diff --git a/compiler-rt/lib/gwp_asan/tests/CMakeLists.txt b/compiler-rt/lib/gwp_asan/tests/CMakeLists.txt index 4915c83d49ca..2ec332ea74c1 100644 --- a/compiler-rt/lib/gwp_asan/tests/CMakeLists.txt +++ b/compiler-rt/lib/gwp_asan/tests/CMakeLists.txt @@ -74,7 +74,7 @@ if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST GWP_ASAN_SUPPORTED_ARCH) GwpAsanUnitTests "GwpAsan-${arch}-Test" ${arch} SOURCES ${GWP_ASAN_UNITTESTS} ${COMPILER_RT_GTEST_SOURCE} RUNTIME ${GWP_ASAN_TEST_RUNTIME} - DEPS llvm_gtest ${GWP_ASAN_UNIT_TEST_HEADERS} + DEPS ${GWP_ASAN_UNIT_TEST_HEADERS} CFLAGS ${GWP_ASAN_UNITTEST_CFLAGS} LINK_FLAGS ${GWP_ASAN_UNITTEST_LINK_FLAGS}) set_target_properties(GwpAsanUnitTests PROPERTIES diff --git a/compiler-rt/lib/interception/tests/CMakeLists.txt b/compiler-rt/lib/interception/tests/CMakeLists.txt index 644a57664cc4..0a235c662af3 100644 --- a/compiler-rt/lib/interception/tests/CMakeLists.txt +++ b/compiler-rt/lib/interception/tests/CMakeLists.txt @@ -107,7 +107,6 @@ macro(add_interception_tests_for_arch arch) RUNTIME ${INTERCEPTION_COMMON_LIB} SOURCES ${INTERCEPTION_UNITTESTS} ${COMPILER_RT_GTEST_SOURCE} COMPILE_DEPS ${INTERCEPTION_TEST_HEADERS} - DEPS llvm_gtest CFLAGS ${INTERCEPTION_TEST_CFLAGS_COMMON} LINK_FLAGS ${INTERCEPTION_TEST_LINK_FLAGS_COMMON}) endmacro() diff --git a/compiler-rt/lib/msan/tests/CMakeLists.txt b/compiler-rt/lib/msan/tests/CMakeLists.txt index 412a0f6b3de7..bc58c0b9fabf 100644 --- a/compiler-rt/lib/msan/tests/CMakeLists.txt +++ b/compiler-rt/lib/msan/tests/CMakeLists.txt @@ -70,7 +70,7 @@ macro(msan_compile obj_list source arch kind cflags) ${obj_list} ${source} ${arch} KIND ${kind} COMPILE_DEPS ${MSAN_UNITTEST_HEADERS} - DEPS llvm_gtest msan + DEPS msan CFLAGS -isystem ${CMAKE_CURRENT_BINARY_DIR}/../libcxx_msan_${arch}/include/c++/v1 ${MSAN_UNITTEST_INSTRUMENTED_CFLAGS} ${cflags} ) diff --git a/compiler-rt/lib/orc/tests/CMakeLists.txt b/compiler-rt/lib/orc/tests/CMakeLists.txt index 2f1cb7657c28..e8f4c95b8a65 100644 --- a/compiler-rt/lib/orc/tests/CMakeLists.txt +++ b/compiler-rt/lib/orc/tests/CMakeLists.txt @@ -73,7 +73,7 @@ macro(add_orc_unittest testname) SOURCES ${TEST_SOURCES} ${COMPILER_RT_GTEST_SOURCE} RUNTIME "${ORC_RUNTIME_LIBS}" COMPILE_DEPS ${TEST_HEADERS} ${ORC_HEADERS} - DEPS llvm_gtest ${ORC_DEPS} + DEPS ${ORC_DEPS} CFLAGS ${ORC_UNITTEST_CFLAGS} ${COMPILER_RT_GTEST_CFLAGS} LINK_FLAGS ${ORC_UNITTEST_LINK_FLAGS}) endif() diff --git a/compiler-rt/lib/sanitizer_common/tests/CMakeLists.txt b/compiler-rt/lib/sanitizer_common/tests/CMakeLists.txt index 3c709e411e48..a3efe6871508 100644 --- a/compiler-rt/lib/sanitizer_common/tests/CMakeLists.txt +++ b/compiler-rt/lib/sanitizer_common/tests/CMakeLists.txt @@ -176,7 +176,6 @@ macro(add_sanitizer_tests_for_arch arch) RUNTIME "${SANITIZER_COMMON_LIB}" SOURCES ${SANITIZER_UNITTESTS} ${COMPILER_RT_GTEST_SOURCE} ${COMPILER_RT_GMOCK_SOURCE} COMPILE_DEPS ${SANITIZER_TEST_HEADERS} - DEPS llvm_gtest CFLAGS ${SANITIZER_TEST_CFLAGS_COMMON} ${extra_flags} LINK_FLAGS ${SANITIZER_TEST_LINK_FLAGS_COMMON} ${TARGET_LINK_FLAGS} ${extra_flags}) diff --git a/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt index c6b6a1cb57ce..ac92805872f9 100644 --- a/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt +++ b/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt @@ -81,7 +81,7 @@ macro(add_scudo_unittest testname) "${testname}-${arch}-Test" ${arch} SOURCES ${TEST_SOURCES} ${COMPILER_RT_GTEST_SOURCE} COMPILE_DEPS ${SCUDO_TEST_HEADERS} - DEPS llvm_gtest scudo_standalone + DEPS scudo_standalone RUNTIME ${RUNTIME} CFLAGS ${SCUDO_UNITTEST_CFLAGS} LINK_FLAGS ${SCUDO_UNITTEST_LINK_FLAGS}) diff --git a/compiler-rt/lib/tsan/tests/CMakeLists.txt b/compiler-rt/lib/tsan/tests/CMakeLists.txt index c02c2279583b..ad8cc9b0eb05 100644 --- a/compiler-rt/lib/tsan/tests/CMakeLists.txt +++ b/compiler-rt/lib/tsan/tests/CMakeLists.txt @@ -64,7 +64,7 @@ foreach (header ${TSAN_HEADERS}) list(APPEND TSAN_RTL_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../${header}) endforeach() -set(TSAN_DEPS llvm_gtest tsan) +set(TSAN_DEPS tsan) # TSan uses C++ standard library headers. if (TARGET cxx-headers OR HAVE_LIBCXX) set(TSAN_DEPS cxx-headers) diff --git a/compiler-rt/lib/xray/tests/CMakeLists.txt b/compiler-rt/lib/xray/tests/CMakeLists.txt index 732f982c932f..0a428b9a30b1 100644 --- a/compiler-rt/lib/xray/tests/CMakeLists.txt +++ b/compiler-rt/lib/xray/tests/CMakeLists.txt @@ -109,7 +109,7 @@ macro(add_xray_unittest testname) ${XRAY_HEADERS} ${XRAY_ALL_SOURCE_FILES_ABS_PATHS} "test_helpers.h" RUNTIME "${XRAY_RUNTIME_LIBS}" - DEPS llvm_gtest xray llvm-xray LLVMXRay LLVMTestingSupport + DEPS xray llvm-xray LLVMXRay LLVMTestingSupport CFLAGS ${XRAY_UNITTEST_CFLAGS} LINK_FLAGS ${TARGET_LINK_FLAGS} ${XRAY_UNITTEST_LINK_FLAGS} ) -- GitLab From 27e5312a8bc8935f9c5620ff061c647d9fbcec85 Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Wed, 13 Mar 2024 11:32:36 -0700 Subject: [PATCH 0168/1407] [compiler-rt] Avoid generating coredumps when piped to a tool I was trying to debug why `ninja check-compiler-rt` was taking so long to run on my system and after some debugging it turned out that most of the time was being spent generating core dumps. On many current Linux systems, coredumps are no longer dumped in the CWD but instead piped to a utility such as systemd-coredumpd that stores them in a deterministic location. This can be done by setting the kernel.core_pattern sysctl to start with a '|'. However, when using such a setup the kernel ignores a coredump limit of 0 (since there is no file being written) and we can end up piping many gigabytes of data to systemd-coredumpd which causes the test suite to freeze for a long time. While most piped coredump handlers do respect the crashing processes' RLIMIT_CORE, this is notable not the case for Debian's systemd-coredump due to a local patch that changes sysctl.d/50-coredump.conf to ignore the specified limit and instead use RLIM_INFINITY (https://salsa.debian.org/systemd-team/systemd/-/commit/64599ffe44f0d). Fortunately there is a workaround: the kernel recognizes the magic value of 1 for RLIMIT_CORE to disable coredumps when piping. One byte is also too small to generate any coredump, so it effectively behaves as if we had set the value to zero. The alternative to using RLIMIT_CORE=1 would be to use prctl() with the PR_SET_DUMPABLE flag, however that also prevents ptrace(), so makes it impossible to attach a debugger. Fixes: https://github.com/llvm/llvm-project/issues/45797 Reviewed By: vitalybuka Pull Request: https://github.com/llvm/llvm-project/pull/83701 --- .../sanitizer_posix_libcdep.cpp | 19 ++++++++++++++++++- .../sanitizer_common/TestCases/corelimit.cpp | 7 ++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp index ef1fc3549743..3605d0d666e3 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp @@ -104,7 +104,24 @@ static void setlim(int res, rlim_t lim) { void DisableCoreDumperIfNecessary() { if (common_flags()->disable_coredump) { - setlim(RLIMIT_CORE, 0); + rlimit rlim; + CHECK_EQ(0, getrlimit(RLIMIT_CORE, &rlim)); + // On Linux, if the kernel.core_pattern sysctl starts with a '|' (i.e. it + // is being piped to a coredump handler such as systemd-coredumpd), the + // kernel ignores RLIMIT_CORE (since we aren't creating a file in the file + // system) except for the magic value of 1, which disables coredumps when + // piping. 1 byte is too small for any kind of valid core dump, so it + // also disables coredumps if kernel.core_pattern creates files directly. + // While most piped coredump handlers do respect the crashing processes' + // RLIMIT_CORE, this is notable not the case for Debian's systemd-coredump + // due to a local patch that changes sysctl.d/50-coredump.conf to ignore + // the specified limit and instead use RLIM_INFINITY. + // + // The alternative to using RLIMIT_CORE=1 would be to use prctl() with the + // PR_SET_DUMPABLE flag, however that also prevents ptrace(), so makes it + // impossible to attach a debugger. + rlim.rlim_cur = Min(SANITIZER_LINUX ? 1 : 0, rlim.rlim_max); + CHECK_EQ(0, setrlimit(RLIMIT_CORE, &rlim)); } } diff --git a/compiler-rt/test/sanitizer_common/TestCases/corelimit.cpp b/compiler-rt/test/sanitizer_common/TestCases/corelimit.cpp index 2378a4cfdced..fed2e1d89cbf 100644 --- a/compiler-rt/test/sanitizer_common/TestCases/corelimit.cpp +++ b/compiler-rt/test/sanitizer_common/TestCases/corelimit.cpp @@ -10,7 +10,12 @@ int main() { getrlimit(RLIMIT_CORE, &lim_core); void *p; if (sizeof(p) == 8) { - assert(0 == lim_core.rlim_cur); +#ifdef __linux__ + // See comments in DisableCoreDumperIfNecessary(). + assert(lim_core.rlim_cur == 1); +#else + assert(lim_core.rlim_cur == 0); +#endif } return 0; } -- GitLab From 06e310fee12c3e5ea5c7ef066eab946eb84f317d Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Wed, 13 Mar 2024 11:32:53 -0700 Subject: [PATCH 0169/1407] Revert "[AArch64] Improve lowering of truncating uzp1" (#85115) Reverts llvm/llvm-project#82457 The bot is broken, likely because of mid-air collision. --- .../Target/AArch64/AArch64ISelLowering.cpp | 39 ++- llvm/lib/Target/AArch64/AArch64InstrInfo.td | 53 ++-- .../CodeGen/AArch64/arm64-convert-v4f64.ll | 21 +- llvm/test/CodeGen/AArch64/extbinopload.ll | 31 +-- .../CodeGen/AArch64/fp-conversion-to-tbl.ll | 5 +- llvm/test/CodeGen/AArch64/fptoi.ll | 256 ++++++++++++------ llvm/test/CodeGen/AArch64/neon-truncstore.ll | 5 +- llvm/test/CodeGen/AArch64/sadd_sat_vec.ll | 2 +- llvm/test/CodeGen/AArch64/shuffle-tbl34.ll | 14 +- llvm/test/CodeGen/AArch64/ssub_sat_vec.ll | 2 +- llvm/test/CodeGen/AArch64/tbl-loops.ll | 4 +- llvm/test/CodeGen/AArch64/trunc-to-tbl.ll | 28 +- llvm/test/CodeGen/AArch64/uadd_sat_vec.ll | 2 +- llvm/test/CodeGen/AArch64/usub_sat_vec.ll | 2 +- llvm/test/CodeGen/AArch64/vcvt-oversize.ll | 5 +- .../vec-combine-compare-truncate-store.ll | 2 +- .../AArch64/vec3-loads-ext-trunc-stores.ll | 22 +- 17 files changed, 284 insertions(+), 209 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 9665ae5ceb90..5b7a36d2eba7 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -21423,8 +21423,12 @@ static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG, } } - // These optimizations only work on little endian. - if (!DAG.getDataLayout().isLittleEndian()) + // uzp1(xtn x, xtn y) -> xtn(uzp1 (x, y)) + // Only implemented on little-endian subtargets. + bool IsLittleEndian = DAG.getDataLayout().isLittleEndian(); + + // This optimization only works on little endian. + if (!IsLittleEndian) return SDValue(); // uzp1(bitcast(x), bitcast(y)) -> uzp1(x, y) @@ -21443,28 +21447,21 @@ static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG, if (ResVT != MVT::v2i32 && ResVT != MVT::v4i16 && ResVT != MVT::v8i8) return SDValue(); - SDValue SourceOp0 = peekThroughBitcasts(Op0); - SDValue SourceOp1 = peekThroughBitcasts(Op1); + auto getSourceOp = [](SDValue Operand) -> SDValue { + const unsigned Opcode = Operand.getOpcode(); + if (Opcode == ISD::TRUNCATE) + return Operand->getOperand(0); + if (Opcode == ISD::BITCAST && + Operand->getOperand(0).getOpcode() == ISD::TRUNCATE) + return Operand->getOperand(0)->getOperand(0); + return SDValue(); + }; - // truncating uzp1(x, y) -> xtn(concat (x, y)) - if (SourceOp0.getValueType() == SourceOp1.getValueType()) { - EVT Op0Ty = SourceOp0.getValueType(); - if ((ResVT == MVT::v4i16 && Op0Ty == MVT::v2i32) || - (ResVT == MVT::v8i8 && Op0Ty == MVT::v4i16)) { - SDValue Concat = - DAG.getNode(ISD::CONCAT_VECTORS, DL, - Op0Ty.getDoubleNumVectorElementsVT(*DAG.getContext()), - SourceOp0, SourceOp1); - return DAG.getNode(ISD::TRUNCATE, DL, ResVT, Concat); - } - } + SDValue SourceOp0 = getSourceOp(Op0); + SDValue SourceOp1 = getSourceOp(Op1); - // uzp1(xtn x, xtn y) -> xtn(uzp1 (x, y)) - if (SourceOp0.getOpcode() != ISD::TRUNCATE || - SourceOp1.getOpcode() != ISD::TRUNCATE) + if (!SourceOp0 || !SourceOp1) return SDValue(); - SourceOp0 = SourceOp0.getOperand(0); - SourceOp1 = SourceOp1.getOperand(0); if (SourceOp0.getValueType() != SourceOp1.getValueType() || !SourceOp0.getValueType().isSimple()) diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index b4b975cce007..6254e68326f7 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -6153,39 +6153,26 @@ defm UZP2 : SIMDZipVector<0b101, "uzp2", AArch64uzp2>; defm ZIP1 : SIMDZipVector<0b011, "zip1", AArch64zip1>; defm ZIP2 : SIMDZipVector<0b111, "zip2", AArch64zip2>; -def trunc_optional_assert_ext : PatFrags<(ops node:$op0), - [(trunc node:$op0), - (assertzext (trunc node:$op0)), - (assertsext (trunc node:$op0))]>; - -// concat_vectors(trunc(x), trunc(y)) -> uzp1(x, y) -// concat_vectors(assertzext(trunc(x)), assertzext(trunc(y))) -> uzp1(x, y) -// concat_vectors(assertsext(trunc(x)), assertsext(trunc(y))) -> uzp1(x, y) -class concat_trunc_to_uzp1_pat - : Pat<(ConcatTy (concat_vectors (TruncTy (trunc_optional_assert_ext (SrcTy V128:$Vn))), - (TruncTy (trunc_optional_assert_ext (SrcTy V128:$Vm))))), - (!cast("UZP1"#ConcatTy) V128:$Vn, V128:$Vm)>; -def : concat_trunc_to_uzp1_pat; -def : concat_trunc_to_uzp1_pat; -def : concat_trunc_to_uzp1_pat; - -// trunc(concat_vectors(trunc(x), trunc(y))) -> xtn(uzp1(x, y)) -// trunc(concat_vectors(assertzext(trunc(x)), assertzext(trunc(y)))) -> xtn(uzp1(x, y)) -// trunc(concat_vectors(assertsext(trunc(x)), assertsext(trunc(y)))) -> xtn(uzp1(x, y)) -class trunc_concat_trunc_to_xtn_uzp1_pat - : Pat<(Ty (trunc_optional_assert_ext - (ConcatTy (concat_vectors - (TruncTy (trunc_optional_assert_ext (SrcTy V128:$Vn))), - (TruncTy (trunc_optional_assert_ext (SrcTy V128:$Vm))))))), - (!cast("XTN"#Ty) (!cast("UZP1"#ConcatTy) V128:$Vn, V128:$Vm))>; -def : trunc_concat_trunc_to_xtn_uzp1_pat; -def : trunc_concat_trunc_to_xtn_uzp1_pat; - -def : Pat<(v8i8 (trunc (concat_vectors (v4i16 V64:$Vn), (v4i16 V64:$Vm)))), - (UZP1v8i8 V64:$Vn, V64:$Vm)>; -def : Pat<(v4i16 (trunc (concat_vectors (v2i32 V64:$Vn), (v2i32 V64:$Vm)))), - (UZP1v4i16 V64:$Vn, V64:$Vm)>; +def : Pat<(v16i8 (concat_vectors (v8i8 (trunc (v8i16 V128:$Vn))), + (v8i8 (trunc (v8i16 V128:$Vm))))), + (UZP1v16i8 V128:$Vn, V128:$Vm)>; +def : Pat<(v8i16 (concat_vectors (v4i16 (trunc (v4i32 V128:$Vn))), + (v4i16 (trunc (v4i32 V128:$Vm))))), + (UZP1v8i16 V128:$Vn, V128:$Vm)>; +def : Pat<(v4i32 (concat_vectors (v2i32 (trunc (v2i64 V128:$Vn))), + (v2i32 (trunc (v2i64 V128:$Vm))))), + (UZP1v4i32 V128:$Vn, V128:$Vm)>; +// These are the same as above, with an optional assertzext node that can be +// generated from fptoi lowering. +def : Pat<(v16i8 (concat_vectors (v8i8 (assertzext (trunc (v8i16 V128:$Vn)))), + (v8i8 (assertzext (trunc (v8i16 V128:$Vm)))))), + (UZP1v16i8 V128:$Vn, V128:$Vm)>; +def : Pat<(v8i16 (concat_vectors (v4i16 (assertzext (trunc (v4i32 V128:$Vn)))), + (v4i16 (assertzext (trunc (v4i32 V128:$Vm)))))), + (UZP1v8i16 V128:$Vn, V128:$Vm)>; +def : Pat<(v4i32 (concat_vectors (v2i32 (assertzext (trunc (v2i64 V128:$Vn)))), + (v2i32 (assertzext (trunc (v2i64 V128:$Vm)))))), + (UZP1v4i32 V128:$Vn, V128:$Vm)>; def : Pat<(v16i8 (concat_vectors (v8i8 (trunc (AArch64vlshr (v8i16 V128:$Vn), (i32 8)))), diff --git a/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll b/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll index 3007e7ce771e..49325299f74a 100644 --- a/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll +++ b/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll @@ -8,8 +8,9 @@ define <4 x i16> @fptosi_v4f64_to_v4i16(ptr %ptr) { ; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-NEXT: xtn v0.4h, v0.4s +; CHECK-NEXT: xtn v1.2s, v1.2d +; CHECK-NEXT: xtn v0.2s, v0.2d +; CHECK-NEXT: uzp1 v0.4h, v0.4h, v1.4h ; CHECK-NEXT: ret %tmp1 = load <4 x double>, ptr %ptr %tmp2 = fptosi <4 x double> %tmp1 to <4 x i16> @@ -25,10 +26,13 @@ define <8 x i8> @fptosi_v4f64_to_v4i8(ptr %ptr) { ; CHECK-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-NEXT: fcvtzs v3.2d, v3.2d ; CHECK-NEXT: fcvtzs v2.2d, v2.2d -; CHECK-NEXT: uzp1 v0.4s, v1.4s, v0.4s -; CHECK-NEXT: uzp1 v1.4s, v2.4s, v3.4s -; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: xtn v0.2s, v0.2d +; CHECK-NEXT: xtn v1.2s, v1.2d +; CHECK-NEXT: xtn v3.2s, v3.2d +; CHECK-NEXT: xtn v2.2s, v2.2d +; CHECK-NEXT: uzp1 v0.4h, v1.4h, v0.4h +; CHECK-NEXT: uzp1 v1.4h, v2.4h, v3.4h +; CHECK-NEXT: uzp1 v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret %tmp1 = load <8 x double>, ptr %ptr %tmp2 = fptosi <8 x double> %tmp1 to <8 x i8> @@ -92,8 +96,9 @@ define <4 x i16> @fptoui_v4f64_to_v4i16(ptr %ptr) { ; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-NEXT: xtn v0.4h, v0.4s +; CHECK-NEXT: xtn v1.2s, v1.2d +; CHECK-NEXT: xtn v0.2s, v0.2d +; CHECK-NEXT: uzp1 v0.4h, v0.4h, v1.4h ; CHECK-NEXT: ret %tmp1 = load <4 x double>, ptr %ptr %tmp2 = fptoui <4 x double> %tmp1 to <4 x i16> diff --git a/llvm/test/CodeGen/AArch64/extbinopload.ll b/llvm/test/CodeGen/AArch64/extbinopload.ll index dff4831330de..1f68c77611e1 100644 --- a/llvm/test/CodeGen/AArch64/extbinopload.ll +++ b/llvm/test/CodeGen/AArch64/extbinopload.ll @@ -650,7 +650,7 @@ define <16 x i32> @extrause_load(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: add x11, x3, #12 ; CHECK-NEXT: str s1, [x4] ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: ldp s0, s4, [x2] +; CHECK-NEXT: ldp s0, s5, [x2] ; CHECK-NEXT: ushll v2.8h, v0.8b, #0 ; CHECK-NEXT: umov w9, v2.h[0] ; CHECK-NEXT: umov w10, v2.h[1] @@ -662,25 +662,24 @@ define <16 x i32> @extrause_load(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: ushll v2.8h, v2.8b, #0 ; CHECK-NEXT: mov v0.b[10], w9 ; CHECK-NEXT: add x9, x1, #4 -; CHECK-NEXT: mov v1.d[1], v2.d[0] +; CHECK-NEXT: uzp1 v1.8b, v1.8b, v2.8b ; CHECK-NEXT: mov v0.b[11], w10 ; CHECK-NEXT: add x10, x1, #12 -; CHECK-NEXT: bic v1.8h, #255, lsl #8 ; CHECK-NEXT: ld1 { v0.s }[3], [x3], #4 -; CHECK-NEXT: ldr s3, [x0, #12] -; CHECK-NEXT: ldp s2, s7, [x0, #4] -; CHECK-NEXT: ld1 { v4.s }[1], [x3] -; CHECK-NEXT: ldp s5, s6, [x2, #8] -; CHECK-NEXT: ld1 { v3.s }[1], [x10] -; CHECK-NEXT: ld1 { v2.s }[1], [x9] -; CHECK-NEXT: ld1 { v5.s }[1], [x8] -; CHECK-NEXT: ld1 { v6.s }[1], [x11] +; CHECK-NEXT: ldr s4, [x0, #12] +; CHECK-NEXT: ldp s3, s16, [x0, #4] +; CHECK-NEXT: ld1 { v5.s }[1], [x3] +; CHECK-NEXT: ldp s6, s7, [x2, #8] +; CHECK-NEXT: ld1 { v4.s }[1], [x10] +; CHECK-NEXT: ld1 { v3.s }[1], [x9] +; CHECK-NEXT: ld1 { v6.s }[1], [x8] +; CHECK-NEXT: ld1 { v7.s }[1], [x11] ; CHECK-NEXT: add x8, x1, #8 -; CHECK-NEXT: ld1 { v7.s }[1], [x8] -; CHECK-NEXT: uaddl v2.8h, v2.8b, v3.8b -; CHECK-NEXT: ushll v3.8h, v5.8b, #0 -; CHECK-NEXT: uaddl v4.8h, v4.8b, v6.8b -; CHECK-NEXT: uaddw v1.8h, v1.8h, v7.8b +; CHECK-NEXT: ld1 { v16.s }[1], [x8] +; CHECK-NEXT: uaddl v2.8h, v3.8b, v4.8b +; CHECK-NEXT: ushll v3.8h, v6.8b, #0 +; CHECK-NEXT: uaddl v4.8h, v5.8b, v7.8b +; CHECK-NEXT: uaddl v1.8h, v1.8b, v16.8b ; CHECK-NEXT: uaddw2 v5.8h, v3.8h, v0.16b ; CHECK-NEXT: ushll v0.4s, v2.4h, #3 ; CHECK-NEXT: ushll2 v2.4s, v2.8h, #3 diff --git a/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll b/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll index 0a3b9a070c2b..1ea87bb6b04b 100644 --- a/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll +++ b/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll @@ -73,8 +73,9 @@ define void @fptoui_v8f32_to_v8i8_no_loop(ptr %A, ptr %dst) { ; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: fcvtzs.4s v1, v1 ; CHECK-NEXT: fcvtzs.4s v0, v0 -; CHECK-NEXT: uzp1.8h v0, v0, v1 -; CHECK-NEXT: xtn.8b v0, v0 +; CHECK-NEXT: xtn.4h v1, v1 +; CHECK-NEXT: xtn.4h v0, v0 +; CHECK-NEXT: uzp1.8b v0, v0, v1 ; CHECK-NEXT: str d0, [x1] ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/fptoi.ll b/llvm/test/CodeGen/AArch64/fptoi.ll index 7af01b53dae7..67190e8596c4 100644 --- a/llvm/test/CodeGen/AArch64/fptoi.ll +++ b/llvm/test/CodeGen/AArch64/fptoi.ll @@ -1096,17 +1096,30 @@ entry: } define <3 x i16> @fptos_v3f64_v3i16(<3 x double> %a) { -; CHECK-LABEL: fptos_v3f64_v3i16: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2 -; CHECK-NEXT: mov v0.d[1], v1.d[0] -; CHECK-NEXT: fcvtzs v1.2d, v2.2d -; CHECK-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-NEXT: xtn v0.4h, v0.4s -; CHECK-NEXT: ret +; CHECK-SD-LABEL: fptos_v3f64_v3i16: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2 +; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] +; CHECK-SD-NEXT: fcvtzs v1.2d, v2.2d +; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: fptos_v3f64_v3i16: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2 +; CHECK-GI-NEXT: mov v0.d[1], v1.d[0] +; CHECK-GI-NEXT: fcvtzs v1.2d, v2.2d +; CHECK-GI-NEXT: fcvtzs v0.2d, v0.2d +; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: xtn v0.4h, v0.4s +; CHECK-GI-NEXT: ret entry: %c = fptosi <3 x double> %a to <3 x i16> ret <3 x i16> %c @@ -1121,8 +1134,9 @@ define <3 x i16> @fptou_v3f64_v3i16(<3 x double> %a) { ; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] ; CHECK-SD-NEXT: fcvtzs v1.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v3f64_v3i16: @@ -1146,8 +1160,9 @@ define <4 x i16> @fptos_v4f64_v4i16(<4 x double> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v4f64_v4i16: @@ -1167,8 +1182,9 @@ define <4 x i16> @fptou_v4f64_v4i16(<4 x double> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v4f64_v4i16: @@ -1584,8 +1600,9 @@ define <3 x i8> @fptos_v3f64_v3i8(<3 x double> %a) { ; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] ; CHECK-SD-NEXT: fcvtzs v1.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h ; CHECK-SD-NEXT: umov w0, v0.h[0] ; CHECK-SD-NEXT: umov w1, v0.h[1] ; CHECK-SD-NEXT: umov w2, v0.h[2] @@ -1621,8 +1638,9 @@ define <3 x i8> @fptou_v3f64_v3i8(<3 x double> %a) { ; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] ; CHECK-SD-NEXT: fcvtzs v1.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h ; CHECK-SD-NEXT: umov w0, v0.h[0] ; CHECK-SD-NEXT: umov w1, v0.h[1] ; CHECK-SD-NEXT: umov w2, v0.h[2] @@ -1654,8 +1672,9 @@ define <4 x i8> @fptos_v4f64_v4i8(<4 x double> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v4f64_v4i8: @@ -1675,8 +1694,9 @@ define <4 x i8> @fptou_v4f64_v4i8(<4 x double> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v4f64_v4i8: @@ -1698,10 +1718,13 @@ define <8 x i8> @fptos_v8f64_v8i8(<8 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v2.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h -; CHECK-SD-NEXT: xtn v0.8b, v0.8h +; CHECK-SD-NEXT: xtn v3.2s, v3.2d +; CHECK-SD-NEXT: xtn v2.2s, v2.2d +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v2.8b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v8f64_v8i8: @@ -1727,10 +1750,13 @@ define <8 x i8> @fptou_v8f64_v8i8(<8 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v2.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h -; CHECK-SD-NEXT: xtn v0.8b, v0.8h +; CHECK-SD-NEXT: xtn v3.2s, v3.2d +; CHECK-SD-NEXT: xtn v2.2s, v2.2d +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v2.8b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v8f64_v8i8: @@ -1760,13 +1786,21 @@ define <16 x i8> @fptos_v16f64_v16i8(<16 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v2.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: uzp1 v6.4s, v6.4s, v7.4s -; CHECK-SD-NEXT: uzp1 v4.4s, v4.4s, v5.4s -; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: uzp1 v1.8h, v4.8h, v6.8h -; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h -; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v1.16b +; CHECK-SD-NEXT: xtn v7.2s, v7.2d +; CHECK-SD-NEXT: xtn v6.2s, v6.2d +; CHECK-SD-NEXT: xtn v5.2s, v5.2d +; CHECK-SD-NEXT: xtn v4.2s, v4.2d +; CHECK-SD-NEXT: xtn v3.2s, v3.2d +; CHECK-SD-NEXT: xtn v2.2s, v2.2d +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: uzp1 v6.4h, v6.4h, v7.4h +; CHECK-SD-NEXT: uzp1 v4.4h, v4.4h, v5.4h +; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: mov v4.d[1], v6.d[0] +; CHECK-SD-NEXT: mov v0.d[1], v2.d[0] +; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v4.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v16f64_v16i8: @@ -1803,13 +1837,21 @@ define <16 x i8> @fptou_v16f64_v16i8(<16 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v2.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: uzp1 v6.4s, v6.4s, v7.4s -; CHECK-SD-NEXT: uzp1 v4.4s, v4.4s, v5.4s -; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: uzp1 v1.8h, v4.8h, v6.8h -; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h -; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v1.16b +; CHECK-SD-NEXT: xtn v7.2s, v7.2d +; CHECK-SD-NEXT: xtn v6.2s, v6.2d +; CHECK-SD-NEXT: xtn v5.2s, v5.2d +; CHECK-SD-NEXT: xtn v4.2s, v4.2d +; CHECK-SD-NEXT: xtn v3.2s, v3.2d +; CHECK-SD-NEXT: xtn v2.2s, v2.2d +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: uzp1 v6.4h, v6.4h, v7.4h +; CHECK-SD-NEXT: uzp1 v4.4h, v4.4h, v5.4h +; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: mov v4.d[1], v6.d[0] +; CHECK-SD-NEXT: mov v0.d[1], v2.d[0] +; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v4.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v16f64_v16i8: @@ -1858,20 +1900,36 @@ define <32 x i8> @fptos_v32f64_v32i8(<32 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v18.2d, v18.2d ; CHECK-SD-NEXT: fcvtzs v17.2d, v17.2d ; CHECK-SD-NEXT: fcvtzs v16.2d, v16.2d -; CHECK-SD-NEXT: uzp1 v6.4s, v6.4s, v7.4s -; CHECK-SD-NEXT: uzp1 v4.4s, v4.4s, v5.4s -; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: uzp1 v3.4s, v20.4s, v21.4s -; CHECK-SD-NEXT: uzp1 v1.4s, v22.4s, v23.4s -; CHECK-SD-NEXT: uzp1 v5.4s, v18.4s, v19.4s -; CHECK-SD-NEXT: uzp1 v7.4s, v16.4s, v17.4s -; CHECK-SD-NEXT: uzp1 v4.8h, v4.8h, v6.8h -; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h -; CHECK-SD-NEXT: uzp1 v1.8h, v3.8h, v1.8h -; CHECK-SD-NEXT: uzp1 v2.8h, v7.8h, v5.8h +; CHECK-SD-NEXT: xtn v7.2s, v7.2d +; CHECK-SD-NEXT: xtn v6.2s, v6.2d +; CHECK-SD-NEXT: xtn v5.2s, v5.2d +; CHECK-SD-NEXT: xtn v4.2s, v4.2d +; CHECK-SD-NEXT: xtn v3.2s, v3.2d +; CHECK-SD-NEXT: xtn v2.2s, v2.2d +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: xtn v23.2s, v23.2d +; CHECK-SD-NEXT: xtn v22.2s, v22.2d +; CHECK-SD-NEXT: xtn v21.2s, v21.2d +; CHECK-SD-NEXT: xtn v20.2s, v20.2d +; CHECK-SD-NEXT: xtn v19.2s, v19.2d +; CHECK-SD-NEXT: xtn v18.2s, v18.2d +; CHECK-SD-NEXT: xtn v17.2s, v17.2d +; CHECK-SD-NEXT: xtn v16.2s, v16.2d +; CHECK-SD-NEXT: uzp1 v6.4h, v6.4h, v7.4h +; CHECK-SD-NEXT: uzp1 v4.4h, v4.4h, v5.4h +; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v1.4h, v22.4h, v23.4h +; CHECK-SD-NEXT: uzp1 v3.4h, v20.4h, v21.4h +; CHECK-SD-NEXT: uzp1 v5.4h, v18.4h, v19.4h +; CHECK-SD-NEXT: uzp1 v7.4h, v16.4h, v17.4h +; CHECK-SD-NEXT: mov v4.d[1], v6.d[0] +; CHECK-SD-NEXT: mov v0.d[1], v2.d[0] +; CHECK-SD-NEXT: mov v3.d[1], v1.d[0] +; CHECK-SD-NEXT: mov v7.d[1], v5.d[0] ; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v4.16b -; CHECK-SD-NEXT: uzp1 v1.16b, v2.16b, v1.16b +; CHECK-SD-NEXT: uzp1 v1.16b, v7.16b, v3.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v32f64_v32i8: @@ -1939,20 +1997,36 @@ define <32 x i8> @fptou_v32f64_v32i8(<32 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v18.2d, v18.2d ; CHECK-SD-NEXT: fcvtzs v17.2d, v17.2d ; CHECK-SD-NEXT: fcvtzs v16.2d, v16.2d -; CHECK-SD-NEXT: uzp1 v6.4s, v6.4s, v7.4s -; CHECK-SD-NEXT: uzp1 v4.4s, v4.4s, v5.4s -; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: uzp1 v3.4s, v20.4s, v21.4s -; CHECK-SD-NEXT: uzp1 v1.4s, v22.4s, v23.4s -; CHECK-SD-NEXT: uzp1 v5.4s, v18.4s, v19.4s -; CHECK-SD-NEXT: uzp1 v7.4s, v16.4s, v17.4s -; CHECK-SD-NEXT: uzp1 v4.8h, v4.8h, v6.8h -; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h -; CHECK-SD-NEXT: uzp1 v1.8h, v3.8h, v1.8h -; CHECK-SD-NEXT: uzp1 v2.8h, v7.8h, v5.8h +; CHECK-SD-NEXT: xtn v7.2s, v7.2d +; CHECK-SD-NEXT: xtn v6.2s, v6.2d +; CHECK-SD-NEXT: xtn v5.2s, v5.2d +; CHECK-SD-NEXT: xtn v4.2s, v4.2d +; CHECK-SD-NEXT: xtn v3.2s, v3.2d +; CHECK-SD-NEXT: xtn v2.2s, v2.2d +; CHECK-SD-NEXT: xtn v1.2s, v1.2d +; CHECK-SD-NEXT: xtn v0.2s, v0.2d +; CHECK-SD-NEXT: xtn v23.2s, v23.2d +; CHECK-SD-NEXT: xtn v22.2s, v22.2d +; CHECK-SD-NEXT: xtn v21.2s, v21.2d +; CHECK-SD-NEXT: xtn v20.2s, v20.2d +; CHECK-SD-NEXT: xtn v19.2s, v19.2d +; CHECK-SD-NEXT: xtn v18.2s, v18.2d +; CHECK-SD-NEXT: xtn v17.2s, v17.2d +; CHECK-SD-NEXT: xtn v16.2s, v16.2d +; CHECK-SD-NEXT: uzp1 v6.4h, v6.4h, v7.4h +; CHECK-SD-NEXT: uzp1 v4.4h, v4.4h, v5.4h +; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v1.4h, v22.4h, v23.4h +; CHECK-SD-NEXT: uzp1 v3.4h, v20.4h, v21.4h +; CHECK-SD-NEXT: uzp1 v5.4h, v18.4h, v19.4h +; CHECK-SD-NEXT: uzp1 v7.4h, v16.4h, v17.4h +; CHECK-SD-NEXT: mov v4.d[1], v6.d[0] +; CHECK-SD-NEXT: mov v0.d[1], v2.d[0] +; CHECK-SD-NEXT: mov v3.d[1], v1.d[0] +; CHECK-SD-NEXT: mov v7.d[1], v5.d[0] ; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v4.16b -; CHECK-SD-NEXT: uzp1 v1.16b, v2.16b, v1.16b +; CHECK-SD-NEXT: uzp1 v1.16b, v7.16b, v3.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v32f64_v32i8: @@ -2952,8 +3026,9 @@ define <8 x i8> @fptos_v8f32_v8i8(<8 x float> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.4s, v1.4s ; CHECK-SD-NEXT: fcvtzs v0.4s, v0.4s -; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h -; CHECK-SD-NEXT: xtn v0.8b, v0.8h +; CHECK-SD-NEXT: xtn v1.4h, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s +; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v1.8b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v8f32_v8i8: @@ -2973,8 +3048,9 @@ define <8 x i8> @fptou_v8f32_v8i8(<8 x float> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.4s, v1.4s ; CHECK-SD-NEXT: fcvtzs v0.4s, v0.4s -; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h -; CHECK-SD-NEXT: xtn v0.8b, v0.8h +; CHECK-SD-NEXT: xtn v1.4h, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s +; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v1.8b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v8f32_v8i8: @@ -2996,8 +3072,12 @@ define <16 x i8> @fptos_v16f32_v16i8(<16 x float> %a) { ; CHECK-SD-NEXT: fcvtzs v2.4s, v2.4s ; CHECK-SD-NEXT: fcvtzs v1.4s, v1.4s ; CHECK-SD-NEXT: fcvtzs v0.4s, v0.4s -; CHECK-SD-NEXT: uzp1 v2.8h, v2.8h, v3.8h -; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h +; CHECK-SD-NEXT: xtn v3.4h, v3.4s +; CHECK-SD-NEXT: xtn v2.4h, v2.4s +; CHECK-SD-NEXT: xtn v1.4h, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s +; CHECK-SD-NEXT: mov v2.d[1], v3.d[0] +; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] ; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: ret ; @@ -3054,12 +3134,20 @@ define <32 x i8> @fptos_v32f32_v32i8(<32 x float> %a) { ; CHECK-SD-NEXT: fcvtzs v6.4s, v6.4s ; CHECK-SD-NEXT: fcvtzs v5.4s, v5.4s ; CHECK-SD-NEXT: fcvtzs v4.4s, v4.4s -; CHECK-SD-NEXT: uzp1 v2.8h, v2.8h, v3.8h -; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h -; CHECK-SD-NEXT: uzp1 v1.8h, v6.8h, v7.8h -; CHECK-SD-NEXT: uzp1 v3.8h, v4.8h, v5.8h +; CHECK-SD-NEXT: xtn v3.4h, v3.4s +; CHECK-SD-NEXT: xtn v2.4h, v2.4s +; CHECK-SD-NEXT: xtn v1.4h, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s +; CHECK-SD-NEXT: xtn v7.4h, v7.4s +; CHECK-SD-NEXT: xtn v6.4h, v6.4s +; CHECK-SD-NEXT: xtn v5.4h, v5.4s +; CHECK-SD-NEXT: xtn v4.4h, v4.4s +; CHECK-SD-NEXT: mov v2.d[1], v3.d[0] +; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] +; CHECK-SD-NEXT: mov v6.d[1], v7.d[0] +; CHECK-SD-NEXT: mov v4.d[1], v5.d[0] ; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v2.16b -; CHECK-SD-NEXT: uzp1 v1.16b, v3.16b, v1.16b +; CHECK-SD-NEXT: uzp1 v1.16b, v4.16b, v6.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v32f32_v32i8: diff --git a/llvm/test/CodeGen/AArch64/neon-truncstore.ll b/llvm/test/CodeGen/AArch64/neon-truncstore.ll index 5d78ad24eb33..b677d077b98c 100644 --- a/llvm/test/CodeGen/AArch64/neon-truncstore.ll +++ b/llvm/test/CodeGen/AArch64/neon-truncstore.ll @@ -104,7 +104,7 @@ define void @v4i32_v4i8(<4 x i32> %a, ptr %result) { ; CHECK-LABEL: v4i32_v4i8: ; CHECK: // %bb.0: ; CHECK-NEXT: xtn v0.4h, v0.4s -; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b +; CHECK-NEXT: xtn v0.8b, v0.8h ; CHECK-NEXT: str s0, [x0] ; CHECK-NEXT: ret %b = trunc <4 x i32> %a to <4 x i8> @@ -170,7 +170,8 @@ define void @v2i16_v2i8(<2 x i16> %a, ptr %result) { define void @v4i16_v4i8(<4 x i16> %a, ptr %result) { ; CHECK-LABEL: v4i16_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: xtn v0.8b, v0.8h ; CHECK-NEXT: str s0, [x0] ; CHECK-NEXT: ret %b = trunc <4 x i16> %a to <4 x i8> diff --git a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll index 6f1ae023bf25..5f905d94e357 100644 --- a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll @@ -145,7 +145,7 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-NEXT: shl v0.4h, v0.4h, #8 ; CHECK-NEXT: sqadd v0.4h, v0.4h, v1.4h ; CHECK-NEXT: sshr v0.4h, v0.4h, #8 -; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b +; CHECK-NEXT: xtn v0.8b, v0.8h ; CHECK-NEXT: str s0, [x2] ; CHECK-NEXT: ret %x = load <4 x i8>, ptr %px diff --git a/llvm/test/CodeGen/AArch64/shuffle-tbl34.ll b/llvm/test/CodeGen/AArch64/shuffle-tbl34.ll index fb571eff39fe..0ef64789ad97 100644 --- a/llvm/test/CodeGen/AArch64/shuffle-tbl34.ll +++ b/llvm/test/CodeGen/AArch64/shuffle-tbl34.ll @@ -353,17 +353,13 @@ define <8 x i8> @shuffle4_v8i8_v8i8(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x define <8 x i16> @shuffle4_v4i8_zext(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) { ; CHECK-LABEL: shuffle4_v4i8_zext: ; CHECK: // %bb.0: -; CHECK-NEXT: fmov d5, d2 -; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3 +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v1.8b +; CHECK-NEXT: uzp1 v1.8b, v2.8b, v3.8b ; CHECK-NEXT: adrp x8, .LCPI8_0 -; CHECK-NEXT: fmov d4, d0 +; CHECK-NEXT: ushll v2.8h, v0.8b, #0 ; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI8_0] -; CHECK-NEXT: mov v4.d[1], v1.d[0] -; CHECK-NEXT: mov v5.d[1], v3.d[0] -; CHECK-NEXT: bic v4.8h, #255, lsl #8 -; CHECK-NEXT: bic v5.8h, #255, lsl #8 -; CHECK-NEXT: tbl v0.16b, { v4.16b, v5.16b }, v0.16b +; CHECK-NEXT: ushll v3.8h, v1.8b, #0 +; CHECK-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b ; CHECK-NEXT: ret %x = shufflevector <4 x i8> %a, <4 x i8> %b, <8 x i32> %y = shufflevector <4 x i8> %c, <4 x i8> %d, <8 x i32> diff --git a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll index d1f843a09f74..acec3e74d3e9 100644 --- a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll @@ -146,7 +146,7 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-NEXT: shl v0.4h, v0.4h, #8 ; CHECK-NEXT: sqsub v0.4h, v0.4h, v1.4h ; CHECK-NEXT: sshr v0.4h, v0.4h, #8 -; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b +; CHECK-NEXT: xtn v0.8b, v0.8h ; CHECK-NEXT: str s0, [x2] ; CHECK-NEXT: ret %x = load <4 x i8>, ptr %px diff --git a/llvm/test/CodeGen/AArch64/tbl-loops.ll b/llvm/test/CodeGen/AArch64/tbl-loops.ll index 0ad990086551..4f8a4f7aede3 100644 --- a/llvm/test/CodeGen/AArch64/tbl-loops.ll +++ b/llvm/test/CodeGen/AArch64/tbl-loops.ll @@ -41,8 +41,8 @@ define void @loop1(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n ; CHECK-NEXT: fcvtzs v2.4s, v2.4s ; CHECK-NEXT: xtn v1.4h, v1.4s ; CHECK-NEXT: xtn v2.4h, v2.4s -; CHECK-NEXT: uzp1 v1.8b, v1.8b, v0.8b -; CHECK-NEXT: uzp1 v2.8b, v2.8b, v0.8b +; CHECK-NEXT: xtn v1.8b, v1.8h +; CHECK-NEXT: xtn v2.8b, v2.8h ; CHECK-NEXT: mov v1.s[1], v2.s[0] ; CHECK-NEXT: stur d1, [x12, #-4] ; CHECK-NEXT: add x12, x12, #8 diff --git a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll index 18cd4cc2111a..ba367b0dbfde 100644 --- a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll +++ b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll @@ -710,23 +710,23 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) { ; CHECK-NEXT: LBB6_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldp q4, q1, [x0, #48] -; CHECK-NEXT: add x9, x1, #10 -; CHECK-NEXT: ldr d0, [x0, #80] +; CHECK-NEXT: add x9, x1, #8 ; CHECK-NEXT: ldp q3, q2, [x0] -; CHECK-NEXT: ldr q5, [x0, #32] ; CHECK-NEXT: subs x8, x8, #1 +; CHECK-NEXT: ldr d0, [x0, #80] +; CHECK-NEXT: ldr q5, [x0, #32] ; CHECK-NEXT: add x0, x0, #128 -; CHECK-NEXT: uzp1.4s v0, v1, v0 -; CHECK-NEXT: uzp1.4s v1, v5, v4 +; CHECK-NEXT: uzp1.4s v4, v5, v4 ; CHECK-NEXT: uzp1.4s v2, v3, v2 +; CHECK-NEXT: uzp1.4s v0, v1, v0 +; CHECK-NEXT: uzp1.8h v1, v2, v4 ; CHECK-NEXT: xtn.4h v0, v0 -; CHECK-NEXT: uzp1.8h v1, v2, v1 -; CHECK-NEXT: uzp1.8b v2, v0, v0 -; CHECK-NEXT: uzp1.16b v0, v1, v0 -; CHECK-NEXT: st1.b { v2 }[2], [x9] -; CHECK-NEXT: add x9, x1, #8 -; CHECK-NEXT: st1.h { v0 }[4], [x9] -; CHECK-NEXT: str d0, [x1], #16 +; CHECK-NEXT: uzp1.16b v1, v1, v0 +; CHECK-NEXT: xtn.8b v0, v0 +; CHECK-NEXT: st1.h { v1 }[4], [x9] +; CHECK-NEXT: add x9, x1, #10 +; CHECK-NEXT: st1.b { v0 }[2], [x9] +; CHECK-NEXT: str d1, [x1], #16 ; CHECK-NEXT: b.eq LBB6_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret @@ -755,7 +755,7 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) { ; CHECK-BE-NEXT: xtn v0.4h, v0.4s ; CHECK-BE-NEXT: uzp1 v1.8h, v1.8h, v2.8h ; CHECK-BE-NEXT: uzp1 v1.16b, v1.16b, v0.16b -; CHECK-BE-NEXT: uzp1 v0.8b, v0.8b, v0.8b +; CHECK-BE-NEXT: xtn v0.8b, v0.8h ; CHECK-BE-NEXT: rev16 v2.16b, v1.16b ; CHECK-BE-NEXT: rev64 v1.16b, v1.16b ; CHECK-BE-NEXT: st1 { v0.b }[2], [x9] @@ -790,7 +790,7 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) { ; CHECK-DISABLE-NEXT: xtn v0.4h, v0.4s ; CHECK-DISABLE-NEXT: uzp1 v1.8h, v1.8h, v2.8h ; CHECK-DISABLE-NEXT: uzp1 v1.16b, v1.16b, v0.16b -; CHECK-DISABLE-NEXT: uzp1 v0.8b, v0.8b, v0.8b +; CHECK-DISABLE-NEXT: xtn v0.8b, v0.8h ; CHECK-DISABLE-NEXT: rev16 v2.16b, v1.16b ; CHECK-DISABLE-NEXT: rev64 v1.16b, v1.16b ; CHECK-DISABLE-NEXT: st1 { v0.b }[2], [x9] diff --git a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll index f0bbed59405e..e05c65daf50a 100644 --- a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll @@ -142,7 +142,7 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-NEXT: movi d0, #0xff00ff00ff00ff ; CHECK-NEXT: uaddl v1.8h, v1.8b, v2.8b ; CHECK-NEXT: umin v0.4h, v1.4h, v0.4h -; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b +; CHECK-NEXT: xtn v0.8b, v0.8h ; CHECK-NEXT: str s0, [x2] ; CHECK-NEXT: ret %x = load <4 x i8>, ptr %px diff --git a/llvm/test/CodeGen/AArch64/usub_sat_vec.ll b/llvm/test/CodeGen/AArch64/usub_sat_vec.ll index 82c0327219f5..05f43e7d8427 100644 --- a/llvm/test/CodeGen/AArch64/usub_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/usub_sat_vec.ll @@ -143,7 +143,7 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 ; CHECK-NEXT: uqsub v0.4h, v0.4h, v1.4h -; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b +; CHECK-NEXT: xtn v0.8b, v0.8h ; CHECK-NEXT: str s0, [x2] ; CHECK-NEXT: ret %x = load <4 x i8>, ptr %px diff --git a/llvm/test/CodeGen/AArch64/vcvt-oversize.ll b/llvm/test/CodeGen/AArch64/vcvt-oversize.ll index 611940546bc1..380bdbcc7f74 100644 --- a/llvm/test/CodeGen/AArch64/vcvt-oversize.ll +++ b/llvm/test/CodeGen/AArch64/vcvt-oversize.ll @@ -9,8 +9,9 @@ define <8 x i8> @float_to_i8(ptr %in) { ; CHECK-NEXT: fadd v0.4s, v0.4s, v0.4s ; CHECK-NEXT: fcvtzs v0.4s, v0.4s ; CHECK-NEXT: fcvtzs v1.4s, v1.4s -; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: xtn v0.4h, v0.4s +; CHECK-NEXT: xtn v1.4h, v1.4s +; CHECK-NEXT: uzp1 v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret %l = load <8 x float>, ptr %in %scale = fmul <8 x float> %l, diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll index dd7a9c6d7768..9c6ab8da0fa7 100644 --- a/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll +++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll @@ -210,7 +210,7 @@ define void @no_combine_for_non_bool_truncate(<4 x i32> %vec, ptr %out) { ; CHECK-LABEL: no_combine_for_non_bool_truncate: ; CHECK: ; %bb.0: ; CHECK-NEXT: xtn.4h v0, v0 -; CHECK-NEXT: uzp1.8b v0, v0, v0 +; CHECK-NEXT: xtn.8b v0, v0 ; CHECK-NEXT: str s0, [x0] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll index 71d55df66517..90328f73f86b 100644 --- a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll +++ b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll @@ -410,7 +410,7 @@ define void @store_trunc_from_64bits(ptr %src, ptr %dst) { ; BE-NEXT: ldrh w8, [x0, #4] ; BE-NEXT: rev32 v0.4h, v0.4h ; BE-NEXT: mov v0.h[2], w8 -; BE-NEXT: uzp1 v0.8b, v0.8b, v0.8b +; BE-NEXT: xtn v0.8b, v0.8h ; BE-NEXT: rev32 v0.16b, v0.16b ; BE-NEXT: str s0, [sp, #12] ; BE-NEXT: ldrh w9, [sp, #12] @@ -456,7 +456,7 @@ define void @store_trunc_add_from_64bits(ptr %src, ptr %dst) { ; BE-NEXT: add x8, x8, :lo12:.LCPI11_0 ; BE-NEXT: ld1 { v1.4h }, [x8] ; BE-NEXT: add v0.4h, v0.4h, v1.4h -; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b +; BE-NEXT: xtn v1.8b, v0.8h ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -638,7 +638,7 @@ define void @shift_trunc_store(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b +; BE-NEXT: xtn v1.8b, v0.8h ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -672,7 +672,7 @@ define void @shift_trunc_store_default_align(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b +; BE-NEXT: xtn v1.8b, v0.8h ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -706,7 +706,7 @@ define void @shift_trunc_store_align_4(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b +; BE-NEXT: xtn v1.8b, v0.8h ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -741,7 +741,7 @@ define void @shift_trunc_store_const_offset_1(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b +; BE-NEXT: xtn v1.8b, v0.8h ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -777,7 +777,7 @@ define void @shift_trunc_store_const_offset_3(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b +; BE-NEXT: xtn v1.8b, v0.8h ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -801,7 +801,7 @@ define void @shift_trunc_volatile_store(ptr %src, ptr %dst) { ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: shrn.4h v0, v0, #16 -; CHECK-NEXT: uzp1.8b v1, v0, v0 +; CHECK-NEXT: xtn.8b v1, v0 ; CHECK-NEXT: umov.h w8, v0[2] ; CHECK-NEXT: str s1, [sp, #12] ; CHECK-NEXT: ldrh w9, [sp, #12] @@ -816,7 +816,7 @@ define void @shift_trunc_volatile_store(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b +; BE-NEXT: xtn v1.8b, v0.8h ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -868,7 +868,7 @@ define void @load_v3i8_zext_to_3xi32_add_trunc_store(ptr %src) { ; BE-NEXT: ushll v0.8h, v0.8b, #0 ; BE-NEXT: ld1 { v0.b }[4], [x9] ; BE-NEXT: add v0.4h, v0.4h, v1.4h -; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b +; BE-NEXT: xtn v1.8b, v0.8h ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #8] @@ -921,7 +921,7 @@ define void @load_v3i8_sext_to_3xi32_add_trunc_store(ptr %src) { ; BE-NEXT: ushll v0.8h, v0.8b, #0 ; BE-NEXT: ld1 { v0.b }[4], [x9] ; BE-NEXT: add v0.4h, v0.4h, v1.4h -; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b +; BE-NEXT: xtn v1.8b, v0.8h ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #8] -- GitLab From 417324a6c1e7ecb6c145b20905f918378cc824e3 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 13 Mar 2024 10:59:09 -0700 Subject: [PATCH 0170/1407] [RISCV] Remove unnecessary ArrayRef. NFC --- llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index caff0e8fcefe..0607240efff8 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -2823,9 +2823,8 @@ bool RISCVAsmParser::parseDirectiveOption() { break; } - ArrayRef KVArray(RISCVFeatureKV); - auto Ext = llvm::lower_bound(KVArray, Arch); - if (Ext == KVArray.end() || StringRef(Ext->Key) != Arch || + auto Ext = llvm::lower_bound(RISCVFeatureKV, Arch); + if (Ext == std::end(RISCVFeatureKV) || StringRef(Ext->Key) != Arch || !RISCVISAInfo::isSupportedExtension(Arch)) { if (isDigit(Arch.back())) return Error( @@ -2858,7 +2857,7 @@ bool RISCVAsmParser::parseDirectiveOption() { // It is invalid to disable an extension that there are other enabled // extensions depend on it. // TODO: Make use of RISCVISAInfo to handle this - for (auto Feature : KVArray) { + for (auto Feature : RISCVFeatureKV) { if (getSTI().hasFeature(Feature.Value) && Feature.Implies.test(Ext->Value)) return Error(Loc, -- GitLab From 66dd38e8dfd51209aa1fd9bae0a43a355215768f Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 13 Mar 2024 11:37:31 -0700 Subject: [PATCH 0171/1407] [RISCV] Use references to avoid unnecessary struct copies. NFC --- llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index 0607240efff8..2da75bda8d12 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -2716,7 +2716,7 @@ ParseStatus RISCVAsmParser::parseDirective(AsmToken DirectiveID) { bool RISCVAsmParser::resetToArch(StringRef Arch, SMLoc Loc, std::string &Result, bool FromOptionDirective) { - for (auto Feature : RISCVFeatureKV) + for (auto &Feature : RISCVFeatureKV) if (llvm::RISCVISAInfo::isSupportedExtensionFeature(Feature.Key)) clearFeatureBits(Feature.Value, Feature.Key); @@ -2735,7 +2735,7 @@ bool RISCVAsmParser::resetToArch(StringRef Arch, SMLoc Loc, std::string &Result, } auto &ISAInfo = *ParseResult; - for (auto Feature : RISCVFeatureKV) + for (auto &Feature : RISCVFeatureKV) if (ISAInfo->hasExtension(Feature.Key)) setFeatureBits(Feature.Value, Feature.Key); @@ -2857,7 +2857,7 @@ bool RISCVAsmParser::parseDirectiveOption() { // It is invalid to disable an extension that there are other enabled // extensions depend on it. // TODO: Make use of RISCVISAInfo to handle this - for (auto Feature : RISCVFeatureKV) { + for (auto &Feature : RISCVFeatureKV) { if (getSTI().hasFeature(Feature.Value) && Feature.Implies.test(Ext->Value)) return Error(Loc, -- GitLab From b61fb18456ecd798b2fc340367018ab3109ebfae Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Wed, 13 Mar 2024 18:48:13 +0000 Subject: [PATCH 0172/1407] [libc++] Fix tests on musl (#85085) One or two of the tests need slight tweaks to make them pass when building with musl. rdar://118885724 --- .../generic_category.pass.cpp | 19 ++++--- .../system_category.pass.cpp | 19 ++++--- .../put_long_double.pass.cpp | 51 ++++++++++--------- 3 files changed, 52 insertions(+), 37 deletions(-) diff --git a/libcxx/test/std/diagnostics/syserr/syserr.errcat/syserr.errcat.objects/generic_category.pass.cpp b/libcxx/test/std/diagnostics/syserr/syserr.errcat/syserr.errcat.objects/generic_category.pass.cpp index 068202c6e415..d4bbde75ae88 100644 --- a/libcxx/test/std/diagnostics/syserr/syserr.errcat/syserr.errcat.objects/generic_category.pass.cpp +++ b/libcxx/test/std/diagnostics/syserr/syserr.errcat/syserr.errcat.objects/generic_category.pass.cpp @@ -44,14 +44,19 @@ int main(int, char**) errno = E2BIG; // something that message will never generate const std::error_category& e_cat1 = std::generic_category(); const std::string msg = e_cat1.message(-1); - // Exact message format varies by platform. -#if defined(_AIX) - LIBCPP_ASSERT(msg.rfind("Error -1 occurred", 0) == 0); -#elif defined(_NEWLIB_VERSION) - LIBCPP_ASSERT(msg.empty()); -#else - LIBCPP_ASSERT(msg.rfind("Unknown error", 0) == 0); + // Exact message format varies by platform. We can't detect + // some of these (Musl in particular) using the preprocessor, + // so accept a few sensible messages. Newlib unfortunately + // responds with an empty message, which we probably want to + // treat as a failure code otherwise, but we can detect that + // with the preprocessor. + LIBCPP_ASSERT(msg.rfind("Error -1 occurred", 0) == 0 // AIX + || msg.rfind("No error information", 0) == 0 // Musl + || msg.rfind("Unknown error", 0) == 0 // Glibc +#if defined(_NEWLIB_VERSION) + || msg.empty() #endif + ); assert(errno == E2BIG); } diff --git a/libcxx/test/std/diagnostics/syserr/syserr.errcat/syserr.errcat.objects/system_category.pass.cpp b/libcxx/test/std/diagnostics/syserr/syserr.errcat/syserr.errcat.objects/system_category.pass.cpp index 42fdd1cb3b91..eefbddd27a7f 100644 --- a/libcxx/test/std/diagnostics/syserr/syserr.errcat/syserr.errcat.objects/system_category.pass.cpp +++ b/libcxx/test/std/diagnostics/syserr/syserr.errcat/syserr.errcat.objects/system_category.pass.cpp @@ -50,14 +50,19 @@ int main(int, char**) { errno = E2BIG; // something that message will never generate const std::error_category& e_cat1 = std::system_category(); const std::string msg = e_cat1.message(-1); - // Exact message format varies by platform. -#if defined(_AIX) - LIBCPP_ASSERT(msg.rfind("Error -1 occurred", 0) == 0); -#elif defined(_NEWLIB_VERSION) - LIBCPP_ASSERT(msg.empty()); -#else - LIBCPP_ASSERT(msg.rfind("Unknown error", 0) == 0); + // Exact message format varies by platform. We can't detect + // some of these (Musl in particular) using the preprocessor, + // so accept a few sensible messages. Newlib unfortunately + // responds with an empty message, which we probably want to + // treat as a failure code otherwise, but we can detect that + // with the preprocessor. + LIBCPP_ASSERT(msg.rfind("Error -1 occurred", 0) == 0 // AIX + || msg.rfind("No error information", 0) == 0 // Musl + || msg.rfind("Unknown error", 0) == 0 // Glibc +#if defined(_NEWLIB_VERSION) + || msg.empty() #endif + ); assert(errno == E2BIG); } diff --git a/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.pass.cpp b/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.pass.cpp index 8637a933008f..0258ebf87243 100644 --- a/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.pass.cpp +++ b/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.pass.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include "test_macros.h" @@ -8934,11 +8935,12 @@ void test4() char str[200]; std::locale lc = std::locale::classic(); std::locale lg(lc, new my_numpunct); -#ifdef _AIX - std::string inf = "INF"; -#else - std::string inf = "inf"; -#endif + + std::string inf; + + // This should match the underlying C library + std::sprintf(str, "%f", INFINITY); + inf = str; const my_facet f(1); { @@ -10727,24 +10729,27 @@ void test5() std::locale lc = std::locale::classic(); std::locale lg(lc, new my_numpunct); const my_facet f(1); -#if defined(_AIX) - std::string nan= "NaNQ"; - std::string NaN = "NaNQ"; - std::string nan_padding25 = "*********************"; - std::string pnan_sign = "+"; - std::string pnan_padding25 = "********************"; -#else - std::string nan= "nan"; - std::string NaN = "NAN"; - std::string nan_padding25 = "**********************"; -#if defined(TEST_HAS_GLIBC) || defined(_WIN32) - std::string pnan_sign = "+"; - std::string pnan_padding25 = "*********************"; -#else - std::string pnan_sign = ""; - std::string pnan_padding25 = "**********************"; -#endif -#endif + + std::string nan; + std::string NaN; + std::string pnan_sign; + + // The output here depends on the underlying C library, so work out what + // that does. + std::sprintf(str, "%f", std::nan("")); + nan = str; + + std::sprintf(str, "%F", std::nan("")); + NaN = str; + + std::sprintf(str, "%+f", std::nan("")); + if (str[0] == '+') { + pnan_sign = "+"; + } + + std::string nan_padding25 = std::string(25 - nan.length(), '*'); + std::string pnan_padding25 = std::string(25 - nan.length() - pnan_sign.length(), '*'); + { long double v = std::nan(""); std::ios ios(0); -- GitLab From a8967b060df01e46c021f718b4e2d7ed858b8726 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 13 Mar 2024 11:51:51 -0700 Subject: [PATCH 0173/1407] [SLP][NFC]Add a test with buildvector with minbitwidth Root, NFC. --- ...ather-buildvector-with-minbitwidth-user.ll | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/gather-buildvector-with-minbitwidth-user.ll diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-buildvector-with-minbitwidth-user.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-buildvector-with-minbitwidth-user.ll new file mode 100644 index 000000000000..705e425d3e44 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-buildvector-with-minbitwidth-user.ll @@ -0,0 +1,88 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s + +define void @h() { +; CHECK-LABEL: define void @h() { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr i8, ptr null, i64 16 +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x i32> , i32 0, i32 0 +; CHECK-NEXT: [[TMP1:%.*]] = or <8 x i32> zeroinitializer, [[TMP0]] +; CHECK-NEXT: [[TMP2:%.*]] = or <8 x i32> [[TMP1]], zeroinitializer +; CHECK-NEXT: [[TMP3:%.*]] = trunc <8 x i32> [[TMP2]] to <8 x i16> +; CHECK-NEXT: store <8 x i16> [[TMP3]], ptr [[ARRAYIDX2]], align 2 +; CHECK-NEXT: ret void +; +entry: + %conv9 = zext i16 0 to i32 + %arrayidx2 = getelementptr i8, ptr null, i64 16 + %conv310 = zext i16 0 to i32 + %add4 = or i32 %conv310, %conv9 + %sub = or i32 %conv9, %conv310 + %conv15 = sext i16 0 to i32 + %shr = ashr i32 0, 0 + %arrayidx18 = getelementptr i8, ptr null, i64 24 + %conv19 = sext i16 0 to i32 + %sub20 = or i32 %shr, %conv19 + %shr29 = ashr i32 0, 0 + %add30 = or i32 %shr29, %conv15 + %sub39 = or i32 %sub, %sub20 + %conv40 = trunc i32 %sub39 to i16 + store i16 %conv40, ptr %arrayidx2, align 2 + %sub44 = or i32 %add4, %add30 + %conv45 = trunc i32 %sub44 to i16 + store i16 %conv45, ptr %arrayidx18, align 2 + %arrayidx2.1 = getelementptr i8, ptr null, i64 18 + %conv3.112 = zext i16 0 to i32 + %add4.1 = or i32 %conv3.112, 0 + %sub.1 = or i32 0, %conv3.112 + %conv15.1 = sext i16 0 to i32 + %shr.1 = ashr i32 0, 0 + %arrayidx18.1 = getelementptr i8, ptr null, i64 26 + %conv19.1 = sext i16 0 to i32 + %sub20.1 = or i32 %shr.1, %conv19.1 + %shr29.1 = ashr i32 0, 0 + %add30.1 = or i32 %shr29.1, %conv15.1 + %sub39.1 = or i32 %sub.1, %sub20.1 + %conv40.1 = trunc i32 %sub39.1 to i16 + store i16 %conv40.1, ptr %arrayidx2.1, align 2 + %sub44.1 = or i32 %add4.1, %add30.1 + %conv45.1 = trunc i32 %sub44.1 to i16 + store i16 %conv45.1, ptr %arrayidx18.1, align 2 + %conv.213 = zext i16 0 to i32 + %arrayidx2.2 = getelementptr i8, ptr null, i64 20 + %conv3.214 = zext i16 0 to i32 + %add4.2 = or i32 0, %conv.213 + %sub.2 = or i32 0, %conv3.214 + %conv15.2 = sext i16 0 to i32 + %shr.2 = ashr i32 0, 0 + %arrayidx18.2 = getelementptr i8, ptr null, i64 28 + %conv19.2 = sext i16 0 to i32 + %sub20.2 = or i32 %shr.2, %conv19.2 + %shr29.2 = ashr i32 0, 0 + %add30.2 = or i32 %shr29.2, %conv15.2 + %sub39.2 = or i32 %sub.2, %sub20.2 + %conv40.2 = trunc i32 %sub39.2 to i16 + store i16 %conv40.2, ptr %arrayidx2.2, align 2 + %sub44.2 = or i32 %add4.2, %add30.2 + %conv45.2 = trunc i32 %sub44.2 to i16 + store i16 %conv45.2, ptr %arrayidx18.2, align 2 + %conv.315 = zext i16 0 to i32 + %arrayidx2.3 = getelementptr i8, ptr null, i64 22 + %conv3.316 = zext i16 0 to i32 + %add4.3 = or i32 0, %conv.315 + %sub.3 = or i32 0, %conv3.316 + %conv15.3 = sext i16 0 to i32 + %shr.3 = ashr i32 0, 0 + %arrayidx18.3 = getelementptr i8, ptr null, i64 30 + %conv19.3 = sext i16 0 to i32 + %sub20.3 = or i32 %shr.3, %conv19.3 + %shr29.3 = ashr i32 0, 0 + %add30.3 = or i32 %shr29.3, %conv15.3 + %sub39.3 = or i32 %sub.3, %sub20.3 + %conv40.3 = trunc i32 %sub39.3 to i16 + store i16 %conv40.3, ptr %arrayidx2.3, align 2 + %sub44.3 = or i32 %add4.3, %add30.3 + %conv45.3 = trunc i32 %sub44.3 to i16 + store i16 %conv45.3, ptr %arrayidx18.3, align 2 + ret void +} -- GitLab From 0b4688403672264ab451992a3461a0df113c3bd7 Mon Sep 17 00:00:00 2001 From: Usman Nadeem Date: Wed, 13 Mar 2024 11:58:10 -0700 Subject: [PATCH 0174/1407] Revert "Revert "[AArch64] Improve lowering of truncating uzp1"" (#85119) Reverts llvm/llvm-project#85115 The fix was already merged in https://github.com/llvm/llvm-project/commit/79cd2c0bb9acb4685094d6b3bf21c758aa51d3df --- .../Target/AArch64/AArch64ISelLowering.cpp | 39 +-- llvm/lib/Target/AArch64/AArch64InstrInfo.td | 53 ++-- .../CodeGen/AArch64/arm64-convert-v4f64.ll | 21 +- llvm/test/CodeGen/AArch64/extbinopload.ll | 31 ++- .../CodeGen/AArch64/fp-conversion-to-tbl.ll | 5 +- llvm/test/CodeGen/AArch64/fptoi.ll | 256 ++++++------------ llvm/test/CodeGen/AArch64/neon-truncstore.ll | 5 +- llvm/test/CodeGen/AArch64/sadd_sat_vec.ll | 2 +- llvm/test/CodeGen/AArch64/shuffle-tbl34.ll | 14 +- llvm/test/CodeGen/AArch64/ssub_sat_vec.ll | 2 +- llvm/test/CodeGen/AArch64/tbl-loops.ll | 4 +- llvm/test/CodeGen/AArch64/trunc-to-tbl.ll | 28 +- llvm/test/CodeGen/AArch64/uadd_sat_vec.ll | 2 +- llvm/test/CodeGen/AArch64/usub_sat_vec.ll | 2 +- llvm/test/CodeGen/AArch64/vcvt-oversize.ll | 5 +- .../vec-combine-compare-truncate-store.ll | 2 +- .../AArch64/vec3-loads-ext-trunc-stores.ll | 22 +- 17 files changed, 209 insertions(+), 284 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 5b7a36d2eba7..9665ae5ceb90 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -21423,12 +21423,8 @@ static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG, } } - // uzp1(xtn x, xtn y) -> xtn(uzp1 (x, y)) - // Only implemented on little-endian subtargets. - bool IsLittleEndian = DAG.getDataLayout().isLittleEndian(); - - // This optimization only works on little endian. - if (!IsLittleEndian) + // These optimizations only work on little endian. + if (!DAG.getDataLayout().isLittleEndian()) return SDValue(); // uzp1(bitcast(x), bitcast(y)) -> uzp1(x, y) @@ -21447,21 +21443,28 @@ static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG, if (ResVT != MVT::v2i32 && ResVT != MVT::v4i16 && ResVT != MVT::v8i8) return SDValue(); - auto getSourceOp = [](SDValue Operand) -> SDValue { - const unsigned Opcode = Operand.getOpcode(); - if (Opcode == ISD::TRUNCATE) - return Operand->getOperand(0); - if (Opcode == ISD::BITCAST && - Operand->getOperand(0).getOpcode() == ISD::TRUNCATE) - return Operand->getOperand(0)->getOperand(0); - return SDValue(); - }; + SDValue SourceOp0 = peekThroughBitcasts(Op0); + SDValue SourceOp1 = peekThroughBitcasts(Op1); - SDValue SourceOp0 = getSourceOp(Op0); - SDValue SourceOp1 = getSourceOp(Op1); + // truncating uzp1(x, y) -> xtn(concat (x, y)) + if (SourceOp0.getValueType() == SourceOp1.getValueType()) { + EVT Op0Ty = SourceOp0.getValueType(); + if ((ResVT == MVT::v4i16 && Op0Ty == MVT::v2i32) || + (ResVT == MVT::v8i8 && Op0Ty == MVT::v4i16)) { + SDValue Concat = + DAG.getNode(ISD::CONCAT_VECTORS, DL, + Op0Ty.getDoubleNumVectorElementsVT(*DAG.getContext()), + SourceOp0, SourceOp1); + return DAG.getNode(ISD::TRUNCATE, DL, ResVT, Concat); + } + } - if (!SourceOp0 || !SourceOp1) + // uzp1(xtn x, xtn y) -> xtn(uzp1 (x, y)) + if (SourceOp0.getOpcode() != ISD::TRUNCATE || + SourceOp1.getOpcode() != ISD::TRUNCATE) return SDValue(); + SourceOp0 = SourceOp0.getOperand(0); + SourceOp1 = SourceOp1.getOperand(0); if (SourceOp0.getValueType() != SourceOp1.getValueType() || !SourceOp0.getValueType().isSimple()) diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 6254e68326f7..b4b975cce007 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -6153,26 +6153,39 @@ defm UZP2 : SIMDZipVector<0b101, "uzp2", AArch64uzp2>; defm ZIP1 : SIMDZipVector<0b011, "zip1", AArch64zip1>; defm ZIP2 : SIMDZipVector<0b111, "zip2", AArch64zip2>; -def : Pat<(v16i8 (concat_vectors (v8i8 (trunc (v8i16 V128:$Vn))), - (v8i8 (trunc (v8i16 V128:$Vm))))), - (UZP1v16i8 V128:$Vn, V128:$Vm)>; -def : Pat<(v8i16 (concat_vectors (v4i16 (trunc (v4i32 V128:$Vn))), - (v4i16 (trunc (v4i32 V128:$Vm))))), - (UZP1v8i16 V128:$Vn, V128:$Vm)>; -def : Pat<(v4i32 (concat_vectors (v2i32 (trunc (v2i64 V128:$Vn))), - (v2i32 (trunc (v2i64 V128:$Vm))))), - (UZP1v4i32 V128:$Vn, V128:$Vm)>; -// These are the same as above, with an optional assertzext node that can be -// generated from fptoi lowering. -def : Pat<(v16i8 (concat_vectors (v8i8 (assertzext (trunc (v8i16 V128:$Vn)))), - (v8i8 (assertzext (trunc (v8i16 V128:$Vm)))))), - (UZP1v16i8 V128:$Vn, V128:$Vm)>; -def : Pat<(v8i16 (concat_vectors (v4i16 (assertzext (trunc (v4i32 V128:$Vn)))), - (v4i16 (assertzext (trunc (v4i32 V128:$Vm)))))), - (UZP1v8i16 V128:$Vn, V128:$Vm)>; -def : Pat<(v4i32 (concat_vectors (v2i32 (assertzext (trunc (v2i64 V128:$Vn)))), - (v2i32 (assertzext (trunc (v2i64 V128:$Vm)))))), - (UZP1v4i32 V128:$Vn, V128:$Vm)>; +def trunc_optional_assert_ext : PatFrags<(ops node:$op0), + [(trunc node:$op0), + (assertzext (trunc node:$op0)), + (assertsext (trunc node:$op0))]>; + +// concat_vectors(trunc(x), trunc(y)) -> uzp1(x, y) +// concat_vectors(assertzext(trunc(x)), assertzext(trunc(y))) -> uzp1(x, y) +// concat_vectors(assertsext(trunc(x)), assertsext(trunc(y))) -> uzp1(x, y) +class concat_trunc_to_uzp1_pat + : Pat<(ConcatTy (concat_vectors (TruncTy (trunc_optional_assert_ext (SrcTy V128:$Vn))), + (TruncTy (trunc_optional_assert_ext (SrcTy V128:$Vm))))), + (!cast("UZP1"#ConcatTy) V128:$Vn, V128:$Vm)>; +def : concat_trunc_to_uzp1_pat; +def : concat_trunc_to_uzp1_pat; +def : concat_trunc_to_uzp1_pat; + +// trunc(concat_vectors(trunc(x), trunc(y))) -> xtn(uzp1(x, y)) +// trunc(concat_vectors(assertzext(trunc(x)), assertzext(trunc(y)))) -> xtn(uzp1(x, y)) +// trunc(concat_vectors(assertsext(trunc(x)), assertsext(trunc(y)))) -> xtn(uzp1(x, y)) +class trunc_concat_trunc_to_xtn_uzp1_pat + : Pat<(Ty (trunc_optional_assert_ext + (ConcatTy (concat_vectors + (TruncTy (trunc_optional_assert_ext (SrcTy V128:$Vn))), + (TruncTy (trunc_optional_assert_ext (SrcTy V128:$Vm))))))), + (!cast("XTN"#Ty) (!cast("UZP1"#ConcatTy) V128:$Vn, V128:$Vm))>; +def : trunc_concat_trunc_to_xtn_uzp1_pat; +def : trunc_concat_trunc_to_xtn_uzp1_pat; + +def : Pat<(v8i8 (trunc (concat_vectors (v4i16 V64:$Vn), (v4i16 V64:$Vm)))), + (UZP1v8i8 V64:$Vn, V64:$Vm)>; +def : Pat<(v4i16 (trunc (concat_vectors (v2i32 V64:$Vn), (v2i32 V64:$Vm)))), + (UZP1v4i16 V64:$Vn, V64:$Vm)>; def : Pat<(v16i8 (concat_vectors (v8i8 (trunc (AArch64vlshr (v8i16 V128:$Vn), (i32 8)))), diff --git a/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll b/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll index 49325299f74a..3007e7ce771e 100644 --- a/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll +++ b/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll @@ -8,9 +8,8 @@ define <4 x i16> @fptosi_v4f64_to_v4i16(ptr %ptr) { ; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-NEXT: xtn v1.2s, v1.2d -; CHECK-NEXT: xtn v0.2s, v0.2d -; CHECK-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-NEXT: xtn v0.4h, v0.4s ; CHECK-NEXT: ret %tmp1 = load <4 x double>, ptr %ptr %tmp2 = fptosi <4 x double> %tmp1 to <4 x i16> @@ -26,13 +25,10 @@ define <8 x i8> @fptosi_v4f64_to_v4i8(ptr %ptr) { ; CHECK-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-NEXT: fcvtzs v3.2d, v3.2d ; CHECK-NEXT: fcvtzs v2.2d, v2.2d -; CHECK-NEXT: xtn v0.2s, v0.2d -; CHECK-NEXT: xtn v1.2s, v1.2d -; CHECK-NEXT: xtn v3.2s, v3.2d -; CHECK-NEXT: xtn v2.2s, v2.2d -; CHECK-NEXT: uzp1 v0.4h, v1.4h, v0.4h -; CHECK-NEXT: uzp1 v1.4h, v2.4h, v3.4h -; CHECK-NEXT: uzp1 v0.8b, v1.8b, v0.8b +; CHECK-NEXT: uzp1 v0.4s, v1.4s, v0.4s +; CHECK-NEXT: uzp1 v1.4s, v2.4s, v3.4s +; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h +; CHECK-NEXT: xtn v0.8b, v0.8h ; CHECK-NEXT: ret %tmp1 = load <8 x double>, ptr %ptr %tmp2 = fptosi <8 x double> %tmp1 to <8 x i8> @@ -96,9 +92,8 @@ define <4 x i16> @fptoui_v4f64_to_v4i16(ptr %ptr) { ; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-NEXT: xtn v1.2s, v1.2d -; CHECK-NEXT: xtn v0.2s, v0.2d -; CHECK-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-NEXT: xtn v0.4h, v0.4s ; CHECK-NEXT: ret %tmp1 = load <4 x double>, ptr %ptr %tmp2 = fptoui <4 x double> %tmp1 to <4 x i16> diff --git a/llvm/test/CodeGen/AArch64/extbinopload.ll b/llvm/test/CodeGen/AArch64/extbinopload.ll index 1f68c77611e1..dff4831330de 100644 --- a/llvm/test/CodeGen/AArch64/extbinopload.ll +++ b/llvm/test/CodeGen/AArch64/extbinopload.ll @@ -650,7 +650,7 @@ define <16 x i32> @extrause_load(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: add x11, x3, #12 ; CHECK-NEXT: str s1, [x4] ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: ldp s0, s5, [x2] +; CHECK-NEXT: ldp s0, s4, [x2] ; CHECK-NEXT: ushll v2.8h, v0.8b, #0 ; CHECK-NEXT: umov w9, v2.h[0] ; CHECK-NEXT: umov w10, v2.h[1] @@ -662,24 +662,25 @@ define <16 x i32> @extrause_load(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: ushll v2.8h, v2.8b, #0 ; CHECK-NEXT: mov v0.b[10], w9 ; CHECK-NEXT: add x9, x1, #4 -; CHECK-NEXT: uzp1 v1.8b, v1.8b, v2.8b +; CHECK-NEXT: mov v1.d[1], v2.d[0] ; CHECK-NEXT: mov v0.b[11], w10 ; CHECK-NEXT: add x10, x1, #12 +; CHECK-NEXT: bic v1.8h, #255, lsl #8 ; CHECK-NEXT: ld1 { v0.s }[3], [x3], #4 -; CHECK-NEXT: ldr s4, [x0, #12] -; CHECK-NEXT: ldp s3, s16, [x0, #4] -; CHECK-NEXT: ld1 { v5.s }[1], [x3] -; CHECK-NEXT: ldp s6, s7, [x2, #8] -; CHECK-NEXT: ld1 { v4.s }[1], [x10] -; CHECK-NEXT: ld1 { v3.s }[1], [x9] -; CHECK-NEXT: ld1 { v6.s }[1], [x8] -; CHECK-NEXT: ld1 { v7.s }[1], [x11] +; CHECK-NEXT: ldr s3, [x0, #12] +; CHECK-NEXT: ldp s2, s7, [x0, #4] +; CHECK-NEXT: ld1 { v4.s }[1], [x3] +; CHECK-NEXT: ldp s5, s6, [x2, #8] +; CHECK-NEXT: ld1 { v3.s }[1], [x10] +; CHECK-NEXT: ld1 { v2.s }[1], [x9] +; CHECK-NEXT: ld1 { v5.s }[1], [x8] +; CHECK-NEXT: ld1 { v6.s }[1], [x11] ; CHECK-NEXT: add x8, x1, #8 -; CHECK-NEXT: ld1 { v16.s }[1], [x8] -; CHECK-NEXT: uaddl v2.8h, v3.8b, v4.8b -; CHECK-NEXT: ushll v3.8h, v6.8b, #0 -; CHECK-NEXT: uaddl v4.8h, v5.8b, v7.8b -; CHECK-NEXT: uaddl v1.8h, v1.8b, v16.8b +; CHECK-NEXT: ld1 { v7.s }[1], [x8] +; CHECK-NEXT: uaddl v2.8h, v2.8b, v3.8b +; CHECK-NEXT: ushll v3.8h, v5.8b, #0 +; CHECK-NEXT: uaddl v4.8h, v4.8b, v6.8b +; CHECK-NEXT: uaddw v1.8h, v1.8h, v7.8b ; CHECK-NEXT: uaddw2 v5.8h, v3.8h, v0.16b ; CHECK-NEXT: ushll v0.4s, v2.4h, #3 ; CHECK-NEXT: ushll2 v2.4s, v2.8h, #3 diff --git a/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll b/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll index 1ea87bb6b04b..0a3b9a070c2b 100644 --- a/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll +++ b/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll @@ -73,9 +73,8 @@ define void @fptoui_v8f32_to_v8i8_no_loop(ptr %A, ptr %dst) { ; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: fcvtzs.4s v1, v1 ; CHECK-NEXT: fcvtzs.4s v0, v0 -; CHECK-NEXT: xtn.4h v1, v1 -; CHECK-NEXT: xtn.4h v0, v0 -; CHECK-NEXT: uzp1.8b v0, v0, v1 +; CHECK-NEXT: uzp1.8h v0, v0, v1 +; CHECK-NEXT: xtn.8b v0, v0 ; CHECK-NEXT: str d0, [x1] ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/fptoi.ll b/llvm/test/CodeGen/AArch64/fptoi.ll index 67190e8596c4..7af01b53dae7 100644 --- a/llvm/test/CodeGen/AArch64/fptoi.ll +++ b/llvm/test/CodeGen/AArch64/fptoi.ll @@ -1096,30 +1096,17 @@ entry: } define <3 x i16> @fptos_v3f64_v3i16(<3 x double> %a) { -; CHECK-SD-LABEL: fptos_v3f64_v3i16: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2 -; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] -; CHECK-SD-NEXT: fcvtzs v1.2d, v2.2d -; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: fptos_v3f64_v3i16: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2 -; CHECK-GI-NEXT: mov v0.d[1], v1.d[0] -; CHECK-GI-NEXT: fcvtzs v1.2d, v2.2d -; CHECK-GI-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-GI-NEXT: xtn v0.4h, v0.4s -; CHECK-GI-NEXT: ret +; CHECK-LABEL: fptos_v3f64_v3i16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2 +; CHECK-NEXT: mov v0.d[1], v1.d[0] +; CHECK-NEXT: fcvtzs v1.2d, v2.2d +; CHECK-NEXT: fcvtzs v0.2d, v0.2d +; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-NEXT: xtn v0.4h, v0.4s +; CHECK-NEXT: ret entry: %c = fptosi <3 x double> %a to <3 x i16> ret <3 x i16> %c @@ -1134,9 +1121,8 @@ define <3 x i16> @fptou_v3f64_v3i16(<3 x double> %a) { ; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] ; CHECK-SD-NEXT: fcvtzs v1.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v3f64_v3i16: @@ -1160,9 +1146,8 @@ define <4 x i16> @fptos_v4f64_v4i16(<4 x double> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v4f64_v4i16: @@ -1182,9 +1167,8 @@ define <4 x i16> @fptou_v4f64_v4i16(<4 x double> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v4f64_v4i16: @@ -1600,9 +1584,8 @@ define <3 x i8> @fptos_v3f64_v3i8(<3 x double> %a) { ; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] ; CHECK-SD-NEXT: fcvtzs v1.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: umov w0, v0.h[0] ; CHECK-SD-NEXT: umov w1, v0.h[1] ; CHECK-SD-NEXT: umov w2, v0.h[2] @@ -1638,9 +1621,8 @@ define <3 x i8> @fptou_v3f64_v3i8(<3 x double> %a) { ; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] ; CHECK-SD-NEXT: fcvtzs v1.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: umov w0, v0.h[0] ; CHECK-SD-NEXT: umov w1, v0.h[1] ; CHECK-SD-NEXT: umov w2, v0.h[2] @@ -1672,9 +1654,8 @@ define <4 x i8> @fptos_v4f64_v4i8(<4 x double> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v4f64_v4i8: @@ -1694,9 +1675,8 @@ define <4 x i8> @fptou_v4f64_v4i8(<4 x double> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v4f64_v4i8: @@ -1718,13 +1698,10 @@ define <8 x i8> @fptos_v8f64_v8i8(<8 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v2.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v3.2s, v3.2d -; CHECK-SD-NEXT: xtn v2.2s, v2.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v2.8b +; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h +; CHECK-SD-NEXT: xtn v0.8b, v0.8h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v8f64_v8i8: @@ -1750,13 +1727,10 @@ define <8 x i8> @fptou_v8f64_v8i8(<8 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v2.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v3.2s, v3.2d -; CHECK-SD-NEXT: xtn v2.2s, v2.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v2.8b +; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h +; CHECK-SD-NEXT: xtn v0.8b, v0.8h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v8f64_v8i8: @@ -1786,21 +1760,13 @@ define <16 x i8> @fptos_v16f64_v16i8(<16 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v2.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v7.2s, v7.2d -; CHECK-SD-NEXT: xtn v6.2s, v6.2d -; CHECK-SD-NEXT: xtn v5.2s, v5.2d -; CHECK-SD-NEXT: xtn v4.2s, v4.2d -; CHECK-SD-NEXT: xtn v3.2s, v3.2d -; CHECK-SD-NEXT: xtn v2.2s, v2.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v6.4h, v6.4h, v7.4h -; CHECK-SD-NEXT: uzp1 v4.4h, v4.4h, v5.4h -; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: mov v4.d[1], v6.d[0] -; CHECK-SD-NEXT: mov v0.d[1], v2.d[0] -; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v4.16b +; CHECK-SD-NEXT: uzp1 v6.4s, v6.4s, v7.4s +; CHECK-SD-NEXT: uzp1 v4.4s, v4.4s, v5.4s +; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: uzp1 v1.8h, v4.8h, v6.8h +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h +; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v16f64_v16i8: @@ -1837,21 +1803,13 @@ define <16 x i8> @fptou_v16f64_v16i8(<16 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v2.2d, v2.2d ; CHECK-SD-NEXT: fcvtzs v1.2d, v1.2d ; CHECK-SD-NEXT: fcvtzs v0.2d, v0.2d -; CHECK-SD-NEXT: xtn v7.2s, v7.2d -; CHECK-SD-NEXT: xtn v6.2s, v6.2d -; CHECK-SD-NEXT: xtn v5.2s, v5.2d -; CHECK-SD-NEXT: xtn v4.2s, v4.2d -; CHECK-SD-NEXT: xtn v3.2s, v3.2d -; CHECK-SD-NEXT: xtn v2.2s, v2.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: uzp1 v6.4h, v6.4h, v7.4h -; CHECK-SD-NEXT: uzp1 v4.4h, v4.4h, v5.4h -; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: mov v4.d[1], v6.d[0] -; CHECK-SD-NEXT: mov v0.d[1], v2.d[0] -; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v4.16b +; CHECK-SD-NEXT: uzp1 v6.4s, v6.4s, v7.4s +; CHECK-SD-NEXT: uzp1 v4.4s, v4.4s, v5.4s +; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: uzp1 v1.8h, v4.8h, v6.8h +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h +; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v16f64_v16i8: @@ -1900,36 +1858,20 @@ define <32 x i8> @fptos_v32f64_v32i8(<32 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v18.2d, v18.2d ; CHECK-SD-NEXT: fcvtzs v17.2d, v17.2d ; CHECK-SD-NEXT: fcvtzs v16.2d, v16.2d -; CHECK-SD-NEXT: xtn v7.2s, v7.2d -; CHECK-SD-NEXT: xtn v6.2s, v6.2d -; CHECK-SD-NEXT: xtn v5.2s, v5.2d -; CHECK-SD-NEXT: xtn v4.2s, v4.2d -; CHECK-SD-NEXT: xtn v3.2s, v3.2d -; CHECK-SD-NEXT: xtn v2.2s, v2.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: xtn v23.2s, v23.2d -; CHECK-SD-NEXT: xtn v22.2s, v22.2d -; CHECK-SD-NEXT: xtn v21.2s, v21.2d -; CHECK-SD-NEXT: xtn v20.2s, v20.2d -; CHECK-SD-NEXT: xtn v19.2s, v19.2d -; CHECK-SD-NEXT: xtn v18.2s, v18.2d -; CHECK-SD-NEXT: xtn v17.2s, v17.2d -; CHECK-SD-NEXT: xtn v16.2s, v16.2d -; CHECK-SD-NEXT: uzp1 v6.4h, v6.4h, v7.4h -; CHECK-SD-NEXT: uzp1 v4.4h, v4.4h, v5.4h -; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: uzp1 v1.4h, v22.4h, v23.4h -; CHECK-SD-NEXT: uzp1 v3.4h, v20.4h, v21.4h -; CHECK-SD-NEXT: uzp1 v5.4h, v18.4h, v19.4h -; CHECK-SD-NEXT: uzp1 v7.4h, v16.4h, v17.4h -; CHECK-SD-NEXT: mov v4.d[1], v6.d[0] -; CHECK-SD-NEXT: mov v0.d[1], v2.d[0] -; CHECK-SD-NEXT: mov v3.d[1], v1.d[0] -; CHECK-SD-NEXT: mov v7.d[1], v5.d[0] +; CHECK-SD-NEXT: uzp1 v6.4s, v6.4s, v7.4s +; CHECK-SD-NEXT: uzp1 v4.4s, v4.4s, v5.4s +; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: uzp1 v3.4s, v20.4s, v21.4s +; CHECK-SD-NEXT: uzp1 v1.4s, v22.4s, v23.4s +; CHECK-SD-NEXT: uzp1 v5.4s, v18.4s, v19.4s +; CHECK-SD-NEXT: uzp1 v7.4s, v16.4s, v17.4s +; CHECK-SD-NEXT: uzp1 v4.8h, v4.8h, v6.8h +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h +; CHECK-SD-NEXT: uzp1 v1.8h, v3.8h, v1.8h +; CHECK-SD-NEXT: uzp1 v2.8h, v7.8h, v5.8h ; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v4.16b -; CHECK-SD-NEXT: uzp1 v1.16b, v7.16b, v3.16b +; CHECK-SD-NEXT: uzp1 v1.16b, v2.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v32f64_v32i8: @@ -1997,36 +1939,20 @@ define <32 x i8> @fptou_v32f64_v32i8(<32 x double> %a) { ; CHECK-SD-NEXT: fcvtzs v18.2d, v18.2d ; CHECK-SD-NEXT: fcvtzs v17.2d, v17.2d ; CHECK-SD-NEXT: fcvtzs v16.2d, v16.2d -; CHECK-SD-NEXT: xtn v7.2s, v7.2d -; CHECK-SD-NEXT: xtn v6.2s, v6.2d -; CHECK-SD-NEXT: xtn v5.2s, v5.2d -; CHECK-SD-NEXT: xtn v4.2s, v4.2d -; CHECK-SD-NEXT: xtn v3.2s, v3.2d -; CHECK-SD-NEXT: xtn v2.2s, v2.2d -; CHECK-SD-NEXT: xtn v1.2s, v1.2d -; CHECK-SD-NEXT: xtn v0.2s, v0.2d -; CHECK-SD-NEXT: xtn v23.2s, v23.2d -; CHECK-SD-NEXT: xtn v22.2s, v22.2d -; CHECK-SD-NEXT: xtn v21.2s, v21.2d -; CHECK-SD-NEXT: xtn v20.2s, v20.2d -; CHECK-SD-NEXT: xtn v19.2s, v19.2d -; CHECK-SD-NEXT: xtn v18.2s, v18.2d -; CHECK-SD-NEXT: xtn v17.2s, v17.2d -; CHECK-SD-NEXT: xtn v16.2s, v16.2d -; CHECK-SD-NEXT: uzp1 v6.4h, v6.4h, v7.4h -; CHECK-SD-NEXT: uzp1 v4.4h, v4.4h, v5.4h -; CHECK-SD-NEXT: uzp1 v2.4h, v2.4h, v3.4h -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: uzp1 v1.4h, v22.4h, v23.4h -; CHECK-SD-NEXT: uzp1 v3.4h, v20.4h, v21.4h -; CHECK-SD-NEXT: uzp1 v5.4h, v18.4h, v19.4h -; CHECK-SD-NEXT: uzp1 v7.4h, v16.4h, v17.4h -; CHECK-SD-NEXT: mov v4.d[1], v6.d[0] -; CHECK-SD-NEXT: mov v0.d[1], v2.d[0] -; CHECK-SD-NEXT: mov v3.d[1], v1.d[0] -; CHECK-SD-NEXT: mov v7.d[1], v5.d[0] +; CHECK-SD-NEXT: uzp1 v6.4s, v6.4s, v7.4s +; CHECK-SD-NEXT: uzp1 v4.4s, v4.4s, v5.4s +; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s +; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: uzp1 v3.4s, v20.4s, v21.4s +; CHECK-SD-NEXT: uzp1 v1.4s, v22.4s, v23.4s +; CHECK-SD-NEXT: uzp1 v5.4s, v18.4s, v19.4s +; CHECK-SD-NEXT: uzp1 v7.4s, v16.4s, v17.4s +; CHECK-SD-NEXT: uzp1 v4.8h, v4.8h, v6.8h +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h +; CHECK-SD-NEXT: uzp1 v1.8h, v3.8h, v1.8h +; CHECK-SD-NEXT: uzp1 v2.8h, v7.8h, v5.8h ; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v4.16b -; CHECK-SD-NEXT: uzp1 v1.16b, v7.16b, v3.16b +; CHECK-SD-NEXT: uzp1 v1.16b, v2.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v32f64_v32i8: @@ -3026,9 +2952,8 @@ define <8 x i8> @fptos_v8f32_v8i8(<8 x float> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.4s, v1.4s ; CHECK-SD-NEXT: fcvtzs v0.4s, v0.4s -; CHECK-SD-NEXT: xtn v1.4h, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s -; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v1.8b +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h +; CHECK-SD-NEXT: xtn v0.8b, v0.8h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v8f32_v8i8: @@ -3048,9 +2973,8 @@ define <8 x i8> @fptou_v8f32_v8i8(<8 x float> %a) { ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fcvtzs v1.4s, v1.4s ; CHECK-SD-NEXT: fcvtzs v0.4s, v0.4s -; CHECK-SD-NEXT: xtn v1.4h, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s -; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v1.8b +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h +; CHECK-SD-NEXT: xtn v0.8b, v0.8h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptou_v8f32_v8i8: @@ -3072,12 +2996,8 @@ define <16 x i8> @fptos_v16f32_v16i8(<16 x float> %a) { ; CHECK-SD-NEXT: fcvtzs v2.4s, v2.4s ; CHECK-SD-NEXT: fcvtzs v1.4s, v1.4s ; CHECK-SD-NEXT: fcvtzs v0.4s, v0.4s -; CHECK-SD-NEXT: xtn v3.4h, v3.4s -; CHECK-SD-NEXT: xtn v2.4h, v2.4s -; CHECK-SD-NEXT: xtn v1.4h, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s -; CHECK-SD-NEXT: mov v2.d[1], v3.d[0] -; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] +; CHECK-SD-NEXT: uzp1 v2.8h, v2.8h, v3.8h +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h ; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: ret ; @@ -3134,20 +3054,12 @@ define <32 x i8> @fptos_v32f32_v32i8(<32 x float> %a) { ; CHECK-SD-NEXT: fcvtzs v6.4s, v6.4s ; CHECK-SD-NEXT: fcvtzs v5.4s, v5.4s ; CHECK-SD-NEXT: fcvtzs v4.4s, v4.4s -; CHECK-SD-NEXT: xtn v3.4h, v3.4s -; CHECK-SD-NEXT: xtn v2.4h, v2.4s -; CHECK-SD-NEXT: xtn v1.4h, v1.4s -; CHECK-SD-NEXT: xtn v0.4h, v0.4s -; CHECK-SD-NEXT: xtn v7.4h, v7.4s -; CHECK-SD-NEXT: xtn v6.4h, v6.4s -; CHECK-SD-NEXT: xtn v5.4h, v5.4s -; CHECK-SD-NEXT: xtn v4.4h, v4.4s -; CHECK-SD-NEXT: mov v2.d[1], v3.d[0] -; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] -; CHECK-SD-NEXT: mov v6.d[1], v7.d[0] -; CHECK-SD-NEXT: mov v4.d[1], v5.d[0] +; CHECK-SD-NEXT: uzp1 v2.8h, v2.8h, v3.8h +; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h +; CHECK-SD-NEXT: uzp1 v1.8h, v6.8h, v7.8h +; CHECK-SD-NEXT: uzp1 v3.8h, v4.8h, v5.8h ; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v2.16b -; CHECK-SD-NEXT: uzp1 v1.16b, v4.16b, v6.16b +; CHECK-SD-NEXT: uzp1 v1.16b, v3.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fptos_v32f32_v32i8: diff --git a/llvm/test/CodeGen/AArch64/neon-truncstore.ll b/llvm/test/CodeGen/AArch64/neon-truncstore.ll index b677d077b98c..5d78ad24eb33 100644 --- a/llvm/test/CodeGen/AArch64/neon-truncstore.ll +++ b/llvm/test/CodeGen/AArch64/neon-truncstore.ll @@ -104,7 +104,7 @@ define void @v4i32_v4i8(<4 x i32> %a, ptr %result) { ; CHECK-LABEL: v4i32_v4i8: ; CHECK: // %bb.0: ; CHECK-NEXT: xtn v0.4h, v0.4s -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: str s0, [x0] ; CHECK-NEXT: ret %b = trunc <4 x i32> %a to <4 x i8> @@ -170,8 +170,7 @@ define void @v2i16_v2i8(<2 x i16> %a, ptr %result) { define void @v4i16_v4i8(<4 x i16> %a, ptr %result) { ; CHECK-LABEL: v4i16_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: str s0, [x0] ; CHECK-NEXT: ret %b = trunc <4 x i16> %a to <4 x i8> diff --git a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll index 5f905d94e357..6f1ae023bf25 100644 --- a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll @@ -145,7 +145,7 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-NEXT: shl v0.4h, v0.4h, #8 ; CHECK-NEXT: sqadd v0.4h, v0.4h, v1.4h ; CHECK-NEXT: sshr v0.4h, v0.4h, #8 -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: str s0, [x2] ; CHECK-NEXT: ret %x = load <4 x i8>, ptr %px diff --git a/llvm/test/CodeGen/AArch64/shuffle-tbl34.ll b/llvm/test/CodeGen/AArch64/shuffle-tbl34.ll index 0ef64789ad97..fb571eff39fe 100644 --- a/llvm/test/CodeGen/AArch64/shuffle-tbl34.ll +++ b/llvm/test/CodeGen/AArch64/shuffle-tbl34.ll @@ -353,13 +353,17 @@ define <8 x i8> @shuffle4_v8i8_v8i8(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x define <8 x i16> @shuffle4_v4i8_zext(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) { ; CHECK-LABEL: shuffle4_v4i8_zext: ; CHECK: // %bb.0: -; CHECK-NEXT: uzp1 v0.8b, v0.8b, v1.8b -; CHECK-NEXT: uzp1 v1.8b, v2.8b, v3.8b +; CHECK-NEXT: fmov d5, d2 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3 ; CHECK-NEXT: adrp x8, .LCPI8_0 -; CHECK-NEXT: ushll v2.8h, v0.8b, #0 +; CHECK-NEXT: fmov d4, d0 ; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI8_0] -; CHECK-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b +; CHECK-NEXT: mov v4.d[1], v1.d[0] +; CHECK-NEXT: mov v5.d[1], v3.d[0] +; CHECK-NEXT: bic v4.8h, #255, lsl #8 +; CHECK-NEXT: bic v5.8h, #255, lsl #8 +; CHECK-NEXT: tbl v0.16b, { v4.16b, v5.16b }, v0.16b ; CHECK-NEXT: ret %x = shufflevector <4 x i8> %a, <4 x i8> %b, <8 x i32> %y = shufflevector <4 x i8> %c, <4 x i8> %d, <8 x i32> diff --git a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll index acec3e74d3e9..d1f843a09f74 100644 --- a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll @@ -146,7 +146,7 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-NEXT: shl v0.4h, v0.4h, #8 ; CHECK-NEXT: sqsub v0.4h, v0.4h, v1.4h ; CHECK-NEXT: sshr v0.4h, v0.4h, #8 -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: str s0, [x2] ; CHECK-NEXT: ret %x = load <4 x i8>, ptr %px diff --git a/llvm/test/CodeGen/AArch64/tbl-loops.ll b/llvm/test/CodeGen/AArch64/tbl-loops.ll index 4f8a4f7aede3..0ad990086551 100644 --- a/llvm/test/CodeGen/AArch64/tbl-loops.ll +++ b/llvm/test/CodeGen/AArch64/tbl-loops.ll @@ -41,8 +41,8 @@ define void @loop1(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n ; CHECK-NEXT: fcvtzs v2.4s, v2.4s ; CHECK-NEXT: xtn v1.4h, v1.4s ; CHECK-NEXT: xtn v2.4h, v2.4s -; CHECK-NEXT: xtn v1.8b, v1.8h -; CHECK-NEXT: xtn v2.8b, v2.8h +; CHECK-NEXT: uzp1 v1.8b, v1.8b, v0.8b +; CHECK-NEXT: uzp1 v2.8b, v2.8b, v0.8b ; CHECK-NEXT: mov v1.s[1], v2.s[0] ; CHECK-NEXT: stur d1, [x12, #-4] ; CHECK-NEXT: add x12, x12, #8 diff --git a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll index ba367b0dbfde..18cd4cc2111a 100644 --- a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll +++ b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll @@ -710,23 +710,23 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) { ; CHECK-NEXT: LBB6_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldp q4, q1, [x0, #48] -; CHECK-NEXT: add x9, x1, #8 -; CHECK-NEXT: ldp q3, q2, [x0] -; CHECK-NEXT: subs x8, x8, #1 +; CHECK-NEXT: add x9, x1, #10 ; CHECK-NEXT: ldr d0, [x0, #80] +; CHECK-NEXT: ldp q3, q2, [x0] ; CHECK-NEXT: ldr q5, [x0, #32] +; CHECK-NEXT: subs x8, x8, #1 ; CHECK-NEXT: add x0, x0, #128 -; CHECK-NEXT: uzp1.4s v4, v5, v4 -; CHECK-NEXT: uzp1.4s v2, v3, v2 ; CHECK-NEXT: uzp1.4s v0, v1, v0 -; CHECK-NEXT: uzp1.8h v1, v2, v4 +; CHECK-NEXT: uzp1.4s v1, v5, v4 +; CHECK-NEXT: uzp1.4s v2, v3, v2 ; CHECK-NEXT: xtn.4h v0, v0 -; CHECK-NEXT: uzp1.16b v1, v1, v0 -; CHECK-NEXT: xtn.8b v0, v0 -; CHECK-NEXT: st1.h { v1 }[4], [x9] -; CHECK-NEXT: add x9, x1, #10 -; CHECK-NEXT: st1.b { v0 }[2], [x9] -; CHECK-NEXT: str d1, [x1], #16 +; CHECK-NEXT: uzp1.8h v1, v2, v1 +; CHECK-NEXT: uzp1.8b v2, v0, v0 +; CHECK-NEXT: uzp1.16b v0, v1, v0 +; CHECK-NEXT: st1.b { v2 }[2], [x9] +; CHECK-NEXT: add x9, x1, #8 +; CHECK-NEXT: st1.h { v0 }[4], [x9] +; CHECK-NEXT: str d0, [x1], #16 ; CHECK-NEXT: b.eq LBB6_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret @@ -755,7 +755,7 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) { ; CHECK-BE-NEXT: xtn v0.4h, v0.4s ; CHECK-BE-NEXT: uzp1 v1.8h, v1.8h, v2.8h ; CHECK-BE-NEXT: uzp1 v1.16b, v1.16b, v0.16b -; CHECK-BE-NEXT: xtn v0.8b, v0.8h +; CHECK-BE-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-BE-NEXT: rev16 v2.16b, v1.16b ; CHECK-BE-NEXT: rev64 v1.16b, v1.16b ; CHECK-BE-NEXT: st1 { v0.b }[2], [x9] @@ -790,7 +790,7 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) { ; CHECK-DISABLE-NEXT: xtn v0.4h, v0.4s ; CHECK-DISABLE-NEXT: uzp1 v1.8h, v1.8h, v2.8h ; CHECK-DISABLE-NEXT: uzp1 v1.16b, v1.16b, v0.16b -; CHECK-DISABLE-NEXT: xtn v0.8b, v0.8h +; CHECK-DISABLE-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-DISABLE-NEXT: rev16 v2.16b, v1.16b ; CHECK-DISABLE-NEXT: rev64 v1.16b, v1.16b ; CHECK-DISABLE-NEXT: st1 { v0.b }[2], [x9] diff --git a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll index e05c65daf50a..f0bbed59405e 100644 --- a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll @@ -142,7 +142,7 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-NEXT: movi d0, #0xff00ff00ff00ff ; CHECK-NEXT: uaddl v1.8h, v1.8b, v2.8b ; CHECK-NEXT: umin v0.4h, v1.4h, v0.4h -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: str s0, [x2] ; CHECK-NEXT: ret %x = load <4 x i8>, ptr %px diff --git a/llvm/test/CodeGen/AArch64/usub_sat_vec.ll b/llvm/test/CodeGen/AArch64/usub_sat_vec.ll index 05f43e7d8427..82c0327219f5 100644 --- a/llvm/test/CodeGen/AArch64/usub_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/usub_sat_vec.ll @@ -143,7 +143,7 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 ; CHECK-NEXT: uqsub v0.4h, v0.4h, v1.4h -; CHECK-NEXT: xtn v0.8b, v0.8h +; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-NEXT: str s0, [x2] ; CHECK-NEXT: ret %x = load <4 x i8>, ptr %px diff --git a/llvm/test/CodeGen/AArch64/vcvt-oversize.ll b/llvm/test/CodeGen/AArch64/vcvt-oversize.ll index 380bdbcc7f74..611940546bc1 100644 --- a/llvm/test/CodeGen/AArch64/vcvt-oversize.ll +++ b/llvm/test/CodeGen/AArch64/vcvt-oversize.ll @@ -9,9 +9,8 @@ define <8 x i8> @float_to_i8(ptr %in) { ; CHECK-NEXT: fadd v0.4s, v0.4s, v0.4s ; CHECK-NEXT: fcvtzs v0.4s, v0.4s ; CHECK-NEXT: fcvtzs v1.4s, v1.4s -; CHECK-NEXT: xtn v0.4h, v0.4s -; CHECK-NEXT: xtn v1.4h, v1.4s -; CHECK-NEXT: uzp1 v0.8b, v1.8b, v0.8b +; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h +; CHECK-NEXT: xtn v0.8b, v0.8h ; CHECK-NEXT: ret %l = load <8 x float>, ptr %in %scale = fmul <8 x float> %l, diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll index 9c6ab8da0fa7..dd7a9c6d7768 100644 --- a/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll +++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll @@ -210,7 +210,7 @@ define void @no_combine_for_non_bool_truncate(<4 x i32> %vec, ptr %out) { ; CHECK-LABEL: no_combine_for_non_bool_truncate: ; CHECK: ; %bb.0: ; CHECK-NEXT: xtn.4h v0, v0 -; CHECK-NEXT: xtn.8b v0, v0 +; CHECK-NEXT: uzp1.8b v0, v0, v0 ; CHECK-NEXT: str s0, [x0] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll index 90328f73f86b..71d55df66517 100644 --- a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll +++ b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll @@ -410,7 +410,7 @@ define void @store_trunc_from_64bits(ptr %src, ptr %dst) { ; BE-NEXT: ldrh w8, [x0, #4] ; BE-NEXT: rev32 v0.4h, v0.4h ; BE-NEXT: mov v0.h[2], w8 -; BE-NEXT: xtn v0.8b, v0.8h +; BE-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; BE-NEXT: rev32 v0.16b, v0.16b ; BE-NEXT: str s0, [sp, #12] ; BE-NEXT: ldrh w9, [sp, #12] @@ -456,7 +456,7 @@ define void @store_trunc_add_from_64bits(ptr %src, ptr %dst) { ; BE-NEXT: add x8, x8, :lo12:.LCPI11_0 ; BE-NEXT: ld1 { v1.4h }, [x8] ; BE-NEXT: add v0.4h, v0.4h, v1.4h -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -638,7 +638,7 @@ define void @shift_trunc_store(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -672,7 +672,7 @@ define void @shift_trunc_store_default_align(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -706,7 +706,7 @@ define void @shift_trunc_store_align_4(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -741,7 +741,7 @@ define void @shift_trunc_store_const_offset_1(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -777,7 +777,7 @@ define void @shift_trunc_store_const_offset_3(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -801,7 +801,7 @@ define void @shift_trunc_volatile_store(ptr %src, ptr %dst) { ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: shrn.4h v0, v0, #16 -; CHECK-NEXT: xtn.8b v1, v0 +; CHECK-NEXT: uzp1.8b v1, v0, v0 ; CHECK-NEXT: umov.h w8, v0[2] ; CHECK-NEXT: str s1, [sp, #12] ; CHECK-NEXT: ldrh w9, [sp, #12] @@ -816,7 +816,7 @@ define void @shift_trunc_volatile_store(ptr %src, ptr %dst) { ; BE-NEXT: .cfi_def_cfa_offset 16 ; BE-NEXT: ld1 { v0.4s }, [x0] ; BE-NEXT: shrn v0.4h, v0.4s, #16 -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #12] @@ -868,7 +868,7 @@ define void @load_v3i8_zext_to_3xi32_add_trunc_store(ptr %src) { ; BE-NEXT: ushll v0.8h, v0.8b, #0 ; BE-NEXT: ld1 { v0.b }[4], [x9] ; BE-NEXT: add v0.4h, v0.4h, v1.4h -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #8] @@ -921,7 +921,7 @@ define void @load_v3i8_sext_to_3xi32_add_trunc_store(ptr %src) { ; BE-NEXT: ushll v0.8h, v0.8b, #0 ; BE-NEXT: ld1 { v0.b }[4], [x9] ; BE-NEXT: add v0.4h, v0.4h, v1.4h -; BE-NEXT: xtn v1.8b, v0.8h +; BE-NEXT: uzp1 v1.8b, v0.8b, v0.8b ; BE-NEXT: umov w8, v0.h[2] ; BE-NEXT: rev32 v1.16b, v1.16b ; BE-NEXT: str s1, [sp, #8] -- GitLab From 3d45d8bc70d437283f8afe422011420d0fe6533e Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 13 Mar 2024 12:03:45 -0700 Subject: [PATCH 0175/1407] [SLP][NFC]Add a test with the operand node, not being in MinBWs, though user is in. --- .../AArch64/user-node-not-in-bitwidths.ll | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll new file mode 100644 index 000000000000..6404cf4a2cd1 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll @@ -0,0 +1,83 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s + +define void @h() { +; CHECK-LABEL: define void @h() { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr i8, ptr null, i64 16 +; CHECK-NEXT: store <8 x i16> zeroinitializer, ptr [[ARRAYIDX2]], align 2 +; CHECK-NEXT: ret void +; +entry: + %arrayidx2 = getelementptr i8, ptr null, i64 16 + %conv310 = zext i16 0 to i32 + %add4 = or i32 %conv310, 0 + %sub = or i32 0, %conv310 + %conv15 = sext i16 0 to i32 + %shr = ashr i32 %conv15, 0 + %arrayidx18 = getelementptr i8, ptr null, i64 24 + %conv19 = sext i16 0 to i32 + %sub20 = or i32 %shr, 0 + %shr29 = ashr i32 %conv19, 0 + %add30 = or i32 %shr29, %conv15 + %sub39 = or i32 %sub, %sub20 + %conv40 = trunc i32 %sub39 to i16 + store i16 %conv40, ptr %arrayidx2, align 2 + %sub44 = or i32 %add4, %add30 + %conv45 = trunc i32 %sub44 to i16 + store i16 %conv45, ptr %arrayidx18, align 2 + %arrayidx2.1 = getelementptr i8, ptr null, i64 18 + %conv3.112 = zext i16 0 to i32 + %add4.1 = or i32 %conv3.112, 0 + %sub.1 = or i32 0, %conv3.112 + %conv15.1 = sext i16 0 to i32 + %shr.1 = ashr i32 %conv15.1, 0 + %arrayidx18.1 = getelementptr i8, ptr null, i64 26 + %conv19.1 = sext i16 0 to i32 + %sub20.1 = or i32 %shr.1, 0 + %shr29.1 = ashr i32 %conv19.1, 0 + %add30.1 = or i32 %shr29.1, 0 + %sub39.1 = or i32 %sub.1, %sub20.1 + %conv40.1 = trunc i32 %sub39.1 to i16 + store i16 %conv40.1, ptr %arrayidx2.1, align 2 + %sub44.1 = or i32 %add4.1, %add30.1 + %conv45.1 = trunc i32 %sub44.1 to i16 + store i16 %conv45.1, ptr %arrayidx18.1, align 2 + %conv.213 = zext i16 0 to i32 + %arrayidx2.2 = getelementptr i8, ptr null, i64 20 + %conv3.214 = zext i16 0 to i32 + %add4.2 = or i32 0, %conv.213 + %sub.2 = or i32 0, %conv3.214 + %conv15.2 = sext i16 0 to i32 + %shr.2 = ashr i32 %conv15.2, 0 + %arrayidx18.2 = getelementptr i8, ptr null, i64 28 + %conv19.2 = sext i16 0 to i32 + %sub20.2 = or i32 %shr.2, 0 + %shr29.2 = ashr i32 %conv19.2, 0 + %add30.2 = or i32 %shr29.2, 0 + %sub39.2 = or i32 %sub.2, %sub20.2 + %conv40.2 = trunc i32 %sub39.2 to i16 + store i16 %conv40.2, ptr %arrayidx2.2, align 2 + %sub44.2 = or i32 %add4.2, %add30.2 + %conv45.2 = trunc i32 %sub44.2 to i16 + store i16 %conv45.2, ptr %arrayidx18.2, align 2 + %conv.315 = zext i16 0 to i32 + %arrayidx2.3 = getelementptr i8, ptr null, i64 22 + %conv3.316 = zext i16 0 to i32 + %add4.3 = or i32 0, %conv.315 + %sub.3 = or i32 0, %conv3.316 + %conv15.3 = sext i16 0 to i32 + %shr.3 = ashr i32 %conv15.3, 0 + %arrayidx18.3 = getelementptr i8, ptr null, i64 30 + %conv19.3 = sext i16 0 to i32 + %sub20.3 = or i32 %shr.3, 0 + %shr29.3 = ashr i32 %conv19.3, 0 + %add30.3 = or i32 %shr29.3, 0 + %sub39.3 = or i32 %sub.3, %sub20.3 + %conv40.3 = trunc i32 %sub39.3 to i16 + store i16 %conv40.3, ptr %arrayidx2.3, align 2 + %sub44.3 = or i32 %add4.3, %add30.3 + %conv45.3 = trunc i32 %sub44.3 to i16 + store i16 %conv45.3, ptr %arrayidx18.3, align 2 + ret void +} -- GitLab From b77c079987182748fe1746466a74633cfe057cc1 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 13 Mar 2024 12:10:40 -0700 Subject: [PATCH 0176/1407] Revert "[SLP]Fix PR85082: PHI node has multiple entries." This reverts commit 59ff907fc14aa2d02e57b4af4140949d4f8caca1 to fix crash revealed in https://lab.llvm.org/buildbot/#/builders/198/builds/8881 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 10 +-- .../X86/same-scalar-in-same-phi-extract.ll | 75 ------------------- 2 files changed, 5 insertions(+), 80 deletions(-) delete mode 100644 llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 5e0f5b7efadc..b8b67609d755 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -12592,11 +12592,6 @@ Value *BoUpSLP::vectorizeTree( } else { Ex = Builder.CreateExtractElement(Vec, Lane); } - // If necessary, sign-extend or zero-extend ScalarRoot - // to the larger type. - if (Scalar->getType() != Ex->getType()) - Ex = Builder.CreateIntCast(Ex, Scalar->getType(), - MinBWs.find(E)->second.second); if (auto *I = dyn_cast(Ex)) ScalarToEEs[Scalar].try_emplace(Builder.GetInsertBlock(), I); } @@ -12606,6 +12601,11 @@ Value *BoUpSLP::vectorizeTree( GatherShuffleExtractSeq.insert(ExI); CSEBlocks.insert(ExI->getParent()); } + // If necessary, sign-extend or zero-extend ScalarRoot + // to the larger type. + if (Scalar->getType() != Ex->getType()) + return Builder.CreateIntCast(Ex, Scalar->getType(), + MinBWs.find(E)->second.second); return Ex; } assert(isa(Scalar->getType()) && diff --git a/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll b/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll deleted file mode 100644 index 35f2f9e052e7..000000000000 --- a/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll +++ /dev/null @@ -1,75 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 -; RUN: opt -S --passes=slp-vectorizer -slp-threshold=-99999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s - -define void @test(i32 %arg) { -; CHECK-LABEL: define void @test( -; CHECK-SAME: i32 [[ARG:%.*]]) { -; CHECK-NEXT: bb: -; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> , i32 [[ARG]], i32 0 -; CHECK-NEXT: br label [[BB2:%.*]] -; CHECK: bb2: -; CHECK-NEXT: switch i32 0, label [[BB10:%.*]] [ -; CHECK-NEXT: i32 0, label [[BB9:%.*]] -; CHECK-NEXT: i32 11, label [[BB9]] -; CHECK-NEXT: i32 1, label [[BB4:%.*]] -; CHECK-NEXT: ] -; CHECK: bb3: -; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0 -; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 -; CHECK-NEXT: switch i32 0, label [[BB10]] [ -; CHECK-NEXT: i32 18, label [[BB7:%.*]] -; CHECK-NEXT: i32 1, label [[BB7]] -; CHECK-NEXT: i32 0, label [[BB10]] -; CHECK-NEXT: ] -; CHECK: bb4: -; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x i32> [ [[TMP0]], [[BB2]] ] -; CHECK-NEXT: [[TMP4:%.*]] = zext <2 x i32> [[TMP3]] to <2 x i64> -; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0 -; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr null, i64 [[TMP5]] -; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1 -; CHECK-NEXT: [[GETELEMENTPTR6:%.*]] = getelementptr i32, ptr null, i64 [[TMP6]] -; CHECK-NEXT: ret void -; CHECK: bb7: -; CHECK-NEXT: [[PHI8:%.*]] = phi i64 [ [[TMP2]], [[BB3:%.*]] ], [ [[TMP2]], [[BB3]] ] -; CHECK-NEXT: br label [[BB9]] -; CHECK: bb9: -; CHECK-NEXT: ret void -; CHECK: bb10: -; CHECK-NEXT: ret void -; -bb: - %zext = zext i32 %arg to i64 - %zext1 = zext i32 0 to i64 - br label %bb2 - -bb2: - switch i32 0, label %bb10 [ - i32 0, label %bb9 - i32 11, label %bb9 - i32 1, label %bb4 - ] - -bb3: - switch i32 0, label %bb10 [ - i32 18, label %bb7 - i32 1, label %bb7 - i32 0, label %bb10 - ] - -bb4: - %phi = phi i64 [ %zext, %bb2 ] - %phi5 = phi i64 [ %zext1, %bb2 ] - %getelementptr = getelementptr i32, ptr null, i64 %phi - %getelementptr6 = getelementptr i32, ptr null, i64 %phi5 - ret void - -bb7: - %phi8 = phi i64 [ %zext, %bb3 ], [ %zext, %bb3 ] - br label %bb9 - -bb9: - ret void - -bb10: - ret void -} -- GitLab From aa68e2814d9a4bad21e4def900152b2e78e25e98 Mon Sep 17 00:00:00 2001 From: Kolya Panchenko <87679760+nikolaypanchenko@users.noreply.github.com> Date: Wed, 13 Mar 2024 12:18:51 -0700 Subject: [PATCH 0177/1407] [RISCV] Support `llvm.masked.compressstore` intrinsic (#83457) The changeset enables lowering of `llvm.masked.compressstore(%data, %ptr, %mask)` for RVV for fixed vector type into: ``` %0 = vcompress %data, %mask, %vl %new_vl = vcpop %mask, %vl vse %0, %ptr, %1, %new_vl ``` Such lowering is only possible when `%data` fits into available LMULs and otherwise `llvm.masked.compressstore` is scalarized by `ScalarizeMaskedMemIntrin` pass. Even though RVV spec in the section `15.8` provide alternative sequence for compressstore, use of `vcompress + vcpop` should be a proper canonical form to lower `llvm.masked.compressstore`. If RISC-V target find the sequence from `15.8` better, peephole optimization can transform `vcompress + vcpop` into that sequence. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 16 +- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 10 + .../Target/RISCV/RISCVTargetTransformInfo.h | 2 + llvm/test/CodeGen/RISCV/rvv/compressstore.ll | 871 ++++++++++++++ .../rvv/fixed-vectors-compressstore-fp.ll | 1004 ++--------------- .../rvv/fixed-vectors-compressstore-int.ll | 928 ++------------- 6 files changed, 1085 insertions(+), 1746 deletions(-) create mode 100644 llvm/test/CodeGen/RISCV/rvv/compressstore.ll diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 08678a859ae2..803774fd16db 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -10466,6 +10466,7 @@ SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op, SDValue BasePtr = MemSD->getBasePtr(); SDValue Val, Mask, VL; + bool IsCompressingStore = false; if (const auto *VPStore = dyn_cast(Op)) { Val = VPStore->getValue(); Mask = VPStore->getMask(); @@ -10474,9 +10475,11 @@ SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op, const auto *MStore = cast(Op); Val = MStore->getValue(); Mask = MStore->getMask(); + IsCompressingStore = MStore->isCompressingStore(); } - bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); + bool IsUnmasked = + ISD::isConstantSplatVectorAllOnes(Mask.getNode()) || IsCompressingStore; MVT VT = Val.getSimpleValueType(); MVT XLenVT = Subtarget.getXLenVT(); @@ -10486,7 +10489,7 @@ SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op, ContainerVT = getContainerForFixedLengthVector(VT); Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget); - if (!IsUnmasked) { + if (!IsUnmasked || IsCompressingStore) { MVT MaskVT = getMaskTypeFor(ContainerVT); Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); } @@ -10495,6 +10498,15 @@ SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op, if (!VL) VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; + if (IsCompressingStore) { + Val = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT, + DAG.getConstant(Intrinsic::riscv_vcompress, DL, XLenVT), + DAG.getUNDEF(ContainerVT), Val, Mask, VL); + VL = + DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Mask, + getAllOnesMask(Mask.getSimpleValueType(), VL, DL, DAG), VL); + } + unsigned IntID = IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask; SmallVector Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)}; diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index ecd373649e2c..8f46fdc2f7ca 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1620,3 +1620,13 @@ bool RISCVTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1, C2.NumIVMuls, C2.NumBaseAdds, C2.ScaleCost, C2.ImmCost, C2.SetupCost); } + +bool RISCVTTIImpl::isLegalMaskedCompressStore(Type *DataTy, Align Alignment) { + auto *VTy = dyn_cast(DataTy); + if (!VTy || VTy->isScalableTy()) + return false; + + if (!isLegalMaskedLoadStore(DataTy, Alignment)) + return false; + return true; +} diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h index af36e9d5d5e8..8daf6845dc8b 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h @@ -261,6 +261,8 @@ public: return TLI->isLegalStridedLoadStore(DataTypeVT, Alignment); } + bool isLegalMaskedCompressStore(Type *DataTy, Align Alignment); + bool isVScaleKnownToBeAPowerOfTwo() const { return TLI->isVScaleKnownToBeAPowerOfTwo(); } diff --git a/llvm/test/CodeGen/RISCV/rvv/compressstore.ll b/llvm/test/CodeGen/RISCV/rvv/compressstore.ll new file mode 100644 index 000000000000..673008d9c0b3 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/compressstore.ll @@ -0,0 +1,871 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -verify-machineinstrs -mtriple=riscv64 -mattr=+v,+d,+m,+zbb %s -o - | FileCheck %s --check-prefix=RV64 +; RUN: llc -verify-machineinstrs -mtriple=riscv32 -mattr=+v,+d,+m,+zbb %s -o - | FileCheck %s --check-prefix=RV32 + +; Compress + store for i8 type + +define void @test_compresstore_v1i8(ptr %p, <1 x i1> %mask, <1 x i8> %data) { +; RV64-LABEL: test_compresstore_v1i8: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 1, e8, mf8, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; RV64-NEXT: vse8.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v1i8: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 1, e8, mf8, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; RV32-NEXT: vse8.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v1i8(<1 x i8> %data, ptr align 1 %p, <1 x i1> %mask) + ret void +} + +define void @test_compresstore_v2i8(ptr %p, <2 x i1> %mask, <2 x i8> %data) { +; RV64-LABEL: test_compresstore_v2i8: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 2, e8, mf8, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; RV64-NEXT: vse8.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v2i8: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 2, e8, mf8, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; RV32-NEXT: vse8.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v2i8(<2 x i8> %data, ptr align 1 %p, <2 x i1> %mask) + ret void +} + +define void @test_compresstore_v4i8(ptr %p, <4 x i1> %mask, <4 x i8> %data) { +; RV64-LABEL: test_compresstore_v4i8: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; RV64-NEXT: vse8.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v4i8: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; RV32-NEXT: vse8.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v4i8(<4 x i8> %data, ptr align 1 %p, <4 x i1> %mask) + ret void +} + +define void @test_compresstore_v8i8(ptr %p, <8 x i1> %mask, <8 x i8> %data) { +; RV64-LABEL: test_compresstore_v8i8: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; RV64-NEXT: vse8.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v8i8: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; RV32-NEXT: vse8.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v8i8(<8 x i8> %data, ptr align 1 %p, <8 x i1> %mask) + ret void +} + +define void @test_compresstore_v16i8(ptr %p, <16 x i1> %mask, <16 x i8> %data) { +; RV64-LABEL: test_compresstore_v16i8: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; RV64-NEXT: vse8.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v16i8: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; RV32-NEXT: vse8.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v16i8(<16 x i8> %data, ptr align 1 %p, <16 x i1> %mask) + ret void +} + +define void @test_compresstore_v32i8(ptr %p, <32 x i1> %mask, <32 x i8> %data) { +; RV64-LABEL: test_compresstore_v32i8: +; RV64: # %bb.0: # %entry +; RV64-NEXT: li a1, 32 +; RV64-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; RV64-NEXT: vcompress.vm v10, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; RV64-NEXT: vse8.v v10, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v32i8: +; RV32: # %bb.0: # %entry +; RV32-NEXT: li a1, 32 +; RV32-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; RV32-NEXT: vcompress.vm v10, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; RV32-NEXT: vse8.v v10, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v32i8(<32 x i8> %data, ptr align 1 %p, <32 x i1> %mask) + ret void +} + +define void @test_compresstore_v64i8(ptr %p, <64 x i1> %mask, <64 x i8> %data) { +; RV64-LABEL: test_compresstore_v64i8: +; RV64: # %bb.0: # %entry +; RV64-NEXT: li a1, 64 +; RV64-NEXT: vsetvli zero, a1, e8, m4, ta, ma +; RV64-NEXT: vcompress.vm v12, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e8, m4, ta, ma +; RV64-NEXT: vse8.v v12, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v64i8: +; RV32: # %bb.0: # %entry +; RV32-NEXT: li a1, 64 +; RV32-NEXT: vsetvli zero, a1, e8, m4, ta, ma +; RV32-NEXT: vcompress.vm v12, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e8, m4, ta, ma +; RV32-NEXT: vse8.v v12, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v64i8(<64 x i8> %data, ptr align 1 %p, <64 x i1> %mask) + ret void +} + +define void @test_compresstore_v128i8(ptr %p, <128 x i1> %mask, <128 x i8> %data) { +; RV64-LABEL: test_compresstore_v128i8: +; RV64: # %bb.0: # %entry +; RV64-NEXT: li a1, 128 +; RV64-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; RV64-NEXT: vcompress.vm v16, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; RV64-NEXT: vse8.v v16, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v128i8: +; RV32: # %bb.0: # %entry +; RV32-NEXT: li a1, 128 +; RV32-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; RV32-NEXT: vcompress.vm v16, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; RV32-NEXT: vse8.v v16, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v128i8(<128 x i8> %data, ptr align 1 %p, <128 x i1> %mask) + ret void +} + +define void @test_compresstore_v256i8(ptr %p, <256 x i1> %mask, <256 x i8> %data) { +; RV64-LABEL: test_compresstore_v256i8: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vmv1r.v v7, v8 +; RV64-NEXT: li a2, 128 +; RV64-NEXT: vsetvli zero, a2, e8, m8, ta, ma +; RV64-NEXT: vle8.v v24, (a1) +; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; RV64-NEXT: vslidedown.vi v9, v0, 1 +; RV64-NEXT: vmv.x.s a1, v9 +; RV64-NEXT: vmv.x.s a3, v0 +; RV64-NEXT: vsetvli zero, a2, e8, m8, ta, ma +; RV64-NEXT: vcompress.vm v8, v16, v0 +; RV64-NEXT: vcpop.m a4, v0 +; RV64-NEXT: vsetvli zero, a4, e8, m8, ta, ma +; RV64-NEXT: vse8.v v8, (a0) +; RV64-NEXT: vsetvli zero, a2, e8, m8, ta, ma +; RV64-NEXT: vcompress.vm v8, v24, v7 +; RV64-NEXT: vcpop.m a2, v7 +; RV64-NEXT: cpop a3, a3 +; RV64-NEXT: cpop a1, a1 +; RV64-NEXT: add a0, a0, a3 +; RV64-NEXT: add a0, a0, a1 +; RV64-NEXT: vsetvli zero, a2, e8, m8, ta, ma +; RV64-NEXT: vse8.v v8, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v256i8: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vmv1r.v v7, v8 +; RV32-NEXT: li a2, 128 +; RV32-NEXT: vsetvli zero, a2, e8, m8, ta, ma +; RV32-NEXT: vle8.v v24, (a1) +; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; RV32-NEXT: vslidedown.vi v9, v0, 1 +; RV32-NEXT: li a1, 32 +; RV32-NEXT: vsrl.vx v10, v9, a1 +; RV32-NEXT: vmv.x.s a3, v10 +; RV32-NEXT: vsrl.vx v10, v0, a1 +; RV32-NEXT: vmv.x.s a1, v10 +; RV32-NEXT: vmv.x.s a4, v9 +; RV32-NEXT: vmv.x.s a5, v0 +; RV32-NEXT: vsetvli zero, a2, e8, m8, ta, ma +; RV32-NEXT: vcompress.vm v8, v16, v0 +; RV32-NEXT: vcpop.m a6, v0 +; RV32-NEXT: vsetvli zero, a6, e8, m8, ta, ma +; RV32-NEXT: vse8.v v8, (a0) +; RV32-NEXT: cpop a1, a1 +; RV32-NEXT: cpop a5, a5 +; RV32-NEXT: add a1, a5, a1 +; RV32-NEXT: cpop a3, a3 +; RV32-NEXT: cpop a4, a4 +; RV32-NEXT: add a3, a4, a3 +; RV32-NEXT: add a1, a1, a3 +; RV32-NEXT: add a0, a0, a1 +; RV32-NEXT: vsetvli zero, a2, e8, m8, ta, ma +; RV32-NEXT: vcompress.vm v8, v24, v7 +; RV32-NEXT: vcpop.m a1, v7 +; RV32-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; RV32-NEXT: vse8.v v8, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v256i8(<256 x i8> %data, ptr align 1 %p, <256 x i1> %mask) + ret void +} + +; Compress + store for i16 type + +define void @test_compresstore_v1i16(ptr %p, <1 x i1> %mask, <1 x i16> %data) { +; RV64-LABEL: test_compresstore_v1i16: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 1, e16, mf4, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e16, mf4, ta, ma +; RV64-NEXT: vse16.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v1i16: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 1, e16, mf4, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e16, mf4, ta, ma +; RV32-NEXT: vse16.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v1i16(<1 x i16> %data, ptr align 2 %p, <1 x i1> %mask) + ret void +} + +define void @test_compresstore_v2i16(ptr %p, <2 x i1> %mask, <2 x i16> %data) { +; RV64-LABEL: test_compresstore_v2i16: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e16, mf4, ta, ma +; RV64-NEXT: vse16.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v2i16: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e16, mf4, ta, ma +; RV32-NEXT: vse16.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v2i16(<2 x i16> %data, ptr align 2 %p, <2 x i1> %mask) + ret void +} + +define void @test_compresstore_v4i16(ptr %p, <4 x i1> %mask, <4 x i16> %data) { +; RV64-LABEL: test_compresstore_v4i16: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e16, mf2, ta, ma +; RV64-NEXT: vse16.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v4i16: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e16, mf2, ta, ma +; RV32-NEXT: vse16.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v4i16(<4 x i16> %data, ptr align 2 %p, <4 x i1> %mask) + ret void +} + +define void @test_compresstore_v8i16(ptr %p, <8 x i1> %mask, <8 x i16> %data) { +; RV64-LABEL: test_compresstore_v8i16: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e16, m1, ta, ma +; RV64-NEXT: vse16.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v8i16: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e16, m1, ta, ma +; RV32-NEXT: vse16.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v8i16(<8 x i16> %data, ptr align 2 %p, <8 x i1> %mask) + ret void +} + +define void @test_compresstore_v16i16(ptr %p, <16 x i1> %mask, <16 x i16> %data) { +; RV64-LABEL: test_compresstore_v16i16: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; RV64-NEXT: vcompress.vm v10, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e16, m2, ta, ma +; RV64-NEXT: vse16.v v10, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v16i16: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; RV32-NEXT: vcompress.vm v10, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e16, m2, ta, ma +; RV32-NEXT: vse16.v v10, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v16i16(<16 x i16> %data, ptr align 2 %p, <16 x i1> %mask) + ret void +} + +define void @test_compresstore_v32i16(ptr %p, <32 x i1> %mask, <32 x i16> %data) { +; RV64-LABEL: test_compresstore_v32i16: +; RV64: # %bb.0: # %entry +; RV64-NEXT: li a1, 32 +; RV64-NEXT: vsetvli zero, a1, e16, m4, ta, ma +; RV64-NEXT: vcompress.vm v12, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e16, m4, ta, ma +; RV64-NEXT: vse16.v v12, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v32i16: +; RV32: # %bb.0: # %entry +; RV32-NEXT: li a1, 32 +; RV32-NEXT: vsetvli zero, a1, e16, m4, ta, ma +; RV32-NEXT: vcompress.vm v12, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e16, m4, ta, ma +; RV32-NEXT: vse16.v v12, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v32i16(<32 x i16> %data, ptr align 2 %p, <32 x i1> %mask) + ret void +} + +define void @test_compresstore_v64i16(ptr %p, <64 x i1> %mask, <64 x i16> %data) { +; RV64-LABEL: test_compresstore_v64i16: +; RV64: # %bb.0: # %entry +; RV64-NEXT: li a1, 64 +; RV64-NEXT: vsetvli zero, a1, e16, m8, ta, ma +; RV64-NEXT: vcompress.vm v16, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e16, m8, ta, ma +; RV64-NEXT: vse16.v v16, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v64i16: +; RV32: # %bb.0: # %entry +; RV32-NEXT: li a1, 64 +; RV32-NEXT: vsetvli zero, a1, e16, m8, ta, ma +; RV32-NEXT: vcompress.vm v16, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e16, m8, ta, ma +; RV32-NEXT: vse16.v v16, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v64i16(<64 x i16> %data, ptr align 2 %p, <64 x i1> %mask) + ret void +} + +define void @test_compresstore_v128i16(ptr %p, <128 x i1> %mask, <128 x i16> %data) { +; RV64-LABEL: test_compresstore_v128i16: +; RV64: # %bb.0: # %entry +; RV64-NEXT: li a1, 64 +; RV64-NEXT: vsetvli zero, a1, e16, m8, ta, ma +; RV64-NEXT: vcompress.vm v24, v8, v0 +; RV64-NEXT: vcpop.m a2, v0 +; RV64-NEXT: vsetvli zero, a2, e16, m8, ta, ma +; RV64-NEXT: vse16.v v24, (a0) +; RV64-NEXT: vsetivli zero, 8, e8, m1, ta, ma +; RV64-NEXT: vslidedown.vi v8, v0, 8 +; RV64-NEXT: vsetvli zero, a1, e16, m8, ta, ma +; RV64-NEXT: vcompress.vm v24, v16, v8 +; RV64-NEXT: vcpop.m a2, v8 +; RV64-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; RV64-NEXT: vmv.x.s a1, v0 +; RV64-NEXT: cpop a1, a1 +; RV64-NEXT: slli a1, a1, 1 +; RV64-NEXT: add a0, a0, a1 +; RV64-NEXT: vsetvli zero, a2, e16, m8, ta, ma +; RV64-NEXT: vse16.v v24, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v128i16: +; RV32: # %bb.0: # %entry +; RV32-NEXT: li a1, 64 +; RV32-NEXT: vsetvli zero, a1, e16, m8, ta, ma +; RV32-NEXT: vcompress.vm v24, v8, v0 +; RV32-NEXT: vcpop.m a2, v0 +; RV32-NEXT: vsetvli zero, a2, e16, m8, ta, ma +; RV32-NEXT: vse16.v v24, (a0) +; RV32-NEXT: vsetivli zero, 8, e8, m1, ta, ma +; RV32-NEXT: vslidedown.vi v24, v0, 8 +; RV32-NEXT: vsetvli zero, a1, e16, m8, ta, ma +; RV32-NEXT: vcompress.vm v8, v16, v24 +; RV32-NEXT: vcpop.m a1, v24 +; RV32-NEXT: li a2, 32 +; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; RV32-NEXT: vsrl.vx v16, v0, a2 +; RV32-NEXT: vmv.x.s a2, v16 +; RV32-NEXT: cpop a2, a2 +; RV32-NEXT: vmv.x.s a3, v0 +; RV32-NEXT: cpop a3, a3 +; RV32-NEXT: add a2, a3, a2 +; RV32-NEXT: slli a2, a2, 1 +; RV32-NEXT: add a0, a0, a2 +; RV32-NEXT: vsetvli zero, a1, e16, m8, ta, ma +; RV32-NEXT: vse16.v v8, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v128i16(<128 x i16> %data, ptr align 2 %p, <128 x i1> %mask) + ret void +} + +; Compress + store for i32 type + +define void @test_compresstore_v1i32(ptr %p, <1 x i1> %mask, <1 x i32> %data) { +; RV64-LABEL: test_compresstore_v1i32: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 1, e32, mf2, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e32, mf2, ta, ma +; RV64-NEXT: vse32.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v1i32: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 1, e32, mf2, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e32, mf2, ta, ma +; RV32-NEXT: vse32.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v1i32(<1 x i32> %data, ptr align 4 %p, <1 x i1> %mask) + ret void +} + +define void @test_compresstore_v2i32(ptr %p, <2 x i1> %mask, <2 x i32> %data) { +; RV64-LABEL: test_compresstore_v2i32: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e32, mf2, ta, ma +; RV64-NEXT: vse32.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v2i32: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e32, mf2, ta, ma +; RV32-NEXT: vse32.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v2i32(<2 x i32> %data, ptr align 4 %p, <2 x i1> %mask) + ret void +} + +define void @test_compresstore_v4i32(ptr %p, <4 x i1> %mask, <4 x i32> %data) { +; RV64-LABEL: test_compresstore_v4i32: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e32, m1, ta, ma +; RV64-NEXT: vse32.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v4i32: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e32, m1, ta, ma +; RV32-NEXT: vse32.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v4i32(<4 x i32> %data, ptr align 4 %p, <4 x i1> %mask) + ret void +} + +define void @test_compresstore_v8i32(ptr %p, <8 x i1> %mask, <8 x i32> %data) { +; RV64-LABEL: test_compresstore_v8i32: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; RV64-NEXT: vcompress.vm v10, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e32, m2, ta, ma +; RV64-NEXT: vse32.v v10, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v8i32: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; RV32-NEXT: vcompress.vm v10, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e32, m2, ta, ma +; RV32-NEXT: vse32.v v10, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v8i32(<8 x i32> %data, ptr align 4 %p, <8 x i1> %mask) + ret void +} + +define void @test_compresstore_v16i32(ptr %p, <16 x i1> %mask, <16 x i32> %data) { +; RV64-LABEL: test_compresstore_v16i32: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; RV64-NEXT: vcompress.vm v12, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e32, m4, ta, ma +; RV64-NEXT: vse32.v v12, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v16i32: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; RV32-NEXT: vcompress.vm v12, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e32, m4, ta, ma +; RV32-NEXT: vse32.v v12, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v16i32(<16 x i32> %data, ptr align 4 %p, <16 x i1> %mask) + ret void +} + +define void @test_compresstore_v32i32(ptr %p, <32 x i1> %mask, <32 x i32> %data) { +; RV64-LABEL: test_compresstore_v32i32: +; RV64: # %bb.0: # %entry +; RV64-NEXT: li a1, 32 +; RV64-NEXT: vsetvli zero, a1, e32, m8, ta, ma +; RV64-NEXT: vcompress.vm v16, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e32, m8, ta, ma +; RV64-NEXT: vse32.v v16, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v32i32: +; RV32: # %bb.0: # %entry +; RV32-NEXT: li a1, 32 +; RV32-NEXT: vsetvli zero, a1, e32, m8, ta, ma +; RV32-NEXT: vcompress.vm v16, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e32, m8, ta, ma +; RV32-NEXT: vse32.v v16, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v32i32(<32 x i32> %data, ptr align 4 %p, <32 x i1> %mask) + ret void +} + +define void @test_compresstore_v64i32(ptr %p, <64 x i1> %mask, <64 x i32> %data) { +; RV64-LABEL: test_compresstore_v64i32: +; RV64: # %bb.0: # %entry +; RV64-NEXT: li a1, 32 +; RV64-NEXT: vsetvli zero, a1, e32, m8, ta, ma +; RV64-NEXT: vcompress.vm v24, v8, v0 +; RV64-NEXT: vcpop.m a2, v0 +; RV64-NEXT: vsetvli zero, a2, e32, m8, ta, ma +; RV64-NEXT: vse32.v v24, (a0) +; RV64-NEXT: vsetivli zero, 4, e8, mf2, ta, ma +; RV64-NEXT: vslidedown.vi v8, v0, 4 +; RV64-NEXT: vsetvli zero, a1, e32, m8, ta, ma +; RV64-NEXT: vcompress.vm v24, v16, v8 +; RV64-NEXT: vcpop.m a1, v8 +; RV64-NEXT: vmv.x.s a2, v0 +; RV64-NEXT: cpopw a2, a2 +; RV64-NEXT: slli a2, a2, 2 +; RV64-NEXT: add a0, a0, a2 +; RV64-NEXT: vsetvli zero, a1, e32, m8, ta, ma +; RV64-NEXT: vse32.v v24, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v64i32: +; RV32: # %bb.0: # %entry +; RV32-NEXT: li a1, 32 +; RV32-NEXT: vsetvli zero, a1, e32, m8, ta, ma +; RV32-NEXT: vcompress.vm v24, v8, v0 +; RV32-NEXT: vcpop.m a2, v0 +; RV32-NEXT: vsetvli zero, a2, e32, m8, ta, ma +; RV32-NEXT: vse32.v v24, (a0) +; RV32-NEXT: vsetivli zero, 4, e8, mf2, ta, ma +; RV32-NEXT: vslidedown.vi v8, v0, 4 +; RV32-NEXT: vsetvli zero, a1, e32, m8, ta, ma +; RV32-NEXT: vcompress.vm v24, v16, v8 +; RV32-NEXT: vcpop.m a1, v8 +; RV32-NEXT: vmv.x.s a2, v0 +; RV32-NEXT: cpop a2, a2 +; RV32-NEXT: slli a2, a2, 2 +; RV32-NEXT: add a0, a0, a2 +; RV32-NEXT: vsetvli zero, a1, e32, m8, ta, ma +; RV32-NEXT: vse32.v v24, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v64i32(<64 x i32> %data, ptr align 4 %p, <64 x i1> %mask) + ret void +} + +; Compress + store for i64 type + +define void @test_compresstore_v1i64(ptr %p, <1 x i1> %mask, <1 x i64> %data) { +; RV64-LABEL: test_compresstore_v1i64: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; RV64-NEXT: vse64.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v1i64: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; RV32-NEXT: vse64.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v1i64(<1 x i64> %data, ptr align 8 %p, <1 x i1> %mask) + ret void +} + +define void @test_compresstore_v2i64(ptr %p, <2 x i1> %mask, <2 x i64> %data) { +; RV64-LABEL: test_compresstore_v2i64: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; RV64-NEXT: vse64.v v9, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v2i64: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; RV32-NEXT: vse64.v v9, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v2i64(<2 x i64> %data, ptr align 8 %p, <2 x i1> %mask) + ret void +} + +define void @test_compresstore_v4i64(ptr %p, <4 x i1> %mask, <4 x i64> %data) { +; RV64-LABEL: test_compresstore_v4i64: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; RV64-NEXT: vcompress.vm v10, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e64, m2, ta, ma +; RV64-NEXT: vse64.v v10, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v4i64: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; RV32-NEXT: vcompress.vm v10, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e64, m2, ta, ma +; RV32-NEXT: vse64.v v10, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v4i64(<4 x i64> %data, ptr align 8 %p, <4 x i1> %mask) + ret void +} + +define void @test_compresstore_v8i64(ptr %p, <8 x i1> %mask, <8 x i64> %data) { +; RV64-LABEL: test_compresstore_v8i64: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 8, e64, m4, ta, ma +; RV64-NEXT: vcompress.vm v12, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e64, m4, ta, ma +; RV64-NEXT: vse64.v v12, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v8i64: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 8, e64, m4, ta, ma +; RV32-NEXT: vcompress.vm v12, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e64, m4, ta, ma +; RV32-NEXT: vse64.v v12, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v8i64(<8 x i64> %data, ptr align 8 %p, <8 x i1> %mask) + ret void +} + +define void @test_compresstore_v16i64(ptr %p, <16 x i1> %mask, <16 x i64> %data) { +; RV64-LABEL: test_compresstore_v16i64: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 16, e64, m8, ta, ma +; RV64-NEXT: vcompress.vm v16, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e64, m8, ta, ma +; RV64-NEXT: vse64.v v16, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v16i64: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 16, e64, m8, ta, ma +; RV32-NEXT: vcompress.vm v16, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e64, m8, ta, ma +; RV32-NEXT: vse64.v v16, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v16i64(<16 x i64> %data, ptr align 8 %p, <16 x i1> %mask) + ret void +} + +define void @test_compresstore_v32i64(ptr %p, <32 x i1> %mask, <32 x i64> %data) { +; RV64-LABEL: test_compresstore_v32i64: +; RV64: # %bb.0: # %entry +; RV64-NEXT: vsetivli zero, 16, e64, m8, ta, ma +; RV64-NEXT: vcompress.vm v24, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e64, m8, ta, ma +; RV64-NEXT: vse64.v v24, (a0) +; RV64-NEXT: vsetivli zero, 2, e8, mf4, ta, ma +; RV64-NEXT: vslidedown.vi v24, v0, 2 +; RV64-NEXT: vsetivli zero, 16, e64, m8, ta, ma +; RV64-NEXT: vcompress.vm v8, v16, v24 +; RV64-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; RV64-NEXT: vmv.x.s a1, v0 +; RV64-NEXT: zext.h a1, a1 +; RV64-NEXT: cpopw a1, a1 +; RV64-NEXT: slli a1, a1, 3 +; RV64-NEXT: add a0, a0, a1 +; RV64-NEXT: vcpop.m a1, v24 +; RV64-NEXT: vsetvli zero, a1, e64, m8, ta, ma +; RV64-NEXT: vse64.v v8, (a0) +; RV64-NEXT: ret +; +; RV32-LABEL: test_compresstore_v32i64: +; RV32: # %bb.0: # %entry +; RV32-NEXT: vsetivli zero, 16, e64, m8, ta, ma +; RV32-NEXT: vcompress.vm v24, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e64, m8, ta, ma +; RV32-NEXT: vse64.v v24, (a0) +; RV32-NEXT: vsetivli zero, 2, e8, mf4, ta, ma +; RV32-NEXT: vslidedown.vi v24, v0, 2 +; RV32-NEXT: vsetivli zero, 16, e64, m8, ta, ma +; RV32-NEXT: vcompress.vm v8, v16, v24 +; RV32-NEXT: vsetvli zero, zero, e16, m2, ta, ma +; RV32-NEXT: vmv.x.s a1, v0 +; RV32-NEXT: zext.h a1, a1 +; RV32-NEXT: cpop a1, a1 +; RV32-NEXT: slli a1, a1, 3 +; RV32-NEXT: add a0, a0, a1 +; RV32-NEXT: vcpop.m a1, v24 +; RV32-NEXT: vsetvli zero, a1, e64, m8, ta, ma +; RV32-NEXT: vse64.v v8, (a0) +; RV32-NEXT: ret +entry: + tail call void @llvm.masked.compressstore.v32i64(<32 x i64> %data, ptr align 8 %p, <32 x i1> %mask) + ret void +} + +declare void @llvm.masked.compressstore.v1i8(<1 x i8>, ptr, <1 x i1>) +declare void @llvm.masked.compressstore.v2i8(<2 x i8>, ptr, <2 x i1>) +declare void @llvm.masked.compressstore.v4i8(<4 x i8>, ptr, <4 x i1>) +declare void @llvm.masked.compressstore.v8i8(<8 x i8>, ptr, <8 x i1>) +declare void @llvm.masked.compressstore.v16i8(<16 x i8>, ptr, <16 x i1>) +declare void @llvm.masked.compressstore.v32i8(<32 x i8>, ptr, <32 x i1>) +declare void @llvm.masked.compressstore.v64i8(<64 x i8>, ptr, <64 x i1>) +declare void @llvm.masked.compressstore.v128i8(<128 x i8>, ptr, <128 x i1>) +declare void @llvm.masked.compressstore.v256i8(<256 x i8>, ptr, <256 x i1>) + +declare void @llvm.masked.compressstore.v1i16(<1 x i16>, ptr, <1 x i1>) +declare void @llvm.masked.compressstore.v2i16(<2 x i16>, ptr, <2 x i1>) +declare void @llvm.masked.compressstore.v4i16(<4 x i16>, ptr, <4 x i1>) +declare void @llvm.masked.compressstore.v8i16(<8 x i16>, ptr, <8 x i1>) +declare void @llvm.masked.compressstore.v16i16(<16 x i16>, ptr, <16 x i1>) +declare void @llvm.masked.compressstore.v32i16(<32 x i16>, ptr, <32 x i1>) +declare void @llvm.masked.compressstore.v64i16(<64 x i16>, ptr, <64 x i1>) +declare void @llvm.masked.compressstore.v128i16(<128 x i16>, ptr, <128 x i1>) + +declare void @llvm.masked.compressstore.v1i32(<1 x i32>, ptr, <1 x i1>) +declare void @llvm.masked.compressstore.v2i32(<2 x i32>, ptr, <2 x i1>) +declare void @llvm.masked.compressstore.v4i32(<4 x i32>, ptr, <4 x i1>) +declare void @llvm.masked.compressstore.v8i32(<8 x i32>, ptr, <8 x i1>) +declare void @llvm.masked.compressstore.v16i32(<16 x i32>, ptr, <16 x i1>) +declare void @llvm.masked.compressstore.v32i32(<32 x i32>, ptr, <32 x i1>) +declare void @llvm.masked.compressstore.v64i32(<64 x i32>, ptr, <64 x i1>) + +declare void @llvm.masked.compressstore.v1i64(<1 x i64>, ptr, <1 x i1>) +declare void @llvm.masked.compressstore.v2i64(<2 x i64>, ptr, <2 x i1>) +declare void @llvm.masked.compressstore.v4i64(<4 x i64>, ptr, <4 x i1>) +declare void @llvm.masked.compressstore.v8i64(<8 x i64>, ptr, <8 x i1>) +declare void @llvm.masked.compressstore.v16i64(<16 x i64>, ptr, <16 x i1>) +declare void @llvm.masked.compressstore.v32i64(<32 x i64>, ptr, <32 x i1>) diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compressstore-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compressstore-fp.ll index 52c52921e7e1..36fbdd8e0664 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compressstore-fp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compressstore-fp.ll @@ -6,24 +6,20 @@ declare void @llvm.masked.compressstore.v1f16(<1 x half>, ptr, <1 x i1>) define void @compressstore_v1f16(ptr %base, <1 x half> %v, <1 x i1> %mask) { ; RV32-LABEL: compressstore_v1f16: ; RV32: # %bb.0: -; RV32-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; RV32-NEXT: vfirst.m a1, v0 -; RV32-NEXT: bnez a1, .LBB0_2 -; RV32-NEXT: # %bb.1: # %cond.store ; RV32-NEXT: vsetivli zero, 1, e16, mf4, ta, ma -; RV32-NEXT: vse16.v v8, (a0) -; RV32-NEXT: .LBB0_2: # %else +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e16, mf4, ta, ma +; RV32-NEXT: vse16.v v9, (a0) ; RV32-NEXT: ret ; ; RV64-LABEL: compressstore_v1f16: ; RV64: # %bb.0: -; RV64-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; RV64-NEXT: vfirst.m a1, v0 -; RV64-NEXT: bnez a1, .LBB0_2 -; RV64-NEXT: # %bb.1: # %cond.store ; RV64-NEXT: vsetivli zero, 1, e16, mf4, ta, ma -; RV64-NEXT: vse16.v v8, (a0) -; RV64-NEXT: .LBB0_2: # %else +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e16, mf4, ta, ma +; RV64-NEXT: vse16.v v9, (a0) ; RV64-NEXT: ret call void @llvm.masked.compressstore.v1f16(<1 x half> %v, ptr align 2 %base, <1 x i1> %mask) ret void @@ -33,48 +29,20 @@ declare void @llvm.masked.compressstore.v2f16(<2 x half>, ptr, <2 x i1>) define void @compressstore_v2f16(ptr %base, <2 x half> %v, <2 x i1> %mask) { ; RV32-LABEL: compressstore_v2f16: ; RV32: # %bb.0: -; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV32-NEXT: vmv.x.s a1, v0 -; RV32-NEXT: andi a2, a1, 1 -; RV32-NEXT: bnez a2, .LBB1_3 -; RV32-NEXT: # %bb.1: # %else -; RV32-NEXT: andi a1, a1, 2 -; RV32-NEXT: bnez a1, .LBB1_4 -; RV32-NEXT: .LBB1_2: # %else2 -; RV32-NEXT: ret -; RV32-NEXT: .LBB1_3: # %cond.store -; RV32-NEXT: vsetivli zero, 1, e16, mf4, ta, ma -; RV32-NEXT: vse16.v v8, (a0) -; RV32-NEXT: addi a0, a0, 2 -; RV32-NEXT: andi a1, a1, 2 -; RV32-NEXT: beqz a1, .LBB1_2 -; RV32-NEXT: .LBB1_4: # %cond.store1 -; RV32-NEXT: vsetivli zero, 1, e16, mf4, ta, ma -; RV32-NEXT: vslidedown.vi v8, v8, 1 -; RV32-NEXT: vse16.v v8, (a0) +; RV32-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e16, mf4, ta, ma +; RV32-NEXT: vse16.v v9, (a0) ; RV32-NEXT: ret ; ; RV64-LABEL: compressstore_v2f16: ; RV64: # %bb.0: -; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV64-NEXT: vmv.x.s a1, v0 -; RV64-NEXT: andi a2, a1, 1 -; RV64-NEXT: bnez a2, .LBB1_3 -; RV64-NEXT: # %bb.1: # %else -; RV64-NEXT: andi a1, a1, 2 -; RV64-NEXT: bnez a1, .LBB1_4 -; RV64-NEXT: .LBB1_2: # %else2 -; RV64-NEXT: ret -; RV64-NEXT: .LBB1_3: # %cond.store -; RV64-NEXT: vsetivli zero, 1, e16, mf4, ta, ma -; RV64-NEXT: vse16.v v8, (a0) -; RV64-NEXT: addi a0, a0, 2 -; RV64-NEXT: andi a1, a1, 2 -; RV64-NEXT: beqz a1, .LBB1_2 -; RV64-NEXT: .LBB1_4: # %cond.store1 -; RV64-NEXT: vsetivli zero, 1, e16, mf4, ta, ma -; RV64-NEXT: vslidedown.vi v8, v8, 1 -; RV64-NEXT: vse16.v v8, (a0) +; RV64-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e16, mf4, ta, ma +; RV64-NEXT: vse16.v v9, (a0) ; RV64-NEXT: ret call void @llvm.masked.compressstore.v2f16(<2 x half> %v, ptr align 2 %base, <2 x i1> %mask) ret void @@ -84,88 +52,20 @@ declare void @llvm.masked.compressstore.v4f16(<4 x half>, ptr, <4 x i1>) define void @compressstore_v4f16(ptr %base, <4 x half> %v, <4 x i1> %mask) { ; RV32-LABEL: compressstore_v4f16: ; RV32: # %bb.0: -; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV32-NEXT: vmv.x.s a1, v0 -; RV32-NEXT: andi a2, a1, 1 -; RV32-NEXT: bnez a2, .LBB2_5 -; RV32-NEXT: # %bb.1: # %else -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: bnez a2, .LBB2_6 -; RV32-NEXT: .LBB2_2: # %else2 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: bnez a2, .LBB2_7 -; RV32-NEXT: .LBB2_3: # %else5 -; RV32-NEXT: andi a1, a1, 8 -; RV32-NEXT: bnez a1, .LBB2_8 -; RV32-NEXT: .LBB2_4: # %else8 -; RV32-NEXT: ret -; RV32-NEXT: .LBB2_5: # %cond.store -; RV32-NEXT: vsetivli zero, 1, e16, mf2, ta, ma -; RV32-NEXT: vse16.v v8, (a0) -; RV32-NEXT: addi a0, a0, 2 -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: beqz a2, .LBB2_2 -; RV32-NEXT: .LBB2_6: # %cond.store1 -; RV32-NEXT: vsetivli zero, 1, e16, mf2, ta, ma -; RV32-NEXT: vslidedown.vi v9, v8, 1 -; RV32-NEXT: vse16.v v9, (a0) -; RV32-NEXT: addi a0, a0, 2 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: beqz a2, .LBB2_3 -; RV32-NEXT: .LBB2_7: # %cond.store4 -; RV32-NEXT: vsetivli zero, 1, e16, mf2, ta, ma -; RV32-NEXT: vslidedown.vi v9, v8, 2 +; RV32-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e16, mf2, ta, ma ; RV32-NEXT: vse16.v v9, (a0) -; RV32-NEXT: addi a0, a0, 2 -; RV32-NEXT: andi a1, a1, 8 -; RV32-NEXT: beqz a1, .LBB2_4 -; RV32-NEXT: .LBB2_8: # %cond.store7 -; RV32-NEXT: vsetivli zero, 1, e16, mf2, ta, ma -; RV32-NEXT: vslidedown.vi v8, v8, 3 -; RV32-NEXT: vse16.v v8, (a0) ; RV32-NEXT: ret ; ; RV64-LABEL: compressstore_v4f16: ; RV64: # %bb.0: -; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV64-NEXT: vmv.x.s a1, v0 -; RV64-NEXT: andi a2, a1, 1 -; RV64-NEXT: bnez a2, .LBB2_5 -; RV64-NEXT: # %bb.1: # %else -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: bnez a2, .LBB2_6 -; RV64-NEXT: .LBB2_2: # %else2 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: bnez a2, .LBB2_7 -; RV64-NEXT: .LBB2_3: # %else5 -; RV64-NEXT: andi a1, a1, 8 -; RV64-NEXT: bnez a1, .LBB2_8 -; RV64-NEXT: .LBB2_4: # %else8 -; RV64-NEXT: ret -; RV64-NEXT: .LBB2_5: # %cond.store -; RV64-NEXT: vsetivli zero, 1, e16, mf2, ta, ma -; RV64-NEXT: vse16.v v8, (a0) -; RV64-NEXT: addi a0, a0, 2 -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: beqz a2, .LBB2_2 -; RV64-NEXT: .LBB2_6: # %cond.store1 -; RV64-NEXT: vsetivli zero, 1, e16, mf2, ta, ma -; RV64-NEXT: vslidedown.vi v9, v8, 1 -; RV64-NEXT: vse16.v v9, (a0) -; RV64-NEXT: addi a0, a0, 2 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: beqz a2, .LBB2_3 -; RV64-NEXT: .LBB2_7: # %cond.store4 -; RV64-NEXT: vsetivli zero, 1, e16, mf2, ta, ma -; RV64-NEXT: vslidedown.vi v9, v8, 2 +; RV64-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e16, mf2, ta, ma ; RV64-NEXT: vse16.v v9, (a0) -; RV64-NEXT: addi a0, a0, 2 -; RV64-NEXT: andi a1, a1, 8 -; RV64-NEXT: beqz a1, .LBB2_4 -; RV64-NEXT: .LBB2_8: # %cond.store7 -; RV64-NEXT: vsetivli zero, 1, e16, mf2, ta, ma -; RV64-NEXT: vslidedown.vi v8, v8, 3 -; RV64-NEXT: vse16.v v8, (a0) ; RV64-NEXT: ret call void @llvm.masked.compressstore.v4f16(<4 x half> %v, ptr align 2 %base, <4 x i1> %mask) ret void @@ -175,168 +75,20 @@ declare void @llvm.masked.compressstore.v8f16(<8 x half>, ptr, <8 x i1>) define void @compressstore_v8f16(ptr %base, <8 x half> %v, <8 x i1> %mask) { ; RV32-LABEL: compressstore_v8f16: ; RV32: # %bb.0: -; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV32-NEXT: vmv.x.s a1, v0 -; RV32-NEXT: andi a2, a1, 1 -; RV32-NEXT: bnez a2, .LBB3_9 -; RV32-NEXT: # %bb.1: # %else -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: bnez a2, .LBB3_10 -; RV32-NEXT: .LBB3_2: # %else2 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: bnez a2, .LBB3_11 -; RV32-NEXT: .LBB3_3: # %else5 -; RV32-NEXT: andi a2, a1, 8 -; RV32-NEXT: bnez a2, .LBB3_12 -; RV32-NEXT: .LBB3_4: # %else8 -; RV32-NEXT: andi a2, a1, 16 -; RV32-NEXT: bnez a2, .LBB3_13 -; RV32-NEXT: .LBB3_5: # %else11 -; RV32-NEXT: andi a2, a1, 32 -; RV32-NEXT: bnez a2, .LBB3_14 -; RV32-NEXT: .LBB3_6: # %else14 -; RV32-NEXT: andi a2, a1, 64 -; RV32-NEXT: bnez a2, .LBB3_15 -; RV32-NEXT: .LBB3_7: # %else17 -; RV32-NEXT: andi a1, a1, -128 -; RV32-NEXT: bnez a1, .LBB3_16 -; RV32-NEXT: .LBB3_8: # %else20 -; RV32-NEXT: ret -; RV32-NEXT: .LBB3_9: # %cond.store -; RV32-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV32-NEXT: vse16.v v8, (a0) -; RV32-NEXT: addi a0, a0, 2 -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: beqz a2, .LBB3_2 -; RV32-NEXT: .LBB3_10: # %cond.store1 -; RV32-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV32-NEXT: vslidedown.vi v9, v8, 1 -; RV32-NEXT: vse16.v v9, (a0) -; RV32-NEXT: addi a0, a0, 2 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: beqz a2, .LBB3_3 -; RV32-NEXT: .LBB3_11: # %cond.store4 -; RV32-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV32-NEXT: vslidedown.vi v9, v8, 2 -; RV32-NEXT: vse16.v v9, (a0) -; RV32-NEXT: addi a0, a0, 2 -; RV32-NEXT: andi a2, a1, 8 -; RV32-NEXT: beqz a2, .LBB3_4 -; RV32-NEXT: .LBB3_12: # %cond.store7 -; RV32-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV32-NEXT: vslidedown.vi v9, v8, 3 -; RV32-NEXT: vse16.v v9, (a0) -; RV32-NEXT: addi a0, a0, 2 -; RV32-NEXT: andi a2, a1, 16 -; RV32-NEXT: beqz a2, .LBB3_5 -; RV32-NEXT: .LBB3_13: # %cond.store10 -; RV32-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV32-NEXT: vslidedown.vi v9, v8, 4 +; RV32-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e16, m1, ta, ma ; RV32-NEXT: vse16.v v9, (a0) -; RV32-NEXT: addi a0, a0, 2 -; RV32-NEXT: andi a2, a1, 32 -; RV32-NEXT: beqz a2, .LBB3_6 -; RV32-NEXT: .LBB3_14: # %cond.store13 -; RV32-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV32-NEXT: vslidedown.vi v9, v8, 5 -; RV32-NEXT: vse16.v v9, (a0) -; RV32-NEXT: addi a0, a0, 2 -; RV32-NEXT: andi a2, a1, 64 -; RV32-NEXT: beqz a2, .LBB3_7 -; RV32-NEXT: .LBB3_15: # %cond.store16 -; RV32-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV32-NEXT: vslidedown.vi v9, v8, 6 -; RV32-NEXT: vse16.v v9, (a0) -; RV32-NEXT: addi a0, a0, 2 -; RV32-NEXT: andi a1, a1, -128 -; RV32-NEXT: beqz a1, .LBB3_8 -; RV32-NEXT: .LBB3_16: # %cond.store19 -; RV32-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV32-NEXT: vslidedown.vi v8, v8, 7 -; RV32-NEXT: vse16.v v8, (a0) ; RV32-NEXT: ret ; ; RV64-LABEL: compressstore_v8f16: ; RV64: # %bb.0: -; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV64-NEXT: vmv.x.s a1, v0 -; RV64-NEXT: andi a2, a1, 1 -; RV64-NEXT: bnez a2, .LBB3_9 -; RV64-NEXT: # %bb.1: # %else -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: bnez a2, .LBB3_10 -; RV64-NEXT: .LBB3_2: # %else2 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: bnez a2, .LBB3_11 -; RV64-NEXT: .LBB3_3: # %else5 -; RV64-NEXT: andi a2, a1, 8 -; RV64-NEXT: bnez a2, .LBB3_12 -; RV64-NEXT: .LBB3_4: # %else8 -; RV64-NEXT: andi a2, a1, 16 -; RV64-NEXT: bnez a2, .LBB3_13 -; RV64-NEXT: .LBB3_5: # %else11 -; RV64-NEXT: andi a2, a1, 32 -; RV64-NEXT: bnez a2, .LBB3_14 -; RV64-NEXT: .LBB3_6: # %else14 -; RV64-NEXT: andi a2, a1, 64 -; RV64-NEXT: bnez a2, .LBB3_15 -; RV64-NEXT: .LBB3_7: # %else17 -; RV64-NEXT: andi a1, a1, -128 -; RV64-NEXT: bnez a1, .LBB3_16 -; RV64-NEXT: .LBB3_8: # %else20 -; RV64-NEXT: ret -; RV64-NEXT: .LBB3_9: # %cond.store -; RV64-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV64-NEXT: vse16.v v8, (a0) -; RV64-NEXT: addi a0, a0, 2 -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: beqz a2, .LBB3_2 -; RV64-NEXT: .LBB3_10: # %cond.store1 -; RV64-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV64-NEXT: vslidedown.vi v9, v8, 1 -; RV64-NEXT: vse16.v v9, (a0) -; RV64-NEXT: addi a0, a0, 2 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: beqz a2, .LBB3_3 -; RV64-NEXT: .LBB3_11: # %cond.store4 -; RV64-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV64-NEXT: vslidedown.vi v9, v8, 2 -; RV64-NEXT: vse16.v v9, (a0) -; RV64-NEXT: addi a0, a0, 2 -; RV64-NEXT: andi a2, a1, 8 -; RV64-NEXT: beqz a2, .LBB3_4 -; RV64-NEXT: .LBB3_12: # %cond.store7 -; RV64-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV64-NEXT: vslidedown.vi v9, v8, 3 +; RV64-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e16, m1, ta, ma ; RV64-NEXT: vse16.v v9, (a0) -; RV64-NEXT: addi a0, a0, 2 -; RV64-NEXT: andi a2, a1, 16 -; RV64-NEXT: beqz a2, .LBB3_5 -; RV64-NEXT: .LBB3_13: # %cond.store10 -; RV64-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV64-NEXT: vslidedown.vi v9, v8, 4 -; RV64-NEXT: vse16.v v9, (a0) -; RV64-NEXT: addi a0, a0, 2 -; RV64-NEXT: andi a2, a1, 32 -; RV64-NEXT: beqz a2, .LBB3_6 -; RV64-NEXT: .LBB3_14: # %cond.store13 -; RV64-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV64-NEXT: vslidedown.vi v9, v8, 5 -; RV64-NEXT: vse16.v v9, (a0) -; RV64-NEXT: addi a0, a0, 2 -; RV64-NEXT: andi a2, a1, 64 -; RV64-NEXT: beqz a2, .LBB3_7 -; RV64-NEXT: .LBB3_15: # %cond.store16 -; RV64-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV64-NEXT: vslidedown.vi v9, v8, 6 -; RV64-NEXT: vse16.v v9, (a0) -; RV64-NEXT: addi a0, a0, 2 -; RV64-NEXT: andi a1, a1, -128 -; RV64-NEXT: beqz a1, .LBB3_8 -; RV64-NEXT: .LBB3_16: # %cond.store19 -; RV64-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; RV64-NEXT: vslidedown.vi v8, v8, 7 -; RV64-NEXT: vse16.v v8, (a0) ; RV64-NEXT: ret call void @llvm.masked.compressstore.v8f16(<8 x half> %v, ptr align 2 %base, <8 x i1> %mask) ret void @@ -346,24 +98,20 @@ declare void @llvm.masked.compressstore.v1f32(<1 x float>, ptr, <1 x i1>) define void @compressstore_v1f32(ptr %base, <1 x float> %v, <1 x i1> %mask) { ; RV32-LABEL: compressstore_v1f32: ; RV32: # %bb.0: -; RV32-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; RV32-NEXT: vfirst.m a1, v0 -; RV32-NEXT: bnez a1, .LBB4_2 -; RV32-NEXT: # %bb.1: # %cond.store ; RV32-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; RV32-NEXT: vse32.v v8, (a0) -; RV32-NEXT: .LBB4_2: # %else +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e32, mf2, ta, ma +; RV32-NEXT: vse32.v v9, (a0) ; RV32-NEXT: ret ; ; RV64-LABEL: compressstore_v1f32: ; RV64: # %bb.0: -; RV64-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; RV64-NEXT: vfirst.m a1, v0 -; RV64-NEXT: bnez a1, .LBB4_2 -; RV64-NEXT: # %bb.1: # %cond.store ; RV64-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; RV64-NEXT: vse32.v v8, (a0) -; RV64-NEXT: .LBB4_2: # %else +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e32, mf2, ta, ma +; RV64-NEXT: vse32.v v9, (a0) ; RV64-NEXT: ret call void @llvm.masked.compressstore.v1f32(<1 x float> %v, ptr align 4 %base, <1 x i1> %mask) ret void @@ -373,48 +121,20 @@ declare void @llvm.masked.compressstore.v2f32(<2 x float>, ptr, <2 x i1>) define void @compressstore_v2f32(ptr %base, <2 x float> %v, <2 x i1> %mask) { ; RV32-LABEL: compressstore_v2f32: ; RV32: # %bb.0: -; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV32-NEXT: vmv.x.s a1, v0 -; RV32-NEXT: andi a2, a1, 1 -; RV32-NEXT: bnez a2, .LBB5_3 -; RV32-NEXT: # %bb.1: # %else -; RV32-NEXT: andi a1, a1, 2 -; RV32-NEXT: bnez a1, .LBB5_4 -; RV32-NEXT: .LBB5_2: # %else2 -; RV32-NEXT: ret -; RV32-NEXT: .LBB5_3: # %cond.store -; RV32-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; RV32-NEXT: vse32.v v8, (a0) -; RV32-NEXT: addi a0, a0, 4 -; RV32-NEXT: andi a1, a1, 2 -; RV32-NEXT: beqz a1, .LBB5_2 -; RV32-NEXT: .LBB5_4: # %cond.store1 -; RV32-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; RV32-NEXT: vslidedown.vi v8, v8, 1 -; RV32-NEXT: vse32.v v8, (a0) +; RV32-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e32, mf2, ta, ma +; RV32-NEXT: vse32.v v9, (a0) ; RV32-NEXT: ret ; ; RV64-LABEL: compressstore_v2f32: ; RV64: # %bb.0: -; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV64-NEXT: vmv.x.s a1, v0 -; RV64-NEXT: andi a2, a1, 1 -; RV64-NEXT: bnez a2, .LBB5_3 -; RV64-NEXT: # %bb.1: # %else -; RV64-NEXT: andi a1, a1, 2 -; RV64-NEXT: bnez a1, .LBB5_4 -; RV64-NEXT: .LBB5_2: # %else2 -; RV64-NEXT: ret -; RV64-NEXT: .LBB5_3: # %cond.store -; RV64-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; RV64-NEXT: vse32.v v8, (a0) -; RV64-NEXT: addi a0, a0, 4 -; RV64-NEXT: andi a1, a1, 2 -; RV64-NEXT: beqz a1, .LBB5_2 -; RV64-NEXT: .LBB5_4: # %cond.store1 -; RV64-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; RV64-NEXT: vslidedown.vi v8, v8, 1 -; RV64-NEXT: vse32.v v8, (a0) +; RV64-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e32, mf2, ta, ma +; RV64-NEXT: vse32.v v9, (a0) ; RV64-NEXT: ret call void @llvm.masked.compressstore.v2f32(<2 x float> %v, ptr align 4 %base, <2 x i1> %mask) ret void @@ -424,88 +144,20 @@ declare void @llvm.masked.compressstore.v4f32(<4 x float>, ptr, <4 x i1>) define void @compressstore_v4f32(ptr %base, <4 x float> %v, <4 x i1> %mask) { ; RV32-LABEL: compressstore_v4f32: ; RV32: # %bb.0: -; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV32-NEXT: vmv.x.s a1, v0 -; RV32-NEXT: andi a2, a1, 1 -; RV32-NEXT: bnez a2, .LBB6_5 -; RV32-NEXT: # %bb.1: # %else -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: bnez a2, .LBB6_6 -; RV32-NEXT: .LBB6_2: # %else2 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: bnez a2, .LBB6_7 -; RV32-NEXT: .LBB6_3: # %else5 -; RV32-NEXT: andi a1, a1, 8 -; RV32-NEXT: bnez a1, .LBB6_8 -; RV32-NEXT: .LBB6_4: # %else8 -; RV32-NEXT: ret -; RV32-NEXT: .LBB6_5: # %cond.store -; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV32-NEXT: vse32.v v8, (a0) -; RV32-NEXT: addi a0, a0, 4 -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: beqz a2, .LBB6_2 -; RV32-NEXT: .LBB6_6: # %cond.store1 -; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV32-NEXT: vslidedown.vi v9, v8, 1 -; RV32-NEXT: vse32.v v9, (a0) -; RV32-NEXT: addi a0, a0, 4 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: beqz a2, .LBB6_3 -; RV32-NEXT: .LBB6_7: # %cond.store4 -; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV32-NEXT: vslidedown.vi v9, v8, 2 +; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e32, m1, ta, ma ; RV32-NEXT: vse32.v v9, (a0) -; RV32-NEXT: addi a0, a0, 4 -; RV32-NEXT: andi a1, a1, 8 -; RV32-NEXT: beqz a1, .LBB6_4 -; RV32-NEXT: .LBB6_8: # %cond.store7 -; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV32-NEXT: vslidedown.vi v8, v8, 3 -; RV32-NEXT: vse32.v v8, (a0) ; RV32-NEXT: ret ; ; RV64-LABEL: compressstore_v4f32: ; RV64: # %bb.0: -; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV64-NEXT: vmv.x.s a1, v0 -; RV64-NEXT: andi a2, a1, 1 -; RV64-NEXT: bnez a2, .LBB6_5 -; RV64-NEXT: # %bb.1: # %else -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: bnez a2, .LBB6_6 -; RV64-NEXT: .LBB6_2: # %else2 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: bnez a2, .LBB6_7 -; RV64-NEXT: .LBB6_3: # %else5 -; RV64-NEXT: andi a1, a1, 8 -; RV64-NEXT: bnez a1, .LBB6_8 -; RV64-NEXT: .LBB6_4: # %else8 -; RV64-NEXT: ret -; RV64-NEXT: .LBB6_5: # %cond.store -; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV64-NEXT: vse32.v v8, (a0) -; RV64-NEXT: addi a0, a0, 4 -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: beqz a2, .LBB6_2 -; RV64-NEXT: .LBB6_6: # %cond.store1 -; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV64-NEXT: vslidedown.vi v9, v8, 1 -; RV64-NEXT: vse32.v v9, (a0) -; RV64-NEXT: addi a0, a0, 4 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: beqz a2, .LBB6_3 -; RV64-NEXT: .LBB6_7: # %cond.store4 -; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV64-NEXT: vslidedown.vi v9, v8, 2 +; RV64-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e32, m1, ta, ma ; RV64-NEXT: vse32.v v9, (a0) -; RV64-NEXT: addi a0, a0, 4 -; RV64-NEXT: andi a1, a1, 8 -; RV64-NEXT: beqz a1, .LBB6_4 -; RV64-NEXT: .LBB6_8: # %cond.store7 -; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV64-NEXT: vslidedown.vi v8, v8, 3 -; RV64-NEXT: vse32.v v8, (a0) ; RV64-NEXT: ret call void @llvm.masked.compressstore.v4f32(<4 x float> %v, ptr align 4 %base, <4 x i1> %mask) ret void @@ -515,176 +167,20 @@ declare void @llvm.masked.compressstore.v8f32(<8 x float>, ptr, <8 x i1>) define void @compressstore_v8f32(ptr %base, <8 x float> %v, <8 x i1> %mask) { ; RV32-LABEL: compressstore_v8f32: ; RV32: # %bb.0: -; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV32-NEXT: vmv.x.s a1, v0 -; RV32-NEXT: andi a2, a1, 1 -; RV32-NEXT: bnez a2, .LBB7_9 -; RV32-NEXT: # %bb.1: # %else -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: bnez a2, .LBB7_10 -; RV32-NEXT: .LBB7_2: # %else2 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: bnez a2, .LBB7_11 -; RV32-NEXT: .LBB7_3: # %else5 -; RV32-NEXT: andi a2, a1, 8 -; RV32-NEXT: bnez a2, .LBB7_12 -; RV32-NEXT: .LBB7_4: # %else8 -; RV32-NEXT: andi a2, a1, 16 -; RV32-NEXT: bnez a2, .LBB7_13 -; RV32-NEXT: .LBB7_5: # %else11 -; RV32-NEXT: andi a2, a1, 32 -; RV32-NEXT: bnez a2, .LBB7_14 -; RV32-NEXT: .LBB7_6: # %else14 -; RV32-NEXT: andi a2, a1, 64 -; RV32-NEXT: bnez a2, .LBB7_15 -; RV32-NEXT: .LBB7_7: # %else17 -; RV32-NEXT: andi a1, a1, -128 -; RV32-NEXT: bnez a1, .LBB7_16 -; RV32-NEXT: .LBB7_8: # %else20 -; RV32-NEXT: ret -; RV32-NEXT: .LBB7_9: # %cond.store -; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV32-NEXT: vse32.v v8, (a0) -; RV32-NEXT: addi a0, a0, 4 -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: beqz a2, .LBB7_2 -; RV32-NEXT: .LBB7_10: # %cond.store1 -; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV32-NEXT: vslidedown.vi v10, v8, 1 -; RV32-NEXT: vse32.v v10, (a0) -; RV32-NEXT: addi a0, a0, 4 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: beqz a2, .LBB7_3 -; RV32-NEXT: .LBB7_11: # %cond.store4 -; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV32-NEXT: vslidedown.vi v10, v8, 2 -; RV32-NEXT: vse32.v v10, (a0) -; RV32-NEXT: addi a0, a0, 4 -; RV32-NEXT: andi a2, a1, 8 -; RV32-NEXT: beqz a2, .LBB7_4 -; RV32-NEXT: .LBB7_12: # %cond.store7 -; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV32-NEXT: vslidedown.vi v10, v8, 3 -; RV32-NEXT: vse32.v v10, (a0) -; RV32-NEXT: addi a0, a0, 4 -; RV32-NEXT: andi a2, a1, 16 -; RV32-NEXT: beqz a2, .LBB7_5 -; RV32-NEXT: .LBB7_13: # %cond.store10 -; RV32-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; RV32-NEXT: vslidedown.vi v10, v8, 4 -; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV32-NEXT: vse32.v v10, (a0) -; RV32-NEXT: addi a0, a0, 4 -; RV32-NEXT: andi a2, a1, 32 -; RV32-NEXT: beqz a2, .LBB7_6 -; RV32-NEXT: .LBB7_14: # %cond.store13 -; RV32-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; RV32-NEXT: vslidedown.vi v10, v8, 5 -; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV32-NEXT: vse32.v v10, (a0) -; RV32-NEXT: addi a0, a0, 4 -; RV32-NEXT: andi a2, a1, 64 -; RV32-NEXT: beqz a2, .LBB7_7 -; RV32-NEXT: .LBB7_15: # %cond.store16 -; RV32-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; RV32-NEXT: vslidedown.vi v10, v8, 6 -; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; RV32-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; RV32-NEXT: vcompress.vm v10, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e32, m2, ta, ma ; RV32-NEXT: vse32.v v10, (a0) -; RV32-NEXT: addi a0, a0, 4 -; RV32-NEXT: andi a1, a1, -128 -; RV32-NEXT: beqz a1, .LBB7_8 -; RV32-NEXT: .LBB7_16: # %cond.store19 -; RV32-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; RV32-NEXT: vslidedown.vi v8, v8, 7 -; RV32-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV32-NEXT: vse32.v v8, (a0) ; RV32-NEXT: ret ; ; RV64-LABEL: compressstore_v8f32: ; RV64: # %bb.0: -; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV64-NEXT: vmv.x.s a1, v0 -; RV64-NEXT: andi a2, a1, 1 -; RV64-NEXT: bnez a2, .LBB7_9 -; RV64-NEXT: # %bb.1: # %else -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: bnez a2, .LBB7_10 -; RV64-NEXT: .LBB7_2: # %else2 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: bnez a2, .LBB7_11 -; RV64-NEXT: .LBB7_3: # %else5 -; RV64-NEXT: andi a2, a1, 8 -; RV64-NEXT: bnez a2, .LBB7_12 -; RV64-NEXT: .LBB7_4: # %else8 -; RV64-NEXT: andi a2, a1, 16 -; RV64-NEXT: bnez a2, .LBB7_13 -; RV64-NEXT: .LBB7_5: # %else11 -; RV64-NEXT: andi a2, a1, 32 -; RV64-NEXT: bnez a2, .LBB7_14 -; RV64-NEXT: .LBB7_6: # %else14 -; RV64-NEXT: andi a2, a1, 64 -; RV64-NEXT: bnez a2, .LBB7_15 -; RV64-NEXT: .LBB7_7: # %else17 -; RV64-NEXT: andi a1, a1, -128 -; RV64-NEXT: bnez a1, .LBB7_16 -; RV64-NEXT: .LBB7_8: # %else20 -; RV64-NEXT: ret -; RV64-NEXT: .LBB7_9: # %cond.store -; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV64-NEXT: vse32.v v8, (a0) -; RV64-NEXT: addi a0, a0, 4 -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: beqz a2, .LBB7_2 -; RV64-NEXT: .LBB7_10: # %cond.store1 -; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV64-NEXT: vslidedown.vi v10, v8, 1 -; RV64-NEXT: vse32.v v10, (a0) -; RV64-NEXT: addi a0, a0, 4 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: beqz a2, .LBB7_3 -; RV64-NEXT: .LBB7_11: # %cond.store4 -; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV64-NEXT: vslidedown.vi v10, v8, 2 -; RV64-NEXT: vse32.v v10, (a0) -; RV64-NEXT: addi a0, a0, 4 -; RV64-NEXT: andi a2, a1, 8 -; RV64-NEXT: beqz a2, .LBB7_4 -; RV64-NEXT: .LBB7_12: # %cond.store7 -; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV64-NEXT: vslidedown.vi v10, v8, 3 -; RV64-NEXT: vse32.v v10, (a0) -; RV64-NEXT: addi a0, a0, 4 -; RV64-NEXT: andi a2, a1, 16 -; RV64-NEXT: beqz a2, .LBB7_5 -; RV64-NEXT: .LBB7_13: # %cond.store10 -; RV64-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; RV64-NEXT: vslidedown.vi v10, v8, 4 -; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV64-NEXT: vse32.v v10, (a0) -; RV64-NEXT: addi a0, a0, 4 -; RV64-NEXT: andi a2, a1, 32 -; RV64-NEXT: beqz a2, .LBB7_6 -; RV64-NEXT: .LBB7_14: # %cond.store13 -; RV64-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; RV64-NEXT: vslidedown.vi v10, v8, 5 -; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV64-NEXT: vse32.v v10, (a0) -; RV64-NEXT: addi a0, a0, 4 -; RV64-NEXT: andi a2, a1, 64 -; RV64-NEXT: beqz a2, .LBB7_7 -; RV64-NEXT: .LBB7_15: # %cond.store16 -; RV64-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; RV64-NEXT: vslidedown.vi v10, v8, 6 -; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; RV64-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; RV64-NEXT: vcompress.vm v10, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e32, m2, ta, ma ; RV64-NEXT: vse32.v v10, (a0) -; RV64-NEXT: addi a0, a0, 4 -; RV64-NEXT: andi a1, a1, -128 -; RV64-NEXT: beqz a1, .LBB7_8 -; RV64-NEXT: .LBB7_16: # %cond.store19 -; RV64-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; RV64-NEXT: vslidedown.vi v8, v8, 7 -; RV64-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; RV64-NEXT: vse32.v v8, (a0) ; RV64-NEXT: ret call void @llvm.masked.compressstore.v8f32(<8 x float> %v, ptr align 4 %base, <8 x i1> %mask) ret void @@ -694,24 +190,20 @@ declare void @llvm.masked.compressstore.v1f64(<1 x double>, ptr, <1 x i1>) define void @compressstore_v1f64(ptr %base, <1 x double> %v, <1 x i1> %mask) { ; RV32-LABEL: compressstore_v1f64: ; RV32: # %bb.0: -; RV32-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; RV32-NEXT: vfirst.m a1, v0 -; RV32-NEXT: bnez a1, .LBB8_2 -; RV32-NEXT: # %bb.1: # %cond.store ; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vse64.v v8, (a0) -; RV32-NEXT: .LBB8_2: # %else +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; RV32-NEXT: vse64.v v9, (a0) ; RV32-NEXT: ret ; ; RV64-LABEL: compressstore_v1f64: ; RV64: # %bb.0: -; RV64-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; RV64-NEXT: vfirst.m a1, v0 -; RV64-NEXT: bnez a1, .LBB8_2 -; RV64-NEXT: # %bb.1: # %cond.store ; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v8, (a0) -; RV64-NEXT: .LBB8_2: # %else +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; RV64-NEXT: vse64.v v9, (a0) ; RV64-NEXT: ret call void @llvm.masked.compressstore.v1f64(<1 x double> %v, ptr align 8 %base, <1 x i1> %mask) ret void @@ -721,48 +213,20 @@ declare void @llvm.masked.compressstore.v2f64(<2 x double>, ptr, <2 x i1>) define void @compressstore_v2f64(ptr %base, <2 x double> %v, <2 x i1> %mask) { ; RV32-LABEL: compressstore_v2f64: ; RV32: # %bb.0: -; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV32-NEXT: vmv.x.s a1, v0 -; RV32-NEXT: andi a2, a1, 1 -; RV32-NEXT: bnez a2, .LBB9_3 -; RV32-NEXT: # %bb.1: # %else -; RV32-NEXT: andi a1, a1, 2 -; RV32-NEXT: bnez a1, .LBB9_4 -; RV32-NEXT: .LBB9_2: # %else2 -; RV32-NEXT: ret -; RV32-NEXT: .LBB9_3: # %cond.store -; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vse64.v v8, (a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a1, a1, 2 -; RV32-NEXT: beqz a1, .LBB9_2 -; RV32-NEXT: .LBB9_4: # %cond.store1 -; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vslidedown.vi v8, v8, 1 -; RV32-NEXT: vse64.v v8, (a0) +; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; RV32-NEXT: vcompress.vm v9, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; RV32-NEXT: vse64.v v9, (a0) ; RV32-NEXT: ret ; ; RV64-LABEL: compressstore_v2f64: ; RV64: # %bb.0: -; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV64-NEXT: vmv.x.s a1, v0 -; RV64-NEXT: andi a2, a1, 1 -; RV64-NEXT: bnez a2, .LBB9_3 -; RV64-NEXT: # %bb.1: # %else -; RV64-NEXT: andi a1, a1, 2 -; RV64-NEXT: bnez a1, .LBB9_4 -; RV64-NEXT: .LBB9_2: # %else2 -; RV64-NEXT: ret -; RV64-NEXT: .LBB9_3: # %cond.store -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v8, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a1, a1, 2 -; RV64-NEXT: beqz a1, .LBB9_2 -; RV64-NEXT: .LBB9_4: # %cond.store1 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vslidedown.vi v8, v8, 1 -; RV64-NEXT: vse64.v v8, (a0) +; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; RV64-NEXT: vcompress.vm v9, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; RV64-NEXT: vse64.v v9, (a0) ; RV64-NEXT: ret call void @llvm.masked.compressstore.v2f64(<2 x double> %v, ptr align 8 %base, <2 x i1> %mask) ret void @@ -772,92 +236,20 @@ declare void @llvm.masked.compressstore.v4f64(<4 x double>, ptr, <4 x i1>) define void @compressstore_v4f64(ptr %base, <4 x double> %v, <4 x i1> %mask) { ; RV32-LABEL: compressstore_v4f64: ; RV32: # %bb.0: -; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV32-NEXT: vmv.x.s a1, v0 -; RV32-NEXT: andi a2, a1, 1 -; RV32-NEXT: bnez a2, .LBB10_5 -; RV32-NEXT: # %bb.1: # %else -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: bnez a2, .LBB10_6 -; RV32-NEXT: .LBB10_2: # %else2 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: bnez a2, .LBB10_7 -; RV32-NEXT: .LBB10_3: # %else5 -; RV32-NEXT: andi a1, a1, 8 -; RV32-NEXT: bnez a1, .LBB10_8 -; RV32-NEXT: .LBB10_4: # %else8 -; RV32-NEXT: ret -; RV32-NEXT: .LBB10_5: # %cond.store -; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vse64.v v8, (a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: beqz a2, .LBB10_2 -; RV32-NEXT: .LBB10_6: # %cond.store1 -; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vslidedown.vi v10, v8, 1 +; RV32-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; RV32-NEXT: vcompress.vm v10, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e64, m2, ta, ma ; RV32-NEXT: vse64.v v10, (a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: beqz a2, .LBB10_3 -; RV32-NEXT: .LBB10_7: # %cond.store4 -; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV32-NEXT: vslidedown.vi v10, v8, 2 -; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vse64.v v10, (a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a1, a1, 8 -; RV32-NEXT: beqz a1, .LBB10_4 -; RV32-NEXT: .LBB10_8: # %cond.store7 -; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV32-NEXT: vslidedown.vi v8, v8, 3 -; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vse64.v v8, (a0) ; RV32-NEXT: ret ; ; RV64-LABEL: compressstore_v4f64: ; RV64: # %bb.0: -; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV64-NEXT: vmv.x.s a1, v0 -; RV64-NEXT: andi a2, a1, 1 -; RV64-NEXT: bnez a2, .LBB10_5 -; RV64-NEXT: # %bb.1: # %else -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: bnez a2, .LBB10_6 -; RV64-NEXT: .LBB10_2: # %else2 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: bnez a2, .LBB10_7 -; RV64-NEXT: .LBB10_3: # %else5 -; RV64-NEXT: andi a1, a1, 8 -; RV64-NEXT: bnez a1, .LBB10_8 -; RV64-NEXT: .LBB10_4: # %else8 -; RV64-NEXT: ret -; RV64-NEXT: .LBB10_5: # %cond.store -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v8, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: beqz a2, .LBB10_2 -; RV64-NEXT: .LBB10_6: # %cond.store1 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vslidedown.vi v10, v8, 1 -; RV64-NEXT: vse64.v v10, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: beqz a2, .LBB10_3 -; RV64-NEXT: .LBB10_7: # %cond.store4 -; RV64-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV64-NEXT: vslidedown.vi v10, v8, 2 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; RV64-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; RV64-NEXT: vcompress.vm v10, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e64, m2, ta, ma ; RV64-NEXT: vse64.v v10, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a1, a1, 8 -; RV64-NEXT: beqz a1, .LBB10_4 -; RV64-NEXT: .LBB10_8: # %cond.store7 -; RV64-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV64-NEXT: vslidedown.vi v8, v8, 3 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v8, (a0) ; RV64-NEXT: ret call void @llvm.masked.compressstore.v4f64(<4 x double> %v, ptr align 8 %base, <4 x i1> %mask) ret void @@ -867,213 +259,21 @@ declare void @llvm.masked.compressstore.v8f64(<8 x double>, ptr, <8 x i1>) define void @compressstore_v8f64(ptr %base, <8 x double> %v, <8 x i1> %mask) { ; RV32-LABEL: compressstore_v8f64: ; RV32: # %bb.0: -; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV32-NEXT: vmv.x.s a1, v0 -; RV32-NEXT: andi a2, a1, 1 -; RV32-NEXT: bnez a2, .LBB11_11 -; RV32-NEXT: # %bb.1: # %else -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: bnez a2, .LBB11_12 -; RV32-NEXT: .LBB11_2: # %else2 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: bnez a2, .LBB11_13 -; RV32-NEXT: .LBB11_3: # %else5 -; RV32-NEXT: andi a2, a1, 8 -; RV32-NEXT: beqz a2, .LBB11_5 -; RV32-NEXT: .LBB11_4: # %cond.store7 -; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV32-NEXT: vslidedown.vi v12, v8, 3 -; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vse64.v v12, (a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: .LBB11_5: # %else8 -; RV32-NEXT: addi sp, sp, -320 -; RV32-NEXT: .cfi_def_cfa_offset 320 -; RV32-NEXT: sw ra, 316(sp) # 4-byte Folded Spill -; RV32-NEXT: sw s0, 312(sp) # 4-byte Folded Spill -; RV32-NEXT: .cfi_offset ra, -4 -; RV32-NEXT: .cfi_offset s0, -8 -; RV32-NEXT: addi s0, sp, 320 -; RV32-NEXT: .cfi_def_cfa s0, 0 -; RV32-NEXT: andi sp, sp, -64 -; RV32-NEXT: andi a2, a1, 16 -; RV32-NEXT: bnez a2, .LBB11_14 -; RV32-NEXT: # %bb.6: # %else11 -; RV32-NEXT: andi a2, a1, 32 -; RV32-NEXT: bnez a2, .LBB11_15 -; RV32-NEXT: .LBB11_7: # %else14 -; RV32-NEXT: andi a2, a1, 64 -; RV32-NEXT: bnez a2, .LBB11_16 -; RV32-NEXT: .LBB11_8: # %else17 -; RV32-NEXT: andi a1, a1, -128 -; RV32-NEXT: beqz a1, .LBB11_10 -; RV32-NEXT: .LBB11_9: # %cond.store19 -; RV32-NEXT: mv a1, sp ; RV32-NEXT: vsetivli zero, 8, e64, m4, ta, ma -; RV32-NEXT: vse64.v v8, (a1) -; RV32-NEXT: fld fa5, 56(sp) -; RV32-NEXT: fsd fa5, 0(a0) -; RV32-NEXT: .LBB11_10: # %else20 -; RV32-NEXT: addi sp, s0, -320 -; RV32-NEXT: lw ra, 316(sp) # 4-byte Folded Reload -; RV32-NEXT: lw s0, 312(sp) # 4-byte Folded Reload -; RV32-NEXT: addi sp, sp, 320 -; RV32-NEXT: ret -; RV32-NEXT: .LBB11_11: # %cond.store -; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vse64.v v8, (a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: beqz a2, .LBB11_2 -; RV32-NEXT: .LBB11_12: # %cond.store1 -; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vslidedown.vi v12, v8, 1 +; RV32-NEXT: vcompress.vm v12, v8, v0 +; RV32-NEXT: vcpop.m a1, v0 +; RV32-NEXT: vsetvli zero, a1, e64, m4, ta, ma ; RV32-NEXT: vse64.v v12, (a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: beqz a2, .LBB11_3 -; RV32-NEXT: .LBB11_13: # %cond.store4 -; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV32-NEXT: vslidedown.vi v12, v8, 2 -; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vse64.v v12, (a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 8 -; RV32-NEXT: bnez a2, .LBB11_4 -; RV32-NEXT: j .LBB11_5 -; RV32-NEXT: .LBB11_14: # %cond.store10 -; RV32-NEXT: addi a2, sp, 192 -; RV32-NEXT: vsetivli zero, 8, e64, m4, ta, ma -; RV32-NEXT: vse64.v v8, (a2) -; RV32-NEXT: fld fa5, 224(sp) -; RV32-NEXT: fsd fa5, 0(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 32 -; RV32-NEXT: beqz a2, .LBB11_7 -; RV32-NEXT: .LBB11_15: # %cond.store13 -; RV32-NEXT: addi a2, sp, 128 -; RV32-NEXT: vsetivli zero, 8, e64, m4, ta, ma -; RV32-NEXT: vse64.v v8, (a2) -; RV32-NEXT: fld fa5, 168(sp) -; RV32-NEXT: fsd fa5, 0(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 64 -; RV32-NEXT: beqz a2, .LBB11_8 -; RV32-NEXT: .LBB11_16: # %cond.store16 -; RV32-NEXT: addi a2, sp, 64 -; RV32-NEXT: vsetivli zero, 8, e64, m4, ta, ma -; RV32-NEXT: vse64.v v8, (a2) -; RV32-NEXT: fld fa5, 112(sp) -; RV32-NEXT: fsd fa5, 0(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a1, a1, -128 -; RV32-NEXT: bnez a1, .LBB11_9 -; RV32-NEXT: j .LBB11_10 +; RV32-NEXT: ret ; ; RV64-LABEL: compressstore_v8f64: ; RV64: # %bb.0: -; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV64-NEXT: vmv.x.s a1, v0 -; RV64-NEXT: andi a2, a1, 1 -; RV64-NEXT: bnez a2, .LBB11_11 -; RV64-NEXT: # %bb.1: # %else -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: bnez a2, .LBB11_12 -; RV64-NEXT: .LBB11_2: # %else2 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: bnez a2, .LBB11_13 -; RV64-NEXT: .LBB11_3: # %else5 -; RV64-NEXT: andi a2, a1, 8 -; RV64-NEXT: beqz a2, .LBB11_5 -; RV64-NEXT: .LBB11_4: # %cond.store7 -; RV64-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV64-NEXT: vslidedown.vi v12, v8, 3 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v12, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: .LBB11_5: # %else8 -; RV64-NEXT: addi sp, sp, -320 -; RV64-NEXT: .cfi_def_cfa_offset 320 -; RV64-NEXT: sd ra, 312(sp) # 8-byte Folded Spill -; RV64-NEXT: sd s0, 304(sp) # 8-byte Folded Spill -; RV64-NEXT: .cfi_offset ra, -8 -; RV64-NEXT: .cfi_offset s0, -16 -; RV64-NEXT: addi s0, sp, 320 -; RV64-NEXT: .cfi_def_cfa s0, 0 -; RV64-NEXT: andi sp, sp, -64 -; RV64-NEXT: andi a2, a1, 16 -; RV64-NEXT: bnez a2, .LBB11_14 -; RV64-NEXT: # %bb.6: # %else11 -; RV64-NEXT: andi a2, a1, 32 -; RV64-NEXT: bnez a2, .LBB11_15 -; RV64-NEXT: .LBB11_7: # %else14 -; RV64-NEXT: andi a2, a1, 64 -; RV64-NEXT: bnez a2, .LBB11_16 -; RV64-NEXT: .LBB11_8: # %else17 -; RV64-NEXT: andi a1, a1, -128 -; RV64-NEXT: beqz a1, .LBB11_10 -; RV64-NEXT: .LBB11_9: # %cond.store19 -; RV64-NEXT: mv a1, sp ; RV64-NEXT: vsetivli zero, 8, e64, m4, ta, ma -; RV64-NEXT: vse64.v v8, (a1) -; RV64-NEXT: fld fa5, 56(sp) -; RV64-NEXT: fsd fa5, 0(a0) -; RV64-NEXT: .LBB11_10: # %else20 -; RV64-NEXT: addi sp, s0, -320 -; RV64-NEXT: ld ra, 312(sp) # 8-byte Folded Reload -; RV64-NEXT: ld s0, 304(sp) # 8-byte Folded Reload -; RV64-NEXT: addi sp, sp, 320 -; RV64-NEXT: ret -; RV64-NEXT: .LBB11_11: # %cond.store -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v8, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: beqz a2, .LBB11_2 -; RV64-NEXT: .LBB11_12: # %cond.store1 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vslidedown.vi v12, v8, 1 +; RV64-NEXT: vcompress.vm v12, v8, v0 +; RV64-NEXT: vcpop.m a1, v0 +; RV64-NEXT: vsetvli zero, a1, e64, m4, ta, ma ; RV64-NEXT: vse64.v v12, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: beqz a2, .LBB11_3 -; RV64-NEXT: .LBB11_13: # %cond.store4 -; RV64-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV64-NEXT: vslidedown.vi v12, v8, 2 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v12, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 8 -; RV64-NEXT: bnez a2, .LBB11_4 -; RV64-NEXT: j .LBB11_5 -; RV64-NEXT: .LBB11_14: # %cond.store10 -; RV64-NEXT: addi a2, sp, 192 -; RV64-NEXT: vsetivli zero, 8, e64, m4, ta, ma -; RV64-NEXT: vse64.v v8, (a2) -; RV64-NEXT: fld fa5, 224(sp) -; RV64-NEXT: fsd fa5, 0(a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 32 -; RV64-NEXT: beqz a2, .LBB11_7 -; RV64-NEXT: .LBB11_15: # %cond.store13 -; RV64-NEXT: addi a2, sp, 128 -; RV64-NEXT: vsetivli zero, 8, e64, m4, ta, ma -; RV64-NEXT: vse64.v v8, (a2) -; RV64-NEXT: fld fa5, 168(sp) -; RV64-NEXT: fsd fa5, 0(a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 64 -; RV64-NEXT: beqz a2, .LBB11_8 -; RV64-NEXT: .LBB11_16: # %cond.store16 -; RV64-NEXT: addi a2, sp, 64 -; RV64-NEXT: vsetivli zero, 8, e64, m4, ta, ma -; RV64-NEXT: vse64.v v8, (a2) -; RV64-NEXT: fld fa5, 112(sp) -; RV64-NEXT: fsd fa5, 0(a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a1, a1, -128 -; RV64-NEXT: bnez a1, .LBB11_9 -; RV64-NEXT: j .LBB11_10 +; RV64-NEXT: ret call void @llvm.masked.compressstore.v8f64(<8 x double> %v, ptr align 8 %base, <8 x i1> %mask) ret void } diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compressstore-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compressstore-int.ll index eb0096dbfba6..a388ba92f302 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compressstore-int.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compressstore-int.ll @@ -6,13 +6,11 @@ declare void @llvm.masked.compressstore.v1i8(<1 x i8>, ptr, <1 x i1>) define void @compressstore_v1i8(ptr %base, <1 x i8> %v, <1 x i1> %mask) { ; CHECK-LABEL: compressstore_v1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; CHECK-NEXT: vfirst.m a1, v0 -; CHECK-NEXT: bnez a1, .LBB0_2 -; CHECK-NEXT: # %bb.1: # %cond.store ; CHECK-NEXT: vsetivli zero, 1, e8, mf8, ta, ma -; CHECK-NEXT: vse8.v v8, (a0) -; CHECK-NEXT: .LBB0_2: # %else +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vse8.v v9, (a0) ; CHECK-NEXT: ret call void @llvm.masked.compressstore.v1i8(<1 x i8> %v, ptr %base, <1 x i1> %mask) ret void @@ -22,25 +20,11 @@ declare void @llvm.masked.compressstore.v2i8(<2 x i8>, ptr, <2 x i1>) define void @compressstore_v2i8(ptr %base, <2 x i8> %v, <2 x i1> %mask) { ; CHECK-LABEL: compressstore_v2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; CHECK-NEXT: vmv.x.s a1, v0 -; CHECK-NEXT: andi a2, a1, 1 -; CHECK-NEXT: bnez a2, .LBB1_3 -; CHECK-NEXT: # %bb.1: # %else -; CHECK-NEXT: andi a1, a1, 2 -; CHECK-NEXT: bnez a1, .LBB1_4 -; CHECK-NEXT: .LBB1_2: # %else2 -; CHECK-NEXT: ret -; CHECK-NEXT: .LBB1_3: # %cond.store -; CHECK-NEXT: vsetivli zero, 1, e8, mf8, ta, ma -; CHECK-NEXT: vse8.v v8, (a0) -; CHECK-NEXT: addi a0, a0, 1 -; CHECK-NEXT: andi a1, a1, 2 -; CHECK-NEXT: beqz a1, .LBB1_2 -; CHECK-NEXT: .LBB1_4: # %cond.store1 -; CHECK-NEXT: vsetivli zero, 1, e8, mf8, ta, ma -; CHECK-NEXT: vslidedown.vi v8, v8, 1 -; CHECK-NEXT: vse8.v v8, (a0) +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vse8.v v9, (a0) ; CHECK-NEXT: ret call void @llvm.masked.compressstore.v2i8(<2 x i8> %v, ptr %base, <2 x i1> %mask) ret void @@ -50,45 +34,11 @@ declare void @llvm.masked.compressstore.v4i8(<4 x i8>, ptr, <4 x i1>) define void @compressstore_v4i8(ptr %base, <4 x i8> %v, <4 x i1> %mask) { ; CHECK-LABEL: compressstore_v4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; CHECK-NEXT: vmv.x.s a1, v0 -; CHECK-NEXT: andi a2, a1, 1 -; CHECK-NEXT: bnez a2, .LBB2_5 -; CHECK-NEXT: # %bb.1: # %else -; CHECK-NEXT: andi a2, a1, 2 -; CHECK-NEXT: bnez a2, .LBB2_6 -; CHECK-NEXT: .LBB2_2: # %else2 -; CHECK-NEXT: andi a2, a1, 4 -; CHECK-NEXT: bnez a2, .LBB2_7 -; CHECK-NEXT: .LBB2_3: # %else5 -; CHECK-NEXT: andi a1, a1, 8 -; CHECK-NEXT: bnez a1, .LBB2_8 -; CHECK-NEXT: .LBB2_4: # %else8 -; CHECK-NEXT: ret -; CHECK-NEXT: .LBB2_5: # %cond.store -; CHECK-NEXT: vsetivli zero, 1, e8, mf4, ta, ma -; CHECK-NEXT: vse8.v v8, (a0) -; CHECK-NEXT: addi a0, a0, 1 -; CHECK-NEXT: andi a2, a1, 2 -; CHECK-NEXT: beqz a2, .LBB2_2 -; CHECK-NEXT: .LBB2_6: # %cond.store1 -; CHECK-NEXT: vsetivli zero, 1, e8, mf4, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 1 +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma ; CHECK-NEXT: vse8.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 1 -; CHECK-NEXT: andi a2, a1, 4 -; CHECK-NEXT: beqz a2, .LBB2_3 -; CHECK-NEXT: .LBB2_7: # %cond.store4 -; CHECK-NEXT: vsetivli zero, 1, e8, mf4, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 2 -; CHECK-NEXT: vse8.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 1 -; CHECK-NEXT: andi a1, a1, 8 -; CHECK-NEXT: beqz a1, .LBB2_4 -; CHECK-NEXT: .LBB2_8: # %cond.store7 -; CHECK-NEXT: vsetivli zero, 1, e8, mf4, ta, ma -; CHECK-NEXT: vslidedown.vi v8, v8, 3 -; CHECK-NEXT: vse8.v v8, (a0) ; CHECK-NEXT: ret call void @llvm.masked.compressstore.v4i8(<4 x i8> %v, ptr %base, <4 x i1> %mask) ret void @@ -98,85 +48,11 @@ declare void @llvm.masked.compressstore.v8i8(<8 x i8>, ptr, <8 x i1>) define void @compressstore_v8i8(ptr %base, <8 x i8> %v, <8 x i1> %mask) { ; CHECK-LABEL: compressstore_v8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; CHECK-NEXT: vmv.x.s a1, v0 -; CHECK-NEXT: andi a2, a1, 1 -; CHECK-NEXT: bnez a2, .LBB3_9 -; CHECK-NEXT: # %bb.1: # %else -; CHECK-NEXT: andi a2, a1, 2 -; CHECK-NEXT: bnez a2, .LBB3_10 -; CHECK-NEXT: .LBB3_2: # %else2 -; CHECK-NEXT: andi a2, a1, 4 -; CHECK-NEXT: bnez a2, .LBB3_11 -; CHECK-NEXT: .LBB3_3: # %else5 -; CHECK-NEXT: andi a2, a1, 8 -; CHECK-NEXT: bnez a2, .LBB3_12 -; CHECK-NEXT: .LBB3_4: # %else8 -; CHECK-NEXT: andi a2, a1, 16 -; CHECK-NEXT: bnez a2, .LBB3_13 -; CHECK-NEXT: .LBB3_5: # %else11 -; CHECK-NEXT: andi a2, a1, 32 -; CHECK-NEXT: bnez a2, .LBB3_14 -; CHECK-NEXT: .LBB3_6: # %else14 -; CHECK-NEXT: andi a2, a1, 64 -; CHECK-NEXT: bnez a2, .LBB3_15 -; CHECK-NEXT: .LBB3_7: # %else17 -; CHECK-NEXT: andi a1, a1, -128 -; CHECK-NEXT: bnez a1, .LBB3_16 -; CHECK-NEXT: .LBB3_8: # %else20 -; CHECK-NEXT: ret -; CHECK-NEXT: .LBB3_9: # %cond.store -; CHECK-NEXT: vsetivli zero, 1, e8, mf2, ta, ma -; CHECK-NEXT: vse8.v v8, (a0) -; CHECK-NEXT: addi a0, a0, 1 -; CHECK-NEXT: andi a2, a1, 2 -; CHECK-NEXT: beqz a2, .LBB3_2 -; CHECK-NEXT: .LBB3_10: # %cond.store1 -; CHECK-NEXT: vsetivli zero, 1, e8, mf2, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 1 -; CHECK-NEXT: vse8.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 1 -; CHECK-NEXT: andi a2, a1, 4 -; CHECK-NEXT: beqz a2, .LBB3_3 -; CHECK-NEXT: .LBB3_11: # %cond.store4 -; CHECK-NEXT: vsetivli zero, 1, e8, mf2, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 2 -; CHECK-NEXT: vse8.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 1 -; CHECK-NEXT: andi a2, a1, 8 -; CHECK-NEXT: beqz a2, .LBB3_4 -; CHECK-NEXT: .LBB3_12: # %cond.store7 -; CHECK-NEXT: vsetivli zero, 1, e8, mf2, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 3 -; CHECK-NEXT: vse8.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 1 -; CHECK-NEXT: andi a2, a1, 16 -; CHECK-NEXT: beqz a2, .LBB3_5 -; CHECK-NEXT: .LBB3_13: # %cond.store10 -; CHECK-NEXT: vsetivli zero, 1, e8, mf2, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 4 -; CHECK-NEXT: vse8.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 1 -; CHECK-NEXT: andi a2, a1, 32 -; CHECK-NEXT: beqz a2, .LBB3_6 -; CHECK-NEXT: .LBB3_14: # %cond.store13 -; CHECK-NEXT: vsetivli zero, 1, e8, mf2, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 5 +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma ; CHECK-NEXT: vse8.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 1 -; CHECK-NEXT: andi a2, a1, 64 -; CHECK-NEXT: beqz a2, .LBB3_7 -; CHECK-NEXT: .LBB3_15: # %cond.store16 -; CHECK-NEXT: vsetivli zero, 1, e8, mf2, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 6 -; CHECK-NEXT: vse8.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 1 -; CHECK-NEXT: andi a1, a1, -128 -; CHECK-NEXT: beqz a1, .LBB3_8 -; CHECK-NEXT: .LBB3_16: # %cond.store19 -; CHECK-NEXT: vsetivli zero, 1, e8, mf2, ta, ma -; CHECK-NEXT: vslidedown.vi v8, v8, 7 -; CHECK-NEXT: vse8.v v8, (a0) ; CHECK-NEXT: ret call void @llvm.masked.compressstore.v8i8(<8 x i8> %v, ptr %base, <8 x i1> %mask) ret void @@ -186,13 +62,11 @@ declare void @llvm.masked.compressstore.v1i16(<1 x i16>, ptr, <1 x i1>) define void @compressstore_v1i16(ptr %base, <1 x i16> %v, <1 x i1> %mask) { ; CHECK-LABEL: compressstore_v1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; CHECK-NEXT: vfirst.m a1, v0 -; CHECK-NEXT: bnez a1, .LBB4_2 -; CHECK-NEXT: # %bb.1: # %cond.store ; CHECK-NEXT: vsetivli zero, 1, e16, mf4, ta, ma -; CHECK-NEXT: vse16.v v8, (a0) -; CHECK-NEXT: .LBB4_2: # %else +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e16, mf4, ta, ma +; CHECK-NEXT: vse16.v v9, (a0) ; CHECK-NEXT: ret call void @llvm.masked.compressstore.v1i16(<1 x i16> %v, ptr align 2 %base, <1 x i1> %mask) ret void @@ -202,25 +76,11 @@ declare void @llvm.masked.compressstore.v2i16(<2 x i16>, ptr, <2 x i1>) define void @compressstore_v2i16(ptr %base, <2 x i16> %v, <2 x i1> %mask) { ; CHECK-LABEL: compressstore_v2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; CHECK-NEXT: vmv.x.s a1, v0 -; CHECK-NEXT: andi a2, a1, 1 -; CHECK-NEXT: bnez a2, .LBB5_3 -; CHECK-NEXT: # %bb.1: # %else -; CHECK-NEXT: andi a1, a1, 2 -; CHECK-NEXT: bnez a1, .LBB5_4 -; CHECK-NEXT: .LBB5_2: # %else2 -; CHECK-NEXT: ret -; CHECK-NEXT: .LBB5_3: # %cond.store -; CHECK-NEXT: vsetivli zero, 1, e16, mf4, ta, ma -; CHECK-NEXT: vse16.v v8, (a0) -; CHECK-NEXT: addi a0, a0, 2 -; CHECK-NEXT: andi a1, a1, 2 -; CHECK-NEXT: beqz a1, .LBB5_2 -; CHECK-NEXT: .LBB5_4: # %cond.store1 -; CHECK-NEXT: vsetivli zero, 1, e16, mf4, ta, ma -; CHECK-NEXT: vslidedown.vi v8, v8, 1 -; CHECK-NEXT: vse16.v v8, (a0) +; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e16, mf4, ta, ma +; CHECK-NEXT: vse16.v v9, (a0) ; CHECK-NEXT: ret call void @llvm.masked.compressstore.v2i16(<2 x i16> %v, ptr align 2 %base, <2 x i1> %mask) ret void @@ -230,45 +90,11 @@ declare void @llvm.masked.compressstore.v4i16(<4 x i16>, ptr, <4 x i1>) define void @compressstore_v4i16(ptr %base, <4 x i16> %v, <4 x i1> %mask) { ; CHECK-LABEL: compressstore_v4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; CHECK-NEXT: vmv.x.s a1, v0 -; CHECK-NEXT: andi a2, a1, 1 -; CHECK-NEXT: bnez a2, .LBB6_5 -; CHECK-NEXT: # %bb.1: # %else -; CHECK-NEXT: andi a2, a1, 2 -; CHECK-NEXT: bnez a2, .LBB6_6 -; CHECK-NEXT: .LBB6_2: # %else2 -; CHECK-NEXT: andi a2, a1, 4 -; CHECK-NEXT: bnez a2, .LBB6_7 -; CHECK-NEXT: .LBB6_3: # %else5 -; CHECK-NEXT: andi a1, a1, 8 -; CHECK-NEXT: bnez a1, .LBB6_8 -; CHECK-NEXT: .LBB6_4: # %else8 -; CHECK-NEXT: ret -; CHECK-NEXT: .LBB6_5: # %cond.store -; CHECK-NEXT: vsetivli zero, 1, e16, mf2, ta, ma -; CHECK-NEXT: vse16.v v8, (a0) -; CHECK-NEXT: addi a0, a0, 2 -; CHECK-NEXT: andi a2, a1, 2 -; CHECK-NEXT: beqz a2, .LBB6_2 -; CHECK-NEXT: .LBB6_6: # %cond.store1 -; CHECK-NEXT: vsetivli zero, 1, e16, mf2, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 1 -; CHECK-NEXT: vse16.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 2 -; CHECK-NEXT: andi a2, a1, 4 -; CHECK-NEXT: beqz a2, .LBB6_3 -; CHECK-NEXT: .LBB6_7: # %cond.store4 -; CHECK-NEXT: vsetivli zero, 1, e16, mf2, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 2 +; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e16, mf2, ta, ma ; CHECK-NEXT: vse16.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 2 -; CHECK-NEXT: andi a1, a1, 8 -; CHECK-NEXT: beqz a1, .LBB6_4 -; CHECK-NEXT: .LBB6_8: # %cond.store7 -; CHECK-NEXT: vsetivli zero, 1, e16, mf2, ta, ma -; CHECK-NEXT: vslidedown.vi v8, v8, 3 -; CHECK-NEXT: vse16.v v8, (a0) ; CHECK-NEXT: ret call void @llvm.masked.compressstore.v4i16(<4 x i16> %v, ptr align 2 %base, <4 x i1> %mask) ret void @@ -278,85 +104,11 @@ declare void @llvm.masked.compressstore.v8i16(<8 x i16>, ptr, <8 x i1>) define void @compressstore_v8i16(ptr %base, <8 x i16> %v, <8 x i1> %mask) { ; CHECK-LABEL: compressstore_v8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; CHECK-NEXT: vmv.x.s a1, v0 -; CHECK-NEXT: andi a2, a1, 1 -; CHECK-NEXT: bnez a2, .LBB7_9 -; CHECK-NEXT: # %bb.1: # %else -; CHECK-NEXT: andi a2, a1, 2 -; CHECK-NEXT: bnez a2, .LBB7_10 -; CHECK-NEXT: .LBB7_2: # %else2 -; CHECK-NEXT: andi a2, a1, 4 -; CHECK-NEXT: bnez a2, .LBB7_11 -; CHECK-NEXT: .LBB7_3: # %else5 -; CHECK-NEXT: andi a2, a1, 8 -; CHECK-NEXT: bnez a2, .LBB7_12 -; CHECK-NEXT: .LBB7_4: # %else8 -; CHECK-NEXT: andi a2, a1, 16 -; CHECK-NEXT: bnez a2, .LBB7_13 -; CHECK-NEXT: .LBB7_5: # %else11 -; CHECK-NEXT: andi a2, a1, 32 -; CHECK-NEXT: bnez a2, .LBB7_14 -; CHECK-NEXT: .LBB7_6: # %else14 -; CHECK-NEXT: andi a2, a1, 64 -; CHECK-NEXT: bnez a2, .LBB7_15 -; CHECK-NEXT: .LBB7_7: # %else17 -; CHECK-NEXT: andi a1, a1, -128 -; CHECK-NEXT: bnez a1, .LBB7_16 -; CHECK-NEXT: .LBB7_8: # %else20 -; CHECK-NEXT: ret -; CHECK-NEXT: .LBB7_9: # %cond.store -; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; CHECK-NEXT: vse16.v v8, (a0) -; CHECK-NEXT: addi a0, a0, 2 -; CHECK-NEXT: andi a2, a1, 2 -; CHECK-NEXT: beqz a2, .LBB7_2 -; CHECK-NEXT: .LBB7_10: # %cond.store1 -; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 1 -; CHECK-NEXT: vse16.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 2 -; CHECK-NEXT: andi a2, a1, 4 -; CHECK-NEXT: beqz a2, .LBB7_3 -; CHECK-NEXT: .LBB7_11: # %cond.store4 -; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 2 -; CHECK-NEXT: vse16.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 2 -; CHECK-NEXT: andi a2, a1, 8 -; CHECK-NEXT: beqz a2, .LBB7_4 -; CHECK-NEXT: .LBB7_12: # %cond.store7 -; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 3 -; CHECK-NEXT: vse16.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 2 -; CHECK-NEXT: andi a2, a1, 16 -; CHECK-NEXT: beqz a2, .LBB7_5 -; CHECK-NEXT: .LBB7_13: # %cond.store10 -; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 4 +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e16, m1, ta, ma ; CHECK-NEXT: vse16.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 2 -; CHECK-NEXT: andi a2, a1, 32 -; CHECK-NEXT: beqz a2, .LBB7_6 -; CHECK-NEXT: .LBB7_14: # %cond.store13 -; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 5 -; CHECK-NEXT: vse16.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 2 -; CHECK-NEXT: andi a2, a1, 64 -; CHECK-NEXT: beqz a2, .LBB7_7 -; CHECK-NEXT: .LBB7_15: # %cond.store16 -; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 6 -; CHECK-NEXT: vse16.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 2 -; CHECK-NEXT: andi a1, a1, -128 -; CHECK-NEXT: beqz a1, .LBB7_8 -; CHECK-NEXT: .LBB7_16: # %cond.store19 -; CHECK-NEXT: vsetivli zero, 1, e16, m1, ta, ma -; CHECK-NEXT: vslidedown.vi v8, v8, 7 -; CHECK-NEXT: vse16.v v8, (a0) ; CHECK-NEXT: ret call void @llvm.masked.compressstore.v8i16(<8 x i16> %v, ptr align 2 %base, <8 x i1> %mask) ret void @@ -366,13 +118,11 @@ declare void @llvm.masked.compressstore.v1i32(<1 x i32>, ptr, <1 x i1>) define void @compressstore_v1i32(ptr %base, <1 x i32> %v, <1 x i1> %mask) { ; CHECK-LABEL: compressstore_v1i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; CHECK-NEXT: vfirst.m a1, v0 -; CHECK-NEXT: bnez a1, .LBB8_2 -; CHECK-NEXT: # %bb.1: # %cond.store ; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-NEXT: vse32.v v8, (a0) -; CHECK-NEXT: .LBB8_2: # %else +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e32, mf2, ta, ma +; CHECK-NEXT: vse32.v v9, (a0) ; CHECK-NEXT: ret call void @llvm.masked.compressstore.v1i32(<1 x i32> %v, ptr align 4 %base, <1 x i1> %mask) ret void @@ -382,25 +132,11 @@ declare void @llvm.masked.compressstore.v2i32(<2 x i32>, ptr, <2 x i1>) define void @compressstore_v2i32(ptr %base, <2 x i32> %v, <2 x i1> %mask) { ; CHECK-LABEL: compressstore_v2i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; CHECK-NEXT: vmv.x.s a1, v0 -; CHECK-NEXT: andi a2, a1, 1 -; CHECK-NEXT: bnez a2, .LBB9_3 -; CHECK-NEXT: # %bb.1: # %else -; CHECK-NEXT: andi a1, a1, 2 -; CHECK-NEXT: bnez a1, .LBB9_4 -; CHECK-NEXT: .LBB9_2: # %else2 -; CHECK-NEXT: ret -; CHECK-NEXT: .LBB9_3: # %cond.store -; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-NEXT: vse32.v v8, (a0) -; CHECK-NEXT: addi a0, a0, 4 -; CHECK-NEXT: andi a1, a1, 2 -; CHECK-NEXT: beqz a1, .LBB9_2 -; CHECK-NEXT: .LBB9_4: # %cond.store1 -; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-NEXT: vslidedown.vi v8, v8, 1 -; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e32, mf2, ta, ma +; CHECK-NEXT: vse32.v v9, (a0) ; CHECK-NEXT: ret call void @llvm.masked.compressstore.v2i32(<2 x i32> %v, ptr align 4 %base, <2 x i1> %mask) ret void @@ -410,45 +146,11 @@ declare void @llvm.masked.compressstore.v4i32(<4 x i32>, ptr, <4 x i1>) define void @compressstore_v4i32(ptr %base, <4 x i32> %v, <4 x i1> %mask) { ; CHECK-LABEL: compressstore_v4i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; CHECK-NEXT: vmv.x.s a1, v0 -; CHECK-NEXT: andi a2, a1, 1 -; CHECK-NEXT: bnez a2, .LBB10_5 -; CHECK-NEXT: # %bb.1: # %else -; CHECK-NEXT: andi a2, a1, 2 -; CHECK-NEXT: bnez a2, .LBB10_6 -; CHECK-NEXT: .LBB10_2: # %else2 -; CHECK-NEXT: andi a2, a1, 4 -; CHECK-NEXT: bnez a2, .LBB10_7 -; CHECK-NEXT: .LBB10_3: # %else5 -; CHECK-NEXT: andi a1, a1, 8 -; CHECK-NEXT: bnez a1, .LBB10_8 -; CHECK-NEXT: .LBB10_4: # %else8 -; CHECK-NEXT: ret -; CHECK-NEXT: .LBB10_5: # %cond.store -; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-NEXT: vse32.v v8, (a0) -; CHECK-NEXT: addi a0, a0, 4 -; CHECK-NEXT: andi a2, a1, 2 -; CHECK-NEXT: beqz a2, .LBB10_2 -; CHECK-NEXT: .LBB10_6: # %cond.store1 -; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 1 -; CHECK-NEXT: vse32.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 4 -; CHECK-NEXT: andi a2, a1, 4 -; CHECK-NEXT: beqz a2, .LBB10_3 -; CHECK-NEXT: .LBB10_7: # %cond.store4 -; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-NEXT: vslidedown.vi v9, v8, 2 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma ; CHECK-NEXT: vse32.v v9, (a0) -; CHECK-NEXT: addi a0, a0, 4 -; CHECK-NEXT: andi a1, a1, 8 -; CHECK-NEXT: beqz a1, .LBB10_4 -; CHECK-NEXT: .LBB10_8: # %cond.store7 -; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-NEXT: vslidedown.vi v8, v8, 3 -; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: ret call void @llvm.masked.compressstore.v4i32(<4 x i32> %v, ptr align 4 %base, <4 x i1> %mask) ret void @@ -458,89 +160,11 @@ declare void @llvm.masked.compressstore.v8i32(<8 x i32>, ptr, <8 x i1>) define void @compressstore_v8i32(ptr %base, <8 x i32> %v, <8 x i1> %mask) { ; CHECK-LABEL: compressstore_v8i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; CHECK-NEXT: vmv.x.s a1, v0 -; CHECK-NEXT: andi a2, a1, 1 -; CHECK-NEXT: bnez a2, .LBB11_9 -; CHECK-NEXT: # %bb.1: # %else -; CHECK-NEXT: andi a2, a1, 2 -; CHECK-NEXT: bnez a2, .LBB11_10 -; CHECK-NEXT: .LBB11_2: # %else2 -; CHECK-NEXT: andi a2, a1, 4 -; CHECK-NEXT: bnez a2, .LBB11_11 -; CHECK-NEXT: .LBB11_3: # %else5 -; CHECK-NEXT: andi a2, a1, 8 -; CHECK-NEXT: bnez a2, .LBB11_12 -; CHECK-NEXT: .LBB11_4: # %else8 -; CHECK-NEXT: andi a2, a1, 16 -; CHECK-NEXT: bnez a2, .LBB11_13 -; CHECK-NEXT: .LBB11_5: # %else11 -; CHECK-NEXT: andi a2, a1, 32 -; CHECK-NEXT: bnez a2, .LBB11_14 -; CHECK-NEXT: .LBB11_6: # %else14 -; CHECK-NEXT: andi a2, a1, 64 -; CHECK-NEXT: bnez a2, .LBB11_15 -; CHECK-NEXT: .LBB11_7: # %else17 -; CHECK-NEXT: andi a1, a1, -128 -; CHECK-NEXT: bnez a1, .LBB11_16 -; CHECK-NEXT: .LBB11_8: # %else20 -; CHECK-NEXT: ret -; CHECK-NEXT: .LBB11_9: # %cond.store -; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-NEXT: vse32.v v8, (a0) -; CHECK-NEXT: addi a0, a0, 4 -; CHECK-NEXT: andi a2, a1, 2 -; CHECK-NEXT: beqz a2, .LBB11_2 -; CHECK-NEXT: .LBB11_10: # %cond.store1 -; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-NEXT: vslidedown.vi v10, v8, 1 -; CHECK-NEXT: vse32.v v10, (a0) -; CHECK-NEXT: addi a0, a0, 4 -; CHECK-NEXT: andi a2, a1, 4 -; CHECK-NEXT: beqz a2, .LBB11_3 -; CHECK-NEXT: .LBB11_11: # %cond.store4 -; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-NEXT: vslidedown.vi v10, v8, 2 -; CHECK-NEXT: vse32.v v10, (a0) -; CHECK-NEXT: addi a0, a0, 4 -; CHECK-NEXT: andi a2, a1, 8 -; CHECK-NEXT: beqz a2, .LBB11_4 -; CHECK-NEXT: .LBB11_12: # %cond.store7 -; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-NEXT: vslidedown.vi v10, v8, 3 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vcompress.vm v10, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e32, m2, ta, ma ; CHECK-NEXT: vse32.v v10, (a0) -; CHECK-NEXT: addi a0, a0, 4 -; CHECK-NEXT: andi a2, a1, 16 -; CHECK-NEXT: beqz a2, .LBB11_5 -; CHECK-NEXT: .LBB11_13: # %cond.store10 -; CHECK-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; CHECK-NEXT: vslidedown.vi v10, v8, 4 -; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-NEXT: vse32.v v10, (a0) -; CHECK-NEXT: addi a0, a0, 4 -; CHECK-NEXT: andi a2, a1, 32 -; CHECK-NEXT: beqz a2, .LBB11_6 -; CHECK-NEXT: .LBB11_14: # %cond.store13 -; CHECK-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; CHECK-NEXT: vslidedown.vi v10, v8, 5 -; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-NEXT: vse32.v v10, (a0) -; CHECK-NEXT: addi a0, a0, 4 -; CHECK-NEXT: andi a2, a1, 64 -; CHECK-NEXT: beqz a2, .LBB11_7 -; CHECK-NEXT: .LBB11_15: # %cond.store16 -; CHECK-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; CHECK-NEXT: vslidedown.vi v10, v8, 6 -; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-NEXT: vse32.v v10, (a0) -; CHECK-NEXT: addi a0, a0, 4 -; CHECK-NEXT: andi a1, a1, -128 -; CHECK-NEXT: beqz a1, .LBB11_8 -; CHECK-NEXT: .LBB11_16: # %cond.store19 -; CHECK-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; CHECK-NEXT: vslidedown.vi v8, v8, 7 -; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: ret call void @llvm.masked.compressstore.v8i32(<8 x i32> %v, ptr align 4 %base, <8 x i1> %mask) ret void @@ -548,439 +172,59 @@ define void @compressstore_v8i32(ptr %base, <8 x i32> %v, <8 x i1> %mask) { declare void @llvm.masked.compressstore.v1i64(<1 x i64>, ptr, <1 x i1>) define void @compressstore_v1i64(ptr %base, <1 x i64> %v, <1 x i1> %mask) { -; RV32-LABEL: compressstore_v1i64: -; RV32: # %bb.0: -; RV32-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; RV32-NEXT: vfirst.m a1, v0 -; RV32-NEXT: bnez a1, .LBB12_2 -; RV32-NEXT: # %bb.1: # %cond.store -; RV32-NEXT: li a1, 32 -; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vsrl.vx v9, v8, a1 -; RV32-NEXT: vmv.x.s a1, v9 -; RV32-NEXT: vmv.x.s a2, v8 -; RV32-NEXT: sw a2, 0(a0) -; RV32-NEXT: sw a1, 4(a0) -; RV32-NEXT: .LBB12_2: # %else -; RV32-NEXT: ret -; -; RV64-LABEL: compressstore_v1i64: -; RV64: # %bb.0: -; RV64-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; RV64-NEXT: vfirst.m a1, v0 -; RV64-NEXT: bnez a1, .LBB12_2 -; RV64-NEXT: # %bb.1: # %cond.store -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v8, (a0) -; RV64-NEXT: .LBB12_2: # %else -; RV64-NEXT: ret +; CHECK-LABEL: compressstore_v1i64: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; CHECK-NEXT: vse64.v v9, (a0) +; CHECK-NEXT: ret call void @llvm.masked.compressstore.v1i64(<1 x i64> %v, ptr align 8 %base, <1 x i1> %mask) ret void } declare void @llvm.masked.compressstore.v2i64(<2 x i64>, ptr, <2 x i1>) define void @compressstore_v2i64(ptr %base, <2 x i64> %v, <2 x i1> %mask) { -; RV32-LABEL: compressstore_v2i64: -; RV32: # %bb.0: -; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV32-NEXT: vmv.x.s a1, v0 -; RV32-NEXT: andi a2, a1, 1 -; RV32-NEXT: bnez a2, .LBB13_3 -; RV32-NEXT: # %bb.1: # %else -; RV32-NEXT: andi a1, a1, 2 -; RV32-NEXT: bnez a1, .LBB13_4 -; RV32-NEXT: .LBB13_2: # %else2 -; RV32-NEXT: ret -; RV32-NEXT: .LBB13_3: # %cond.store -; RV32-NEXT: li a2, 32 -; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vsrl.vx v9, v8, a2 -; RV32-NEXT: vmv.x.s a2, v9 -; RV32-NEXT: vmv.x.s a3, v8 -; RV32-NEXT: sw a3, 0(a0) -; RV32-NEXT: sw a2, 4(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a1, a1, 2 -; RV32-NEXT: beqz a1, .LBB13_2 -; RV32-NEXT: .LBB13_4: # %cond.store1 -; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV32-NEXT: vslidedown.vi v8, v8, 1 -; RV32-NEXT: li a1, 32 -; RV32-NEXT: vsrl.vx v9, v8, a1 -; RV32-NEXT: vmv.x.s a1, v9 -; RV32-NEXT: vmv.x.s a2, v8 -; RV32-NEXT: sw a2, 0(a0) -; RV32-NEXT: sw a1, 4(a0) -; RV32-NEXT: ret -; -; RV64-LABEL: compressstore_v2i64: -; RV64: # %bb.0: -; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV64-NEXT: vmv.x.s a1, v0 -; RV64-NEXT: andi a2, a1, 1 -; RV64-NEXT: bnez a2, .LBB13_3 -; RV64-NEXT: # %bb.1: # %else -; RV64-NEXT: andi a1, a1, 2 -; RV64-NEXT: bnez a1, .LBB13_4 -; RV64-NEXT: .LBB13_2: # %else2 -; RV64-NEXT: ret -; RV64-NEXT: .LBB13_3: # %cond.store -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v8, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a1, a1, 2 -; RV64-NEXT: beqz a1, .LBB13_2 -; RV64-NEXT: .LBB13_4: # %cond.store1 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vslidedown.vi v8, v8, 1 -; RV64-NEXT: vse64.v v8, (a0) -; RV64-NEXT: ret +; CHECK-LABEL: compressstore_v2i64: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; CHECK-NEXT: vse64.v v9, (a0) +; CHECK-NEXT: ret call void @llvm.masked.compressstore.v2i64(<2 x i64> %v, ptr align 8 %base, <2 x i1> %mask) ret void } declare void @llvm.masked.compressstore.v4i64(<4 x i64>, ptr, <4 x i1>) define void @compressstore_v4i64(ptr %base, <4 x i64> %v, <4 x i1> %mask) { -; RV32-LABEL: compressstore_v4i64: -; RV32: # %bb.0: -; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV32-NEXT: vmv.x.s a1, v0 -; RV32-NEXT: andi a2, a1, 1 -; RV32-NEXT: bnez a2, .LBB14_5 -; RV32-NEXT: # %bb.1: # %else -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: bnez a2, .LBB14_6 -; RV32-NEXT: .LBB14_2: # %else2 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: bnez a2, .LBB14_7 -; RV32-NEXT: .LBB14_3: # %else5 -; RV32-NEXT: andi a1, a1, 8 -; RV32-NEXT: bnez a1, .LBB14_8 -; RV32-NEXT: .LBB14_4: # %else8 -; RV32-NEXT: ret -; RV32-NEXT: .LBB14_5: # %cond.store -; RV32-NEXT: li a2, 32 -; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV32-NEXT: vsrl.vx v10, v8, a2 -; RV32-NEXT: vmv.x.s a2, v10 -; RV32-NEXT: vmv.x.s a3, v8 -; RV32-NEXT: sw a3, 0(a0) -; RV32-NEXT: sw a2, 4(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: beqz a2, .LBB14_2 -; RV32-NEXT: .LBB14_6: # %cond.store1 -; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV32-NEXT: vslidedown.vi v10, v8, 1 -; RV32-NEXT: li a2, 32 -; RV32-NEXT: vsrl.vx v12, v10, a2 -; RV32-NEXT: vmv.x.s a2, v12 -; RV32-NEXT: vmv.x.s a3, v10 -; RV32-NEXT: sw a3, 0(a0) -; RV32-NEXT: sw a2, 4(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: beqz a2, .LBB14_3 -; RV32-NEXT: .LBB14_7: # %cond.store4 -; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV32-NEXT: vslidedown.vi v10, v8, 2 -; RV32-NEXT: li a2, 32 -; RV32-NEXT: vsrl.vx v12, v10, a2 -; RV32-NEXT: vmv.x.s a2, v12 -; RV32-NEXT: vmv.x.s a3, v10 -; RV32-NEXT: sw a3, 0(a0) -; RV32-NEXT: sw a2, 4(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a1, a1, 8 -; RV32-NEXT: beqz a1, .LBB14_4 -; RV32-NEXT: .LBB14_8: # %cond.store7 -; RV32-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV32-NEXT: vslidedown.vi v8, v8, 3 -; RV32-NEXT: li a1, 32 -; RV32-NEXT: vsrl.vx v10, v8, a1 -; RV32-NEXT: vmv.x.s a1, v10 -; RV32-NEXT: vmv.x.s a2, v8 -; RV32-NEXT: sw a2, 0(a0) -; RV32-NEXT: sw a1, 4(a0) -; RV32-NEXT: ret -; -; RV64-LABEL: compressstore_v4i64: -; RV64: # %bb.0: -; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV64-NEXT: vmv.x.s a1, v0 -; RV64-NEXT: andi a2, a1, 1 -; RV64-NEXT: bnez a2, .LBB14_5 -; RV64-NEXT: # %bb.1: # %else -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: bnez a2, .LBB14_6 -; RV64-NEXT: .LBB14_2: # %else2 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: bnez a2, .LBB14_7 -; RV64-NEXT: .LBB14_3: # %else5 -; RV64-NEXT: andi a1, a1, 8 -; RV64-NEXT: bnez a1, .LBB14_8 -; RV64-NEXT: .LBB14_4: # %else8 -; RV64-NEXT: ret -; RV64-NEXT: .LBB14_5: # %cond.store -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v8, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: beqz a2, .LBB14_2 -; RV64-NEXT: .LBB14_6: # %cond.store1 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vslidedown.vi v10, v8, 1 -; RV64-NEXT: vse64.v v10, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: beqz a2, .LBB14_3 -; RV64-NEXT: .LBB14_7: # %cond.store4 -; RV64-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV64-NEXT: vslidedown.vi v10, v8, 2 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v10, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a1, a1, 8 -; RV64-NEXT: beqz a1, .LBB14_4 -; RV64-NEXT: .LBB14_8: # %cond.store7 -; RV64-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV64-NEXT: vslidedown.vi v8, v8, 3 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v8, (a0) -; RV64-NEXT: ret +; CHECK-LABEL: compressstore_v4i64: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-NEXT: vcompress.vm v10, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e64, m2, ta, ma +; CHECK-NEXT: vse64.v v10, (a0) +; CHECK-NEXT: ret call void @llvm.masked.compressstore.v4i64(<4 x i64> %v, ptr align 8 %base, <4 x i1> %mask) ret void } declare void @llvm.masked.compressstore.v8i64(<8 x i64>, ptr, <8 x i1>) define void @compressstore_v8i64(ptr %base, <8 x i64> %v, <8 x i1> %mask) { -; RV32-LABEL: compressstore_v8i64: -; RV32: # %bb.0: -; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV32-NEXT: vmv.x.s a1, v0 -; RV32-NEXT: andi a2, a1, 1 -; RV32-NEXT: bnez a2, .LBB15_9 -; RV32-NEXT: # %bb.1: # %else -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: bnez a2, .LBB15_10 -; RV32-NEXT: .LBB15_2: # %else2 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: bnez a2, .LBB15_11 -; RV32-NEXT: .LBB15_3: # %else5 -; RV32-NEXT: andi a2, a1, 8 -; RV32-NEXT: bnez a2, .LBB15_12 -; RV32-NEXT: .LBB15_4: # %else8 -; RV32-NEXT: andi a2, a1, 16 -; RV32-NEXT: bnez a2, .LBB15_13 -; RV32-NEXT: .LBB15_5: # %else11 -; RV32-NEXT: andi a2, a1, 32 -; RV32-NEXT: bnez a2, .LBB15_14 -; RV32-NEXT: .LBB15_6: # %else14 -; RV32-NEXT: andi a2, a1, 64 -; RV32-NEXT: bnez a2, .LBB15_15 -; RV32-NEXT: .LBB15_7: # %else17 -; RV32-NEXT: andi a1, a1, -128 -; RV32-NEXT: bnez a1, .LBB15_16 -; RV32-NEXT: .LBB15_8: # %else20 -; RV32-NEXT: ret -; RV32-NEXT: .LBB15_9: # %cond.store -; RV32-NEXT: li a2, 32 -; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma -; RV32-NEXT: vsrl.vx v12, v8, a2 -; RV32-NEXT: vmv.x.s a2, v12 -; RV32-NEXT: vmv.x.s a3, v8 -; RV32-NEXT: sw a3, 0(a0) -; RV32-NEXT: sw a2, 4(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 2 -; RV32-NEXT: beqz a2, .LBB15_2 -; RV32-NEXT: .LBB15_10: # %cond.store1 -; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma -; RV32-NEXT: vslidedown.vi v12, v8, 1 -; RV32-NEXT: li a2, 32 -; RV32-NEXT: vsrl.vx v16, v12, a2 -; RV32-NEXT: vmv.x.s a2, v16 -; RV32-NEXT: vmv.x.s a3, v12 -; RV32-NEXT: sw a3, 0(a0) -; RV32-NEXT: sw a2, 4(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 4 -; RV32-NEXT: beqz a2, .LBB15_3 -; RV32-NEXT: .LBB15_11: # %cond.store4 -; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma -; RV32-NEXT: vslidedown.vi v12, v8, 2 -; RV32-NEXT: li a2, 32 -; RV32-NEXT: vsrl.vx v16, v12, a2 -; RV32-NEXT: vmv.x.s a2, v16 -; RV32-NEXT: vmv.x.s a3, v12 -; RV32-NEXT: sw a3, 0(a0) -; RV32-NEXT: sw a2, 4(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 8 -; RV32-NEXT: beqz a2, .LBB15_4 -; RV32-NEXT: .LBB15_12: # %cond.store7 -; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma -; RV32-NEXT: vslidedown.vi v12, v8, 3 -; RV32-NEXT: li a2, 32 -; RV32-NEXT: vsrl.vx v16, v12, a2 -; RV32-NEXT: vmv.x.s a2, v16 -; RV32-NEXT: vmv.x.s a3, v12 -; RV32-NEXT: sw a3, 0(a0) -; RV32-NEXT: sw a2, 4(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 16 -; RV32-NEXT: beqz a2, .LBB15_5 -; RV32-NEXT: .LBB15_13: # %cond.store10 -; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma -; RV32-NEXT: vslidedown.vi v12, v8, 4 -; RV32-NEXT: li a2, 32 -; RV32-NEXT: vsrl.vx v16, v12, a2 -; RV32-NEXT: vmv.x.s a2, v16 -; RV32-NEXT: vmv.x.s a3, v12 -; RV32-NEXT: sw a3, 0(a0) -; RV32-NEXT: sw a2, 4(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 32 -; RV32-NEXT: beqz a2, .LBB15_6 -; RV32-NEXT: .LBB15_14: # %cond.store13 -; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma -; RV32-NEXT: vslidedown.vi v12, v8, 5 -; RV32-NEXT: li a2, 32 -; RV32-NEXT: vsrl.vx v16, v12, a2 -; RV32-NEXT: vmv.x.s a2, v16 -; RV32-NEXT: vmv.x.s a3, v12 -; RV32-NEXT: sw a3, 0(a0) -; RV32-NEXT: sw a2, 4(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a2, a1, 64 -; RV32-NEXT: beqz a2, .LBB15_7 -; RV32-NEXT: .LBB15_15: # %cond.store16 -; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma -; RV32-NEXT: vslidedown.vi v12, v8, 6 -; RV32-NEXT: li a2, 32 -; RV32-NEXT: vsrl.vx v16, v12, a2 -; RV32-NEXT: vmv.x.s a2, v16 -; RV32-NEXT: vmv.x.s a3, v12 -; RV32-NEXT: sw a3, 0(a0) -; RV32-NEXT: sw a2, 4(a0) -; RV32-NEXT: addi a0, a0, 8 -; RV32-NEXT: andi a1, a1, -128 -; RV32-NEXT: beqz a1, .LBB15_8 -; RV32-NEXT: .LBB15_16: # %cond.store19 -; RV32-NEXT: vsetivli zero, 1, e64, m4, ta, ma -; RV32-NEXT: vslidedown.vi v8, v8, 7 -; RV32-NEXT: li a1, 32 -; RV32-NEXT: vsrl.vx v12, v8, a1 -; RV32-NEXT: vmv.x.s a1, v12 -; RV32-NEXT: vmv.x.s a2, v8 -; RV32-NEXT: sw a2, 0(a0) -; RV32-NEXT: sw a1, 4(a0) -; RV32-NEXT: ret -; -; RV64-LABEL: compressstore_v8i64: -; RV64: # %bb.0: -; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma -; RV64-NEXT: vmv.x.s a1, v0 -; RV64-NEXT: andi a2, a1, 1 -; RV64-NEXT: bnez a2, .LBB15_11 -; RV64-NEXT: # %bb.1: # %else -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: bnez a2, .LBB15_12 -; RV64-NEXT: .LBB15_2: # %else2 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: bnez a2, .LBB15_13 -; RV64-NEXT: .LBB15_3: # %else5 -; RV64-NEXT: andi a2, a1, 8 -; RV64-NEXT: beqz a2, .LBB15_5 -; RV64-NEXT: .LBB15_4: # %cond.store7 -; RV64-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV64-NEXT: vslidedown.vi v12, v8, 3 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v12, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: .LBB15_5: # %else8 -; RV64-NEXT: addi sp, sp, -320 -; RV64-NEXT: .cfi_def_cfa_offset 320 -; RV64-NEXT: sd ra, 312(sp) # 8-byte Folded Spill -; RV64-NEXT: sd s0, 304(sp) # 8-byte Folded Spill -; RV64-NEXT: .cfi_offset ra, -8 -; RV64-NEXT: .cfi_offset s0, -16 -; RV64-NEXT: addi s0, sp, 320 -; RV64-NEXT: .cfi_def_cfa s0, 0 -; RV64-NEXT: andi sp, sp, -64 -; RV64-NEXT: andi a2, a1, 16 -; RV64-NEXT: bnez a2, .LBB15_14 -; RV64-NEXT: # %bb.6: # %else11 -; RV64-NEXT: andi a2, a1, 32 -; RV64-NEXT: bnez a2, .LBB15_15 -; RV64-NEXT: .LBB15_7: # %else14 -; RV64-NEXT: andi a2, a1, 64 -; RV64-NEXT: bnez a2, .LBB15_16 -; RV64-NEXT: .LBB15_8: # %else17 -; RV64-NEXT: andi a1, a1, -128 -; RV64-NEXT: beqz a1, .LBB15_10 -; RV64-NEXT: .LBB15_9: # %cond.store19 -; RV64-NEXT: mv a1, sp -; RV64-NEXT: vsetivli zero, 8, e64, m4, ta, ma -; RV64-NEXT: vse64.v v8, (a1) -; RV64-NEXT: ld a1, 56(sp) -; RV64-NEXT: sd a1, 0(a0) -; RV64-NEXT: .LBB15_10: # %else20 -; RV64-NEXT: addi sp, s0, -320 -; RV64-NEXT: ld ra, 312(sp) # 8-byte Folded Reload -; RV64-NEXT: ld s0, 304(sp) # 8-byte Folded Reload -; RV64-NEXT: addi sp, sp, 320 -; RV64-NEXT: ret -; RV64-NEXT: .LBB15_11: # %cond.store -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v8, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 2 -; RV64-NEXT: beqz a2, .LBB15_2 -; RV64-NEXT: .LBB15_12: # %cond.store1 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vslidedown.vi v12, v8, 1 -; RV64-NEXT: vse64.v v12, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 4 -; RV64-NEXT: beqz a2, .LBB15_3 -; RV64-NEXT: .LBB15_13: # %cond.store4 -; RV64-NEXT: vsetivli zero, 1, e64, m2, ta, ma -; RV64-NEXT: vslidedown.vi v12, v8, 2 -; RV64-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; RV64-NEXT: vse64.v v12, (a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 8 -; RV64-NEXT: bnez a2, .LBB15_4 -; RV64-NEXT: j .LBB15_5 -; RV64-NEXT: .LBB15_14: # %cond.store10 -; RV64-NEXT: addi a2, sp, 192 -; RV64-NEXT: vsetivli zero, 8, e64, m4, ta, ma -; RV64-NEXT: vse64.v v8, (a2) -; RV64-NEXT: ld a2, 224(sp) -; RV64-NEXT: sd a2, 0(a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 32 -; RV64-NEXT: beqz a2, .LBB15_7 -; RV64-NEXT: .LBB15_15: # %cond.store13 -; RV64-NEXT: addi a2, sp, 128 -; RV64-NEXT: vsetivli zero, 8, e64, m4, ta, ma -; RV64-NEXT: vse64.v v8, (a2) -; RV64-NEXT: ld a2, 168(sp) -; RV64-NEXT: sd a2, 0(a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a2, a1, 64 -; RV64-NEXT: beqz a2, .LBB15_8 -; RV64-NEXT: .LBB15_16: # %cond.store16 -; RV64-NEXT: addi a2, sp, 64 -; RV64-NEXT: vsetivli zero, 8, e64, m4, ta, ma -; RV64-NEXT: vse64.v v8, (a2) -; RV64-NEXT: ld a2, 112(sp) -; RV64-NEXT: sd a2, 0(a0) -; RV64-NEXT: addi a0, a0, 8 -; RV64-NEXT: andi a1, a1, -128 -; RV64-NEXT: bnez a1, .LBB15_9 -; RV64-NEXT: j .LBB15_10 +; CHECK-LABEL: compressstore_v8i64: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, ma +; CHECK-NEXT: vcompress.vm v12, v8, v0 +; CHECK-NEXT: vcpop.m a1, v0 +; CHECK-NEXT: vsetvli zero, a1, e64, m4, ta, ma +; CHECK-NEXT: vse64.v v12, (a0) +; CHECK-NEXT: ret call void @llvm.masked.compressstore.v8i64(<8 x i64> %v, ptr align 8 %base, <8 x i1> %mask) ret void } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; RV32: {{.*}} +; RV64: {{.*}} -- GitLab From fbd7c50065705c44e1b3d39f456963810124051b Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Wed, 13 Mar 2024 14:25:22 -0500 Subject: [PATCH 0178/1407] [libc] Repurpose `LIBC_GPU_BUILD` option to enable the new format (#82848) Summary: We previously used the `LIBC_GPU_BUILD` option to control whether or not the GPU build was enabled. This was recently replaced with a new format that allows treating the GPU targets more directly. However, the new format is somewhat difficult to use for people unfamiliar with the runtimes builds, and the removal of this option somewhat broke backward compatibility. This patch seeks to simplify enabling the GPU build by repurposing the old enabling option and convert it to the new interface. Unsure what the rules are here, since this is technically a `LIBC` option living in the LLVM location. --- libc/CMakeLists.txt | 1 - llvm/CMakeLists.txt | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt index 6edf5c656193..7afb3c5f0faa 100644 --- a/libc/CMakeLists.txt +++ b/libc/CMakeLists.txt @@ -134,7 +134,6 @@ option(LLVM_LIBC_FULL_BUILD "Build and test LLVM libc as if it is the full libc" option(LLVM_LIBC_IMPLEMENTATION_DEFINED_TEST_BEHAVIOR "Build LLVM libc tests assuming our implementation-defined behavior" ON) option(LLVM_LIBC_ENABLE_LINTING "Enables linting of libc source files" OFF) -option(LIBC_GPU_BUILD "Build libc for the GPU. All CPU build options will be ignored." OFF) set(LIBC_TARGET_TRIPLE "" CACHE STRING "The target triple for the libc build.") option(LIBC_CONFIG_PATH "The path to user provided folder that configures the build for the target system." OFF) diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt index bd141619d03f..6f5647d70d8b 100644 --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -159,6 +159,18 @@ foreach(proj IN LISTS LLVM_ENABLE_RUNTIMES) endif() endforeach() +# Set a shorthand option to enable the GPU build of the 'libc' project. +option(LIBC_GPU_BUILD "Enable the 'libc' project targeting the GPU" OFF) +if(LIBC_GPU_BUILD) + if(LLVM_RUNTIME_TARGETS) + list(APPEND LLVM_RUNTIME_TARGETS "nvptx64-nvidia-cuda" "amdgcn-amd-amdhsa") + else() + set(LLVM_RUNTIME_TARGETS "default;nvptx64-nvidia-cuda;amdgcn-amd-amdhsa") + endif() + list(APPEND RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES "libc") + list(APPEND RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES "libc") +endif() + set(NEED_LIBC_HDRGEN FALSE) if("libc" IN_LIST LLVM_ENABLE_RUNTIMES) set(NEED_LIBC_HDRGEN TRUE) -- GitLab From 882992a951a3d92d5e19d4fe6c6eb9ba1e87d39c Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Sun, 10 Mar 2024 17:24:09 -0500 Subject: [PATCH 0179/1407] [ValueTracking] Add tests for inferring select arm bits from condition; NFC --- .../knownbits-select-from-cond.ll | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 llvm/test/Analysis/ValueTracking/knownbits-select-from-cond.ll diff --git a/llvm/test/Analysis/ValueTracking/knownbits-select-from-cond.ll b/llvm/test/Analysis/ValueTracking/knownbits-select-from-cond.ll new file mode 100644 index 000000000000..0a1cccaca2bb --- /dev/null +++ b/llvm/test/Analysis/ValueTracking/knownbits-select-from-cond.ll @@ -0,0 +1,81 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -passes=instcombine -S < %s | FileCheck %s + +define i8 @select_condition_implies_highbits_op1(i8 %xx, i8 noundef %y) { +; CHECK-LABEL: @select_condition_implies_highbits_op1( +; CHECK-NEXT: [[X:%.*]] = and i8 [[XX:%.*]], 15 +; CHECK-NEXT: [[COND:%.*]] = icmp ult i8 [[Y:%.*]], 3 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[COND]], i8 [[Y]], i8 [[X]] +; CHECK-NEXT: [[R:%.*]] = add i8 [[SEL]], 32 +; CHECK-NEXT: ret i8 [[R]] +; + %x = and i8 %xx, 15 + %cond = icmp ult i8 %y, 3 + %sel = select i1 %cond, i8 %y, i8 %x + %r = add i8 %sel, 32 + ret i8 %r +} + +define i8 @select_condition_implies_highbits_op1_maybe_undef_fail(i8 %xx, i8 %y) { +; CHECK-LABEL: @select_condition_implies_highbits_op1_maybe_undef_fail( +; CHECK-NEXT: [[X:%.*]] = and i8 [[XX:%.*]], 15 +; CHECK-NEXT: [[COND:%.*]] = icmp ult i8 [[Y:%.*]], 3 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[COND]], i8 [[Y]], i8 [[X]] +; CHECK-NEXT: [[R:%.*]] = add i8 [[SEL]], 32 +; CHECK-NEXT: ret i8 [[R]] +; + %x = and i8 %xx, 15 + %cond = icmp ult i8 %y, 3 + %sel = select i1 %cond, i8 %y, i8 %x + %r = add i8 %sel, 32 + ret i8 %r +} + +define i8 @select_condition_implies_highbits_op2(i8 %xx, i8 noundef %y) { +; CHECK-LABEL: @select_condition_implies_highbits_op2( +; CHECK-NEXT: [[X:%.*]] = and i8 [[XX:%.*]], 15 +; CHECK-NEXT: [[COND:%.*]] = icmp ugt i8 [[Y:%.*]], 3 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[COND]], i8 [[X]], i8 [[Y]] +; CHECK-NEXT: [[R:%.*]] = add i8 [[SEL]], 32 +; CHECK-NEXT: ret i8 [[R]] +; + %x = and i8 %xx, 15 + %cond = icmp ugt i8 %y, 3 + %sel = select i1 %cond, i8 %x, i8 %y + %r = add i8 %sel, 32 + ret i8 %r +} + +define i8 @select_condition_implies_highbits_op1_and(i8 %xx, i8 noundef %y, i1 %other_cond) { +; CHECK-LABEL: @select_condition_implies_highbits_op1_and( +; CHECK-NEXT: [[X:%.*]] = and i8 [[XX:%.*]], 15 +; CHECK-NEXT: [[COND0:%.*]] = icmp ult i8 [[Y:%.*]], 3 +; CHECK-NEXT: [[COND:%.*]] = and i1 [[COND0]], [[OTHER_COND:%.*]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[COND]], i8 [[Y]], i8 [[X]] +; CHECK-NEXT: [[R:%.*]] = add i8 [[SEL]], 32 +; CHECK-NEXT: ret i8 [[R]] +; + %x = and i8 %xx, 15 + %cond0 = icmp ult i8 %y, 3 + %cond = and i1 %cond0, %other_cond + %sel = select i1 %cond, i8 %y, i8 %x + %r = add i8 %sel, 32 + ret i8 %r +} + +define i8 @select_condition_implies_highbits_op2_or(i8 %xx, i8 noundef %y, i1 %other_cond) { +; CHECK-LABEL: @select_condition_implies_highbits_op2_or( +; CHECK-NEXT: [[X:%.*]] = and i8 [[XX:%.*]], 15 +; CHECK-NEXT: [[COND0:%.*]] = icmp ugt i8 [[Y:%.*]], 3 +; CHECK-NEXT: [[COND:%.*]] = or i1 [[COND0]], [[OTHER_COND:%.*]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[COND]], i8 [[X]], i8 [[Y]] +; CHECK-NEXT: [[R:%.*]] = add i8 [[SEL]], 32 +; CHECK-NEXT: ret i8 [[R]] +; + %x = and i8 %xx, 15 + %cond0 = icmp ugt i8 %y, 3 + %cond = or i1 %cond0, %other_cond + %sel = select i1 %cond, i8 %x, i8 %y + %r = add i8 %sel, 32 + ret i8 %r +} -- GitLab From 744a23f24b08e8b988b176173c433d64761e66b3 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Sun, 10 Mar 2024 17:24:12 -0500 Subject: [PATCH 0180/1407] [ValueTracking] Use select condition to help infer bits of arms If we have something like `(select (icmp ult x, 8), x, y)`, we can use the `(icmp ult x, 8)` to help compute the knownbits of `x`. Closes #84699 --- llvm/lib/Analysis/ValueTracking.cpp | 41 +++++++++++++++++-- .../knownbits-select-from-cond.ll | 8 ++-- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 8a4a2c4f92a0..edbeede910d7 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -1023,11 +1023,44 @@ static void computeKnownBitsFromOperator(const Operator *I, break; } case Instruction::Select: { - computeKnownBits(I->getOperand(2), Known, Depth + 1, Q); - computeKnownBits(I->getOperand(1), Known2, Depth + 1, Q); - + auto ComputeForArm = [&](Value *Arm, bool Invert) { + KnownBits Res(Known.getBitWidth()); + computeKnownBits(Arm, Res, Depth + 1, Q); + // If we have a constant arm, we are done. + if (Res.isConstant()) + return Res; + + // See what condition implies about the bits of the two select arms. + KnownBits CondRes(Res.getBitWidth()); + computeKnownBitsFromCond(Arm, I->getOperand(0), CondRes, Depth + 1, Q, + Invert); + // If we don't get any information from the condition, no reason to + // proceed. + if (CondRes.isUnknown()) + return Res; + + // We can have conflict if the condition is dead. I.e if we have + // (x | 64) < 32 ? (x | 64) : y + // we will have conflict at bit 6 from the condition/the `or`. + // In that case just return. Its not particularly important + // what we do, as this select is going to be simplified soon. + CondRes = CondRes.unionWith(Res); + if (CondRes.hasConflict()) + return Res; + + // Finally make sure the information we found is valid. This is relatively + // expensive so it's left for the very end. + if (!isGuaranteedNotToBeUndef(Arm, Q.AC, Q.CxtI, Q.DT, Depth + 1)) + return Res; + + // Finally, we know we get information from the condition and its valid, + // so return it. + return CondRes; + }; // Only known if known in both the LHS and RHS. - Known = Known.intersectWith(Known2); + Known = + ComputeForArm(I->getOperand(1), /*Invert=*/false) + .intersectWith(ComputeForArm(I->getOperand(2), /*Invert=*/true)); break; } case Instruction::FPTrunc: diff --git a/llvm/test/Analysis/ValueTracking/knownbits-select-from-cond.ll b/llvm/test/Analysis/ValueTracking/knownbits-select-from-cond.ll index 0a1cccaca2bb..c3343edfb4c9 100644 --- a/llvm/test/Analysis/ValueTracking/knownbits-select-from-cond.ll +++ b/llvm/test/Analysis/ValueTracking/knownbits-select-from-cond.ll @@ -6,7 +6,7 @@ define i8 @select_condition_implies_highbits_op1(i8 %xx, i8 noundef %y) { ; CHECK-NEXT: [[X:%.*]] = and i8 [[XX:%.*]], 15 ; CHECK-NEXT: [[COND:%.*]] = icmp ult i8 [[Y:%.*]], 3 ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[COND]], i8 [[Y]], i8 [[X]] -; CHECK-NEXT: [[R:%.*]] = add i8 [[SEL]], 32 +; CHECK-NEXT: [[R:%.*]] = or disjoint i8 [[SEL]], 32 ; CHECK-NEXT: ret i8 [[R]] ; %x = and i8 %xx, 15 @@ -36,7 +36,7 @@ define i8 @select_condition_implies_highbits_op2(i8 %xx, i8 noundef %y) { ; CHECK-NEXT: [[X:%.*]] = and i8 [[XX:%.*]], 15 ; CHECK-NEXT: [[COND:%.*]] = icmp ugt i8 [[Y:%.*]], 3 ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[COND]], i8 [[X]], i8 [[Y]] -; CHECK-NEXT: [[R:%.*]] = add i8 [[SEL]], 32 +; CHECK-NEXT: [[R:%.*]] = or disjoint i8 [[SEL]], 32 ; CHECK-NEXT: ret i8 [[R]] ; %x = and i8 %xx, 15 @@ -52,7 +52,7 @@ define i8 @select_condition_implies_highbits_op1_and(i8 %xx, i8 noundef %y, i1 % ; CHECK-NEXT: [[COND0:%.*]] = icmp ult i8 [[Y:%.*]], 3 ; CHECK-NEXT: [[COND:%.*]] = and i1 [[COND0]], [[OTHER_COND:%.*]] ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[COND]], i8 [[Y]], i8 [[X]] -; CHECK-NEXT: [[R:%.*]] = add i8 [[SEL]], 32 +; CHECK-NEXT: [[R:%.*]] = or disjoint i8 [[SEL]], 32 ; CHECK-NEXT: ret i8 [[R]] ; %x = and i8 %xx, 15 @@ -69,7 +69,7 @@ define i8 @select_condition_implies_highbits_op2_or(i8 %xx, i8 noundef %y, i1 %o ; CHECK-NEXT: [[COND0:%.*]] = icmp ugt i8 [[Y:%.*]], 3 ; CHECK-NEXT: [[COND:%.*]] = or i1 [[COND0]], [[OTHER_COND:%.*]] ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[COND]], i8 [[X]], i8 [[Y]] -; CHECK-NEXT: [[R:%.*]] = add i8 [[SEL]], 32 +; CHECK-NEXT: [[R:%.*]] = or disjoint i8 [[SEL]], 32 ; CHECK-NEXT: ret i8 [[R]] ; %x = and i8 %xx, 15 -- GitLab From 8237520eb42b37d7ed353d64a865d3ba5ac24ec6 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 13 Mar 2024 07:45:14 -0700 Subject: [PATCH 0181/1407] [SLP]Fix PR85082: PHI node has multiple entries. Need to record casted extractelement for the externally used scalar, not original extract instruction. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 16 ++-- .../X86/same-scalar-in-same-phi-extract.ll | 75 +++++++++++++++++++ 2 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index b8b67609d755..1da509ce4794 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -12582,6 +12582,7 @@ Value *BoUpSLP::vectorizeTree( Ex = I; } } + Value *ExV = Ex; if (!Ex) { // "Reuse" the existing extract to improve final codegen. if (auto *ES = dyn_cast(Scalar)) { @@ -12592,7 +12593,13 @@ Value *BoUpSLP::vectorizeTree( } else { Ex = Builder.CreateExtractElement(Vec, Lane); } - if (auto *I = dyn_cast(Ex)) + // If necessary, sign-extend or zero-extend ScalarRoot + // to the larger type. + ExV = Ex; + if (Scalar->getType() != Ex->getType()) + ExV = Builder.CreateIntCast(Ex, Scalar->getType(), + MinBWs.find(E)->second.second); + if (auto *I = dyn_cast(ExV)) ScalarToEEs[Scalar].try_emplace(Builder.GetInsertBlock(), I); } // The then branch of the previous if may produce constants, since 0 @@ -12601,12 +12608,7 @@ Value *BoUpSLP::vectorizeTree( GatherShuffleExtractSeq.insert(ExI); CSEBlocks.insert(ExI->getParent()); } - // If necessary, sign-extend or zero-extend ScalarRoot - // to the larger type. - if (Scalar->getType() != Ex->getType()) - return Builder.CreateIntCast(Ex, Scalar->getType(), - MinBWs.find(E)->second.second); - return Ex; + return ExV; } assert(isa(Scalar->getType()) && isa(Scalar) && diff --git a/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll b/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll new file mode 100644 index 000000000000..35f2f9e052e7 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll @@ -0,0 +1,75 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S --passes=slp-vectorizer -slp-threshold=-99999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s + +define void @test(i32 %arg) { +; CHECK-LABEL: define void @test( +; CHECK-SAME: i32 [[ARG:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> , i32 [[ARG]], i32 0 +; CHECK-NEXT: br label [[BB2:%.*]] +; CHECK: bb2: +; CHECK-NEXT: switch i32 0, label [[BB10:%.*]] [ +; CHECK-NEXT: i32 0, label [[BB9:%.*]] +; CHECK-NEXT: i32 11, label [[BB9]] +; CHECK-NEXT: i32 1, label [[BB4:%.*]] +; CHECK-NEXT: ] +; CHECK: bb3: +; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0 +; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 +; CHECK-NEXT: switch i32 0, label [[BB10]] [ +; CHECK-NEXT: i32 18, label [[BB7:%.*]] +; CHECK-NEXT: i32 1, label [[BB7]] +; CHECK-NEXT: i32 0, label [[BB10]] +; CHECK-NEXT: ] +; CHECK: bb4: +; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x i32> [ [[TMP0]], [[BB2]] ] +; CHECK-NEXT: [[TMP4:%.*]] = zext <2 x i32> [[TMP3]] to <2 x i64> +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0 +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr null, i64 [[TMP5]] +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1 +; CHECK-NEXT: [[GETELEMENTPTR6:%.*]] = getelementptr i32, ptr null, i64 [[TMP6]] +; CHECK-NEXT: ret void +; CHECK: bb7: +; CHECK-NEXT: [[PHI8:%.*]] = phi i64 [ [[TMP2]], [[BB3:%.*]] ], [ [[TMP2]], [[BB3]] ] +; CHECK-NEXT: br label [[BB9]] +; CHECK: bb9: +; CHECK-NEXT: ret void +; CHECK: bb10: +; CHECK-NEXT: ret void +; +bb: + %zext = zext i32 %arg to i64 + %zext1 = zext i32 0 to i64 + br label %bb2 + +bb2: + switch i32 0, label %bb10 [ + i32 0, label %bb9 + i32 11, label %bb9 + i32 1, label %bb4 + ] + +bb3: + switch i32 0, label %bb10 [ + i32 18, label %bb7 + i32 1, label %bb7 + i32 0, label %bb10 + ] + +bb4: + %phi = phi i64 [ %zext, %bb2 ] + %phi5 = phi i64 [ %zext1, %bb2 ] + %getelementptr = getelementptr i32, ptr null, i64 %phi + %getelementptr6 = getelementptr i32, ptr null, i64 %phi5 + ret void + +bb7: + %phi8 = phi i64 [ %zext, %bb3 ], [ %zext, %bb3 ] + br label %bb9 + +bb9: + ret void + +bb10: + ret void +} -- GitLab From 1f973efd335f34c75fcba1ccbe288fd5ece15a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Wed, 13 Mar 2024 22:01:01 +0200 Subject: [PATCH 0182/1407] [runtimes] Prefer -fvisibility-global-new-delete=force-hidden (#84917) 27ce26b06655cfece3d54b30e442ef93d3e78ac7 added the new option -fvisibility-global-new-delete=, where -fvisibility-global-new-delete=force-hidden is equivalent to the old option -fvisibility-global-new-delete-hidden. At the same time, the old option was deprecated. Test for and use the new option form first; if unsupported, try using the old form. This avoids warnings in the MinGW builds, if built with Clang 18 or newer. --- libcxx/src/CMakeLists.txt | 5 ++++- libcxxabi/src/CMakeLists.txt | 5 ++++- libunwind/src/CMakeLists.txt | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt index 07ffc8bfdaae..1110a79ddcac 100644 --- a/libcxx/src/CMakeLists.txt +++ b/libcxx/src/CMakeLists.txt @@ -301,7 +301,10 @@ if (LIBCXX_ENABLE_STATIC) # then its code shouldn't declare them with hidden visibility. They might # actually be provided by a shared library at link time. if (LIBCXX_ENABLE_NEW_DELETE_DEFINITIONS) - append_flags_if_supported(CXX_STATIC_LIBRARY_FLAGS -fvisibility-global-new-delete-hidden) + append_flags_if_supported(CXX_STATIC_LIBRARY_FLAGS -fvisibility-global-new-delete=force-hidden) + if (NOT CXX_SUPPORTS_FVISIBILITY_GLOBAL_NEW_DELETE_EQ_FORCE_HIDDEN_FLAG) + append_flags_if_supported(CXX_STATIC_LIBRARY_FLAGS -fvisibility-global-new-delete-hidden) + endif() endif() target_compile_options(cxx_static PRIVATE ${CXX_STATIC_LIBRARY_FLAGS}) # _LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS can be defined in __config_site diff --git a/libcxxabi/src/CMakeLists.txt b/libcxxabi/src/CMakeLists.txt index 0af4dc1448e9..c8cc93de5077 100644 --- a/libcxxabi/src/CMakeLists.txt +++ b/libcxxabi/src/CMakeLists.txt @@ -268,7 +268,10 @@ if(LIBCXXABI_HERMETIC_STATIC_LIBRARY) # then its code shouldn't declare them with hidden visibility. They might # actually be provided by a shared library at link time. if (LIBCXXABI_ENABLE_NEW_DELETE_DEFINITIONS) - target_add_compile_flags_if_supported(cxxabi_static_objects PRIVATE -fvisibility-global-new-delete-hidden) + target_add_compile_flags_if_supported(cxxabi_static_objects PRIVATE -fvisibility-global-new-delete=force-hidden) + if (NOT CXX_SUPPORTS_FVISIBILITY_GLOBAL_NEW_DELETE_EQ_FORCE_HIDDEN_FLAG) + target_add_compile_flags_if_supported(cxxabi_static_objects PRIVATE -fvisibility-global-new-delete-hidden) + endif() endif() # _LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS can be defined in libcxx's # __config_site too. Define it in the same way here, to avoid redefinition diff --git a/libunwind/src/CMakeLists.txt b/libunwind/src/CMakeLists.txt index 9c6f5d908b09..780430ba70ba 100644 --- a/libunwind/src/CMakeLists.txt +++ b/libunwind/src/CMakeLists.txt @@ -201,7 +201,10 @@ set_target_properties(unwind_static_objects if(LIBUNWIND_HIDE_SYMBOLS) target_add_compile_flags_if_supported(unwind_static_objects PRIVATE -fvisibility=hidden) - target_add_compile_flags_if_supported(unwind_static_objects PRIVATE -fvisibility-global-new-delete-hidden) + target_add_compile_flags_if_supported(unwind_static_objects PRIVATE -fvisibility-global-new-delete=force-hidden) + if (NOT CXX_SUPPORTS_FVISIBILITY_GLOBAL_NEW_DELETE_EQ_FORCE_HIDDEN_FLAG) + target_add_compile_flags_if_supported(unwind_static_objects PRIVATE -fvisibility-global-new-delete-hidden) + endif() target_compile_definitions(unwind_static_objects PRIVATE _LIBUNWIND_HIDE_SYMBOLS) endif() -- GitLab From cd8843f87af2f04a85dda12b37738596cbf4cd5e Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Wed, 13 Mar 2024 15:05:22 -0500 Subject: [PATCH 0183/1407] [OpenMP] Disable flaky barrier fence test (#85093) Summary: This test is flaky on all targets I know of. We should disable it for now so running the test suite doesn't randomly fail 50% of the time. --- openmp/libomptarget/test/offloading/barrier_fence.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openmp/libomptarget/test/offloading/barrier_fence.c b/openmp/libomptarget/test/offloading/barrier_fence.c index 5d1096478ed9..3eeaac52a2d6 100644 --- a/openmp/libomptarget/test/offloading/barrier_fence.c +++ b/openmp/libomptarget/test/offloading/barrier_fence.c @@ -3,6 +3,10 @@ // RUN: %libomptarget-compileopt-generic -fopenmp-offload-mandatory -O3 // RUN: %libomptarget-run-generic +// FIXME: This test is flaky on all targets +// UNSUPPORTED: amdgcn-amd-amdhsa +// UNSUPPORTED: nvptx64-nvidia-cuda +// UNSUPPORTED: nvptx64-nvidia-cuda-LTO // UNSUPPORTED: aarch64-unknown-linux-gnu // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu -- GitLab From 8bed754c2f965c8cbbb050be6f650b78f7fd78a6 Mon Sep 17 00:00:00 2001 From: Jordan Rupprecht Date: Wed, 13 Mar 2024 15:16:15 -0500 Subject: [PATCH 0184/1407] [lldb][test] Add `pexpect` category for tests that `import pexpect` (#84860) Instead of directly annotating pexpect-based tests with `@skipIfWindows`, we can tag them with a new `pexpect` category. We still automatically skip windows behavior by adding `pexpect` to the skip category list if the platform is windows, but also allow non-Windows users to skip them by configuring cmake with `-DLLDB_TEST_USER_ARGS=--skip-category=pexpect` As a prerequisite, remove the restriction that `@add_test_categories` can only apply to test cases, and we make the test runner look for categories on both the class and the test method. --- lldb/packages/Python/lldbsuite/test/decorators.py | 4 ---- lldb/packages/Python/lldbsuite/test/dotest.py | 13 +++++++++++++ lldb/packages/Python/lldbsuite/test/lldbpexpect.py | 2 +- .../Python/lldbsuite/test/test_categories.py | 1 + lldb/packages/Python/lldbsuite/test/test_result.py | 6 ++++-- .../API/benchmarks/expression/TestExpressionCmd.py | 5 +---- .../API/benchmarks/expression/TestRepeatedExprs.py | 5 +---- .../frame_variable/TestFrameVariableResponse.py | 5 +---- .../API/benchmarks/startup/TestStartupDelays.py | 5 +---- .../API/benchmarks/stepping/TestSteppingSpeed.py | 5 +---- .../TestCompileRunToBreakpointTurnaround.py | 5 +---- lldb/test/API/terminal/TestSTTYBeforeAndAfter.py | 2 +- 12 files changed, 26 insertions(+), 32 deletions(-) diff --git a/lldb/packages/Python/lldbsuite/test/decorators.py b/lldb/packages/Python/lldbsuite/test/decorators.py index b691f82b9065..8e13aa6a1388 100644 --- a/lldb/packages/Python/lldbsuite/test/decorators.py +++ b/lldb/packages/Python/lldbsuite/test/decorators.py @@ -409,10 +409,6 @@ def add_test_categories(cat): cat = test_categories.validate(cat, True) def impl(func): - if isinstance(func, type) and issubclass(func, unittest.TestCase): - raise Exception( - "@add_test_categories can only be used to decorate a test method" - ) try: if hasattr(func, "categories"): cat.extend(func.categories) diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py index 291d7bad5c08..8c29145ecc52 100644 --- a/lldb/packages/Python/lldbsuite/test/dotest.py +++ b/lldb/packages/Python/lldbsuite/test/dotest.py @@ -914,6 +914,18 @@ def checkForkVForkSupport(): configuration.skip_categories.append("fork") +def checkPexpectSupport(): + from lldbsuite.test import lldbplatformutil + + platform = lldbplatformutil.getPlatform() + + # llvm.org/pr22274: need a pexpect replacement for windows + if platform in ["windows"]: + if configuration.verbose: + print("pexpect tests will be skipped because of unsupported platform") + configuration.skip_categories.append("pexpect") + + def run_suite(): # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults # does not exist before proceeding to running the test suite. @@ -1013,6 +1025,7 @@ def run_suite(): checkDebugServerSupport() checkObjcSupport() checkForkVForkSupport() + checkPexpectSupport() skipped_categories_list = ", ".join(configuration.skip_categories) print( diff --git a/lldb/packages/Python/lldbsuite/test/lldbpexpect.py b/lldb/packages/Python/lldbsuite/test/lldbpexpect.py index 9d216d903074..998a080565b6 100644 --- a/lldb/packages/Python/lldbsuite/test/lldbpexpect.py +++ b/lldb/packages/Python/lldbsuite/test/lldbpexpect.py @@ -10,7 +10,7 @@ from lldbsuite.test.decorators import * @skipIfRemote -@skipIfWindows # llvm.org/pr22274: need a pexpect replacement for windows +@add_test_categories(["pexpect"]) class PExpectTest(TestBase): NO_DEBUG_INFO_TESTCASE = True PROMPT = "(lldb) " diff --git a/lldb/packages/Python/lldbsuite/test/test_categories.py b/lldb/packages/Python/lldbsuite/test/test_categories.py index 3f8de175e29d..036bda9c957d 100644 --- a/lldb/packages/Python/lldbsuite/test/test_categories.py +++ b/lldb/packages/Python/lldbsuite/test/test_categories.py @@ -33,6 +33,7 @@ all_categories = { "lldb-server": "Tests related to lldb-server", "lldb-dap": "Tests for the Debug Adaptor Protocol with lldb-dap", "llgs": "Tests for the gdb-server functionality of lldb-server", + "pexpect": "Tests requiring the pexpect library to be available", "objc": "Tests related to the Objective-C programming language support", "pyapi": "Tests related to the Python API", "std-module": "Tests related to importing the std module", diff --git a/lldb/packages/Python/lldbsuite/test/test_result.py b/lldb/packages/Python/lldbsuite/test/test_result.py index 20365f53a675..2d574b343b41 100644 --- a/lldb/packages/Python/lldbsuite/test/test_result.py +++ b/lldb/packages/Python/lldbsuite/test/test_result.py @@ -148,9 +148,11 @@ class LLDBTestResult(unittest.TextTestResult): Gets all the categories for the currently running test method in test case """ test_categories = [] + test_categories.extend(getattr(test, "categories", [])) + test_method = getattr(test, test._testMethodName) - if test_method is not None and hasattr(test_method, "categories"): - test_categories.extend(test_method.categories) + if test_method is not None: + test_categories.extend(getattr(test_method, "categories", [])) test_categories.extend(self._getFileBasedCategories(test)) diff --git a/lldb/test/API/benchmarks/expression/TestExpressionCmd.py b/lldb/test/API/benchmarks/expression/TestExpressionCmd.py index 9b512305d626..8261b1b25da9 100644 --- a/lldb/test/API/benchmarks/expression/TestExpressionCmd.py +++ b/lldb/test/API/benchmarks/expression/TestExpressionCmd.py @@ -17,10 +17,7 @@ class ExpressionEvaluationCase(BenchBase): self.count = 25 @benchmarks_test - @expectedFailureAll( - oslist=["windows"], - bugnumber="llvm.org/pr22274: need a pexpect replacement for windows", - ) + @add_test_categories(["pexpect"]) def test_expr_cmd(self): """Test lldb's expression commands and collect statistics.""" self.build() diff --git a/lldb/test/API/benchmarks/expression/TestRepeatedExprs.py b/lldb/test/API/benchmarks/expression/TestRepeatedExprs.py index 104e69b38423..acc6b74c17b7 100644 --- a/lldb/test/API/benchmarks/expression/TestRepeatedExprs.py +++ b/lldb/test/API/benchmarks/expression/TestRepeatedExprs.py @@ -19,10 +19,7 @@ class RepeatedExprsCase(BenchBase): self.count = 100 @benchmarks_test - @expectedFailureAll( - oslist=["windows"], - bugnumber="llvm.org/pr22274: need a pexpect replacement for windows", - ) + @add_test_categories(["pexpect"]) def test_compare_lldb_to_gdb(self): """Test repeated expressions with lldb vs. gdb.""" self.build() diff --git a/lldb/test/API/benchmarks/frame_variable/TestFrameVariableResponse.py b/lldb/test/API/benchmarks/frame_variable/TestFrameVariableResponse.py index f3989fc0ff48..e364fb8ce778 100644 --- a/lldb/test/API/benchmarks/frame_variable/TestFrameVariableResponse.py +++ b/lldb/test/API/benchmarks/frame_variable/TestFrameVariableResponse.py @@ -17,10 +17,7 @@ class FrameVariableResponseBench(BenchBase): @benchmarks_test @no_debug_info_test - @expectedFailureAll( - oslist=["windows"], - bugnumber="llvm.org/pr22274: need a pexpect replacement for windows", - ) + @add_test_categories(["pexpect"]) def test_startup_delay(self): """Test response time for the 'frame variable' command.""" print() diff --git a/lldb/test/API/benchmarks/startup/TestStartupDelays.py b/lldb/test/API/benchmarks/startup/TestStartupDelays.py index f31a10507ed7..faec21e95e5d 100644 --- a/lldb/test/API/benchmarks/startup/TestStartupDelays.py +++ b/lldb/test/API/benchmarks/startup/TestStartupDelays.py @@ -22,10 +22,7 @@ class StartupDelaysBench(BenchBase): @benchmarks_test @no_debug_info_test - @expectedFailureAll( - oslist=["windows"], - bugnumber="llvm.org/pr22274: need a pexpect replacement for windows", - ) + @add_test_categories(["pexpect"]) def test_startup_delay(self): """Test start up delays creating a target, setting a breakpoint, and run to breakpoint stop.""" print() diff --git a/lldb/test/API/benchmarks/stepping/TestSteppingSpeed.py b/lldb/test/API/benchmarks/stepping/TestSteppingSpeed.py index a3264e3f3253..d0f9b0d61d17 100644 --- a/lldb/test/API/benchmarks/stepping/TestSteppingSpeed.py +++ b/lldb/test/API/benchmarks/stepping/TestSteppingSpeed.py @@ -22,10 +22,7 @@ class SteppingSpeedBench(BenchBase): @benchmarks_test @no_debug_info_test - @expectedFailureAll( - oslist=["windows"], - bugnumber="llvm.org/pr22274: need a pexpect replacement for windows", - ) + @add_test_categories(["pexpect"]) def test_run_lldb_steppings(self): """Test lldb steppings on a large executable.""" print() diff --git a/lldb/test/API/benchmarks/turnaround/TestCompileRunToBreakpointTurnaround.py b/lldb/test/API/benchmarks/turnaround/TestCompileRunToBreakpointTurnaround.py index 98a2ec9ebf23..91527cd11453 100644 --- a/lldb/test/API/benchmarks/turnaround/TestCompileRunToBreakpointTurnaround.py +++ b/lldb/test/API/benchmarks/turnaround/TestCompileRunToBreakpointTurnaround.py @@ -21,10 +21,7 @@ class CompileRunToBreakpointBench(BenchBase): @benchmarks_test @no_debug_info_test - @expectedFailureAll( - oslist=["windows"], - bugnumber="llvm.org/pr22274: need a pexpect replacement for windows", - ) + @add_test_categories(["pexpect"]) def test_run_lldb_then_gdb(self): """Benchmark turnaround time with lldb vs. gdb.""" print() diff --git a/lldb/test/API/terminal/TestSTTYBeforeAndAfter.py b/lldb/test/API/terminal/TestSTTYBeforeAndAfter.py index e5663c50c736..21aca5fc85d5 100644 --- a/lldb/test/API/terminal/TestSTTYBeforeAndAfter.py +++ b/lldb/test/API/terminal/TestSTTYBeforeAndAfter.py @@ -19,7 +19,7 @@ class TestSTTYBeforeAndAfter(TestBase): cls.RemoveTempFile("child_send2.txt") cls.RemoveTempFile("child_read2.txt") - @skipIfWindows # llvm.org/pr22274: need a pexpect replacement for windows + @add_test_categories(["pexpect"]) @no_debug_info_test def test_stty_dash_a_before_and_afetr_invoking_lldb_command(self): """Test that 'stty -a' displays the same output before and after running the lldb command.""" -- GitLab From c0f2177dac02903bb1ae85af5d91760d0a1b6a02 Mon Sep 17 00:00:00 2001 From: erichkeane Date: Wed, 13 Mar 2024 13:30:27 -0700 Subject: [PATCH 0185/1407] Revert "[NFC] Remove unnecessary 'Builtins.def' file." This reverts commit 5a95378659506b0ce94ceb79a43477e73c9756f4. It was pointed out that this serves as documentation for the targets where the builtin files have yet to be converted, so this should be left in place. Reverting! --- clang/include/clang/Basic/Builtins.def | 102 +++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 clang/include/clang/Basic/Builtins.def diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def new file mode 100644 index 000000000000..f356f881d5ef --- /dev/null +++ b/clang/include/clang/Basic/Builtins.def @@ -0,0 +1,102 @@ +//===--- Builtins.def - Builtin function info database ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// This is only documentation for the database layout. This will be removed once +// all builtin databases are converted to tablegen files + +// The second value provided to the macro specifies the type of the function +// (result value, then each argument) as follows: +// v -> void +// b -> boolean +// c -> char +// s -> short +// i -> int +// h -> half (__fp16, OpenCL) +// x -> half (_Float16) +// y -> half (__bf16) +// f -> float +// d -> double +// z -> size_t +// w -> wchar_t +// F -> constant CFString +// G -> id +// H -> SEL +// M -> struct objc_super +// a -> __builtin_va_list +// A -> "reference" to __builtin_va_list +// V -> Vector, followed by the number of elements and the base type. +// q -> Scalable vector, followed by the number of elements and the base type. +// Q -> target builtin type, followed by a character to distinguish the builtin type +// Qa -> AArch64 svcount_t builtin type. +// E -> ext_vector, followed by the number of elements and the base type. +// X -> _Complex, followed by the base type. +// Y -> ptrdiff_t +// P -> FILE +// J -> jmp_buf +// SJ -> sigjmp_buf +// K -> ucontext_t +// p -> pid_t +// . -> "...". This may only occur at the end of the function list. +// +// Types may be prefixed with the following modifiers: +// L -> long (e.g. Li for 'long int', Ld for 'long double') +// LL -> long long (e.g. LLi for 'long long int', LLd for __float128) +// LLL -> __int128_t (e.g. LLLi) +// Z -> int32_t (require a native 32-bit integer type on the target) +// W -> int64_t (require a native 64-bit integer type on the target) +// N -> 'int' size if target is LP64, 'L' otherwise. +// O -> long for OpenCL targets, long long otherwise. +// S -> signed +// U -> unsigned +// I -> Required to constant fold to an integer constant expression. +// +// Types may be postfixed with the following modifiers: +// * -> pointer (optionally followed by an address space number, if no address +// space is specified than any address space will be accepted) +// & -> reference (optionally followed by an address space number) +// C -> const +// D -> volatile +// R -> restrict + +// The third value provided to the macro specifies information about attributes +// of the function. These must be kept in sync with the predicates in the +// Builtin::Context class. Currently we have: +// n -> nothrow +// r -> noreturn +// U -> pure +// c -> const +// t -> signature is meaningless, use custom typechecking +// T -> type is not important to semantic analysis and codegen; recognize as +// builtin even if type doesn't match signature, and don't warn if we +// can't be sure the type is right +// F -> this is a libc/libm function with a '__builtin_' prefix added. +// f -> this is a libc/libm function without a '__builtin_' prefix, or with +// 'z', a C++ standard library function in namespace std::. This builtin +// is disableable by '-fno-builtin-foo' / '-fno-builtin-std-foo'. +// h -> this function requires a specific header or an explicit declaration. +// i -> this is a runtime library implemented function without the +// '__builtin_' prefix. It will be implemented in compiler-rt or libgcc. +// p:N: -> this is a printf-like function whose Nth argument is the format +// string. +// P:N: -> similar to the p:N: attribute, but the function is like vprintf +// in that it accepts its arguments as a va_list rather than +// through an ellipsis +// s:N: -> this is a scanf-like function whose Nth argument is the format +// string. +// S:N: -> similar to the s:N: attribute, but the function is like vscanf +// in that it accepts its arguments as a va_list rather than +// through an ellipsis +// e -> const, but only when -fno-math-errno and FP exceptions are ignored +// g -> const when FP exceptions are ignored +// j -> returns_twice (like setjmp) +// u -> arguments are not evaluated for their side-effects +// V:N: -> requires vectors of at least N bits to be legal +// C -> callback behavior: argument N is called with argument +// M_0, ..., M_k as payload +// z -> this is a function in (possibly-versioned) namespace std +// E -> this function can be constant evaluated by Clang frontend -- GitLab From e2b8cc11b307aaf2717c344cbaa1d3eb5a4e0401 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Wed, 13 Mar 2024 13:34:01 -0700 Subject: [PATCH 0186/1407] [lldb] XFAIL TestIndirectSymbols on darwin (#85127) ``` AssertionError: 'main' != 'call_through_indirect_hidden' ``` --- lldb/test/API/macosx/indirect_symbol/TestIndirectSymbols.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lldb/test/API/macosx/indirect_symbol/TestIndirectSymbols.py b/lldb/test/API/macosx/indirect_symbol/TestIndirectSymbols.py index fbe6db9f892d..ad4cb4b12c79 100644 --- a/lldb/test/API/macosx/indirect_symbol/TestIndirectSymbols.py +++ b/lldb/test/API/macosx/indirect_symbol/TestIndirectSymbols.py @@ -16,6 +16,7 @@ class TestIndirectFunctions(TestBase): @skipUnlessDarwin @add_test_categories(["pyapi"]) + @expectedFailureDarwin("rdar://120796553") def test_with_python_api(self): """Test stepping and setting breakpoints in indirect and re-exported symbols.""" self.build() -- GitLab From 26bd3d0f9a5a518de02f4dc1921648cda54a0d4e Mon Sep 17 00:00:00 2001 From: Daniel Thornburgh Date: Wed, 13 Mar 2024 13:34:33 -0700 Subject: [PATCH 0187/1407] [Fuchsia] Add LLDB_TEST_USER_ARGS to stage2 passthrough --- clang/cmake/caches/Fuchsia.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/clang/cmake/caches/Fuchsia.cmake b/clang/cmake/caches/Fuchsia.cmake index 1209fd935986..df69d7d0dd41 100644 --- a/clang/cmake/caches/Fuchsia.cmake +++ b/clang/cmake/caches/Fuchsia.cmake @@ -66,6 +66,7 @@ set(_FUCHSIA_BOOTSTRAP_PASSTHROUGH LLDB_PYTHON_HOME LLDB_PYTHON_RELATIVE_PATH LLDB_TEST_USE_VENDOR_PACKAGES + LLDB_TEST_USER_ARGS Python3_EXECUTABLE Python3_LIBRARIES Python3_INCLUDE_DIRS -- GitLab From b966b224b32aada3d83fb6d58abe413b5c59f3c1 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 13 Mar 2024 13:38:07 -0700 Subject: [PATCH 0188/1407] Revert "[SLP]Fix PR85082: PHI node has multiple entries." This reverts commit 8237520eb42b37d7ed353d64a865d3ba5ac24ec6 to fix a crash in https://lab.llvm.org/buildbot/#/builders/198/builds/8891. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 16 ++-- .../X86/same-scalar-in-same-phi-extract.ll | 75 ------------------- 2 files changed, 7 insertions(+), 84 deletions(-) delete mode 100644 llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 1da509ce4794..b8b67609d755 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -12582,7 +12582,6 @@ Value *BoUpSLP::vectorizeTree( Ex = I; } } - Value *ExV = Ex; if (!Ex) { // "Reuse" the existing extract to improve final codegen. if (auto *ES = dyn_cast(Scalar)) { @@ -12593,13 +12592,7 @@ Value *BoUpSLP::vectorizeTree( } else { Ex = Builder.CreateExtractElement(Vec, Lane); } - // If necessary, sign-extend or zero-extend ScalarRoot - // to the larger type. - ExV = Ex; - if (Scalar->getType() != Ex->getType()) - ExV = Builder.CreateIntCast(Ex, Scalar->getType(), - MinBWs.find(E)->second.second); - if (auto *I = dyn_cast(ExV)) + if (auto *I = dyn_cast(Ex)) ScalarToEEs[Scalar].try_emplace(Builder.GetInsertBlock(), I); } // The then branch of the previous if may produce constants, since 0 @@ -12608,7 +12601,12 @@ Value *BoUpSLP::vectorizeTree( GatherShuffleExtractSeq.insert(ExI); CSEBlocks.insert(ExI->getParent()); } - return ExV; + // If necessary, sign-extend or zero-extend ScalarRoot + // to the larger type. + if (Scalar->getType() != Ex->getType()) + return Builder.CreateIntCast(Ex, Scalar->getType(), + MinBWs.find(E)->second.second); + return Ex; } assert(isa(Scalar->getType()) && isa(Scalar) && diff --git a/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll b/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll deleted file mode 100644 index 35f2f9e052e7..000000000000 --- a/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll +++ /dev/null @@ -1,75 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 -; RUN: opt -S --passes=slp-vectorizer -slp-threshold=-99999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s - -define void @test(i32 %arg) { -; CHECK-LABEL: define void @test( -; CHECK-SAME: i32 [[ARG:%.*]]) { -; CHECK-NEXT: bb: -; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> , i32 [[ARG]], i32 0 -; CHECK-NEXT: br label [[BB2:%.*]] -; CHECK: bb2: -; CHECK-NEXT: switch i32 0, label [[BB10:%.*]] [ -; CHECK-NEXT: i32 0, label [[BB9:%.*]] -; CHECK-NEXT: i32 11, label [[BB9]] -; CHECK-NEXT: i32 1, label [[BB4:%.*]] -; CHECK-NEXT: ] -; CHECK: bb3: -; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0 -; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 -; CHECK-NEXT: switch i32 0, label [[BB10]] [ -; CHECK-NEXT: i32 18, label [[BB7:%.*]] -; CHECK-NEXT: i32 1, label [[BB7]] -; CHECK-NEXT: i32 0, label [[BB10]] -; CHECK-NEXT: ] -; CHECK: bb4: -; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x i32> [ [[TMP0]], [[BB2]] ] -; CHECK-NEXT: [[TMP4:%.*]] = zext <2 x i32> [[TMP3]] to <2 x i64> -; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0 -; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr null, i64 [[TMP5]] -; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1 -; CHECK-NEXT: [[GETELEMENTPTR6:%.*]] = getelementptr i32, ptr null, i64 [[TMP6]] -; CHECK-NEXT: ret void -; CHECK: bb7: -; CHECK-NEXT: [[PHI8:%.*]] = phi i64 [ [[TMP2]], [[BB3:%.*]] ], [ [[TMP2]], [[BB3]] ] -; CHECK-NEXT: br label [[BB9]] -; CHECK: bb9: -; CHECK-NEXT: ret void -; CHECK: bb10: -; CHECK-NEXT: ret void -; -bb: - %zext = zext i32 %arg to i64 - %zext1 = zext i32 0 to i64 - br label %bb2 - -bb2: - switch i32 0, label %bb10 [ - i32 0, label %bb9 - i32 11, label %bb9 - i32 1, label %bb4 - ] - -bb3: - switch i32 0, label %bb10 [ - i32 18, label %bb7 - i32 1, label %bb7 - i32 0, label %bb10 - ] - -bb4: - %phi = phi i64 [ %zext, %bb2 ] - %phi5 = phi i64 [ %zext1, %bb2 ] - %getelementptr = getelementptr i32, ptr null, i64 %phi - %getelementptr6 = getelementptr i32, ptr null, i64 %phi5 - ret void - -bb7: - %phi8 = phi i64 [ %zext, %bb3 ], [ %zext, %bb3 ] - br label %bb9 - -bb9: - ret void - -bb10: - ret void -} -- GitLab From 4dd186afd502e1e56b8f3d6d923b7f8cfa124572 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 13 Mar 2024 07:45:14 -0700 Subject: [PATCH 0189/1407] [SLP]Fix PR85082: PHI node has multiple entries. Need to record casted extractelement for the externally used scalar, not original extract instruction. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 30 +++++--- .../X86/same-scalar-in-same-phi-extract.ll | 75 +++++++++++++++++++ 2 files changed, 95 insertions(+), 10 deletions(-) create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index b8b67609d755..739dae3bdd0c 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -12539,7 +12539,9 @@ Value *BoUpSLP::vectorizeTree( DenseMap VectorToInsertElement; // Maps extract Scalar to the corresponding extractelement instruction in the // basic block. Only one extractelement per block should be emitted. - DenseMap> ScalarToEEs; + DenseMap>> + ScalarToEEs; SmallDenseSet UsedInserts; DenseMap, Value *> VectorCasts; SmallDenseSet ScalarsWithNullptrUser; @@ -12568,18 +12570,23 @@ Value *BoUpSLP::vectorizeTree( auto ExtractAndExtendIfNeeded = [&](Value *Vec) { if (Scalar->getType() != Vec->getType()) { Value *Ex = nullptr; + Value *ExV = nullptr; auto It = ScalarToEEs.find(Scalar); if (It != ScalarToEEs.end()) { // No need to emit many extracts, just move the only one in the // current block. auto EEIt = It->second.find(Builder.GetInsertBlock()); if (EEIt != It->second.end()) { - Instruction *I = EEIt->second; + Instruction *I = EEIt->second.first; if (Builder.GetInsertPoint() != Builder.GetInsertBlock()->end() && - Builder.GetInsertPoint()->comesBefore(I)) + Builder.GetInsertPoint()->comesBefore(I)) { I->moveBefore(*Builder.GetInsertPoint()->getParent(), Builder.GetInsertPoint()); + if (auto *CI = EEIt->second.second) + CI->moveAfter(I); + } Ex = I; + ExV = EEIt->second.second ? EEIt->second.second : Ex; } } if (!Ex) { @@ -12592,8 +12599,16 @@ Value *BoUpSLP::vectorizeTree( } else { Ex = Builder.CreateExtractElement(Vec, Lane); } + // If necessary, sign-extend or zero-extend ScalarRoot + // to the larger type. + ExV = Ex; + if (Scalar->getType() != Ex->getType()) + ExV = Builder.CreateIntCast(Ex, Scalar->getType(), + MinBWs.find(E)->second.second); if (auto *I = dyn_cast(Ex)) - ScalarToEEs[Scalar].try_emplace(Builder.GetInsertBlock(), I); + ScalarToEEs[Scalar].try_emplace( + Builder.GetInsertBlock(), + std::make_pair(I, cast(ExV))); } // The then branch of the previous if may produce constants, since 0 // operand might be a constant. @@ -12601,12 +12616,7 @@ Value *BoUpSLP::vectorizeTree( GatherShuffleExtractSeq.insert(ExI); CSEBlocks.insert(ExI->getParent()); } - // If necessary, sign-extend or zero-extend ScalarRoot - // to the larger type. - if (Scalar->getType() != Ex->getType()) - return Builder.CreateIntCast(Ex, Scalar->getType(), - MinBWs.find(E)->second.second); - return Ex; + return ExV; } assert(isa(Scalar->getType()) && isa(Scalar) && diff --git a/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll b/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll new file mode 100644 index 000000000000..35f2f9e052e7 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/same-scalar-in-same-phi-extract.ll @@ -0,0 +1,75 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S --passes=slp-vectorizer -slp-threshold=-99999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s + +define void @test(i32 %arg) { +; CHECK-LABEL: define void @test( +; CHECK-SAME: i32 [[ARG:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> , i32 [[ARG]], i32 0 +; CHECK-NEXT: br label [[BB2:%.*]] +; CHECK: bb2: +; CHECK-NEXT: switch i32 0, label [[BB10:%.*]] [ +; CHECK-NEXT: i32 0, label [[BB9:%.*]] +; CHECK-NEXT: i32 11, label [[BB9]] +; CHECK-NEXT: i32 1, label [[BB4:%.*]] +; CHECK-NEXT: ] +; CHECK: bb3: +; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0 +; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 +; CHECK-NEXT: switch i32 0, label [[BB10]] [ +; CHECK-NEXT: i32 18, label [[BB7:%.*]] +; CHECK-NEXT: i32 1, label [[BB7]] +; CHECK-NEXT: i32 0, label [[BB10]] +; CHECK-NEXT: ] +; CHECK: bb4: +; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x i32> [ [[TMP0]], [[BB2]] ] +; CHECK-NEXT: [[TMP4:%.*]] = zext <2 x i32> [[TMP3]] to <2 x i64> +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0 +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr null, i64 [[TMP5]] +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1 +; CHECK-NEXT: [[GETELEMENTPTR6:%.*]] = getelementptr i32, ptr null, i64 [[TMP6]] +; CHECK-NEXT: ret void +; CHECK: bb7: +; CHECK-NEXT: [[PHI8:%.*]] = phi i64 [ [[TMP2]], [[BB3:%.*]] ], [ [[TMP2]], [[BB3]] ] +; CHECK-NEXT: br label [[BB9]] +; CHECK: bb9: +; CHECK-NEXT: ret void +; CHECK: bb10: +; CHECK-NEXT: ret void +; +bb: + %zext = zext i32 %arg to i64 + %zext1 = zext i32 0 to i64 + br label %bb2 + +bb2: + switch i32 0, label %bb10 [ + i32 0, label %bb9 + i32 11, label %bb9 + i32 1, label %bb4 + ] + +bb3: + switch i32 0, label %bb10 [ + i32 18, label %bb7 + i32 1, label %bb7 + i32 0, label %bb10 + ] + +bb4: + %phi = phi i64 [ %zext, %bb2 ] + %phi5 = phi i64 [ %zext1, %bb2 ] + %getelementptr = getelementptr i32, ptr null, i64 %phi + %getelementptr6 = getelementptr i32, ptr null, i64 %phi5 + ret void + +bb7: + %phi8 = phi i64 [ %zext, %bb3 ], [ %zext, %bb3 ] + br label %bb9 + +bb9: + ret void + +bb10: + ret void +} -- GitLab From 03e50c451427d908bbf8cf2d455de3ebba49fe4f Mon Sep 17 00:00:00 2001 From: Peter Klausler <35819229+klausler@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:11:45 -0700 Subject: [PATCH 0190/1407] [flang] Emit warning when Hollerith actual passed to CLASS(*) (#84084) When a Hollerith actual argument is associated with an unlimited polymorphic dummy argument, it's treated as if it were CHARACTER. Some other compilers treat it as if it had been BOZ, so emit a portability warning. Resolves https://github.com/llvm/llvm-project/issues/83548. --- flang/include/flang/Evaluate/constant.h | 3 +++ flang/lib/Semantics/check-call.cpp | 10 +++++++++- flang/lib/Semantics/expression.cpp | 7 +++++-- flang/test/Semantics/call41.f90 | 12 ++++++++++++ 4 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 flang/test/Semantics/call41.f90 diff --git a/flang/include/flang/Evaluate/constant.h b/flang/include/flang/Evaluate/constant.h index ee83d9fc04f3..71be7906d2fe 100644 --- a/flang/include/flang/Evaluate/constant.h +++ b/flang/include/flang/Evaluate/constant.h @@ -186,6 +186,8 @@ public: const Scalar &values() const { return values_; } ConstantSubscript LEN() const { return length_; } + bool wasHollerith() const { return wasHollerith_; } + void set_wasHollerith(bool yes = true) { wasHollerith_ = yes; } std::optional> GetScalarValue() const { if (Rank() == 0) { @@ -210,6 +212,7 @@ public: private: Scalar values_; // one contiguous string ConstantSubscript length_; + bool wasHollerith_{false}; }; class StructureConstructor; diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp index 3adbd7cc4177..d625f8c2f7fc 100644 --- a/flang/lib/Semantics/check-call.cpp +++ b/flang/lib/Semantics/check-call.cpp @@ -332,7 +332,15 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy, bool typesCompatible{typesCompatibleWithIgnoreTKR || dummy.type.type().IsTkCompatibleWith(actualType.type())}; int dummyRank{dummy.type.Rank()}; - if (!typesCompatible && dummyRank == 0 && allowActualArgumentConversions) { + if (typesCompatible) { + if (const auto *constantChar{ + evaluate::UnwrapConstantValue(actual)}; + constantChar && constantChar->wasHollerith() && + dummy.type.type().IsUnlimitedPolymorphic()) { + messages.Say( + "passing Hollerith to unlimited polymorphic as if it were CHARACTER"_port_en_US); + } + } else if (dummyRank == 0 && allowActualArgumentConversions) { // Extension: pass Hollerith literal to scalar as if it had been BOZ if (auto converted{evaluate::HollerithToBOZ( foldingContext, actual, dummy.type.type())}) { diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp index 54bfe0f2e156..1015a9e6efce 100644 --- a/flang/lib/Semantics/expression.cpp +++ b/flang/lib/Semantics/expression.cpp @@ -875,8 +875,11 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::CharLiteralConstant &x) { MaybeExpr ExpressionAnalyzer::Analyze( const parser::HollerithLiteralConstant &x) { int kind{GetDefaultKind(TypeCategory::Character)}; - auto value{x.v}; - return AnalyzeString(std::move(value), kind); + auto result{AnalyzeString(std::string{x.v}, kind)}; + if (auto *constant{UnwrapConstantValue(result)}) { + constant->set_wasHollerith(true); + } + return result; } // .TRUE. and .FALSE. of various kinds diff --git a/flang/test/Semantics/call41.f90 b/flang/test/Semantics/call41.f90 new file mode 100644 index 000000000000..a4c7514d99ba --- /dev/null +++ b/flang/test/Semantics/call41.f90 @@ -0,0 +1,12 @@ +! RUN: %python %S/test_errors.py %s %flang_fc1 -Werror +module m + contains + subroutine unlimited(x) + class(*), intent(in) :: x + end + subroutine test + !PORTABILITY: passing Hollerith to unlimited polymorphic as if it were CHARACTER + call unlimited(6HHERMAN) + call unlimited('abc') ! ok + end +end -- GitLab From b49d741c0c3bb21b40c925b4c1a717470181eb8d Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Wed, 13 Mar 2024 14:15:41 -0700 Subject: [PATCH 0191/1407] [lldb] Skip TestIndirectSymbols (#85133) Correction to e2b8cc11b307aaf2717c344cbaa1d3eb5a4e0401 --- lldb/test/API/macosx/indirect_symbol/TestIndirectSymbols.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/test/API/macosx/indirect_symbol/TestIndirectSymbols.py b/lldb/test/API/macosx/indirect_symbol/TestIndirectSymbols.py index ad4cb4b12c79..c4bbedc92891 100644 --- a/lldb/test/API/macosx/indirect_symbol/TestIndirectSymbols.py +++ b/lldb/test/API/macosx/indirect_symbol/TestIndirectSymbols.py @@ -16,7 +16,7 @@ class TestIndirectFunctions(TestBase): @skipUnlessDarwin @add_test_categories(["pyapi"]) - @expectedFailureDarwin("rdar://120796553") + @skipIf(bugnumber="rdar://120796553") def test_with_python_api(self): """Test stepping and setting breakpoints in indirect and re-exported symbols.""" self.build() -- GitLab From ea848d0a6d5c17af3eb1a4e39dc712606ac684f6 Mon Sep 17 00:00:00 2001 From: MessyHack Date: Wed, 13 Mar 2024 14:22:23 -0700 Subject: [PATCH 0192/1407] [OpenMP] Sort topology after adding processor group layer. (#83943) Various behavior around creating affinity masks and detecting uniform topology depends on the topology being sorted. resort topology after adding processor group layer to ensure that the updated topology reflects the newly added processor group info. Observed that the topology was not sorted correctly on high core count AMD Epyc Genoa (2 sockets, 96 cores, 2 threads) using NUMA (NPS 2+). --- openmp/runtime/src/kmp_affinity.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp index c3ee4de75a23..048bd174fc95 100644 --- a/openmp/runtime/src/kmp_affinity.cpp +++ b/openmp/runtime/src/kmp_affinity.cpp @@ -327,6 +327,9 @@ void kmp_topology_t::_insert_windows_proc_groups() { KMP_CPU_FREE(mask); _insert_layer(KMP_HW_PROC_GROUP, ids); __kmp_free(ids); + + // sort topology after adding proc groups + __kmp_topology->sort_ids(); } #endif -- GitLab From ccfb9e6eb7429885e6d09e99cf89bce41f1ca3cc Mon Sep 17 00:00:00 2001 From: Peter Klausler <35819229+klausler@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:30:04 -0700 Subject: [PATCH 0193/1407] [flang] Omit parent components for references to bindings (#84836) https://github.com/llvm/llvm-project/pull/78593 changed expression semantics to always include the names of parent components that were necessary to access an inherited component. This turns out to have broken calls to inherited NOPASS procedure bindings. Update the patch to omit explicit parent components when accessing bindings, while retaining them for component accesses (including procedure components). --- flang/lib/Semantics/expression.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp index 1015a9e6efce..6af86de9dd81 100644 --- a/flang/lib/Semantics/expression.cpp +++ b/flang/lib/Semantics/expression.cpp @@ -1302,7 +1302,8 @@ static NamedEntity IgnoreAnySubscripts(Designator &&designator) { std::move(designator.u)); } -// Components of parent derived types are explicitly represented as such. +// Components, but not bindings, of parent derived types are explicitly +// represented as such. std::optional ExpressionAnalyzer::CreateComponent(DataRef &&base, const Symbol &component, const semantics::Scope &scope, bool C919bAlreadyEnforced) { @@ -1310,7 +1311,8 @@ std::optional ExpressionAnalyzer::CreateComponent(DataRef &&base, base.Rank() > 0) { // C919b Say("An allocatable or pointer component reference must be applied to a scalar base"_err_en_US); } - if (&component.owner() == &scope) { + if (&component.owner() == &scope || + component.has()) { return Component{std::move(base), component}; } if (const Symbol *typeSymbol{scope.GetSymbol()}) { -- GitLab From af61b8e8f18df2545017ade74baee7a8a8ca99f8 Mon Sep 17 00:00:00 2001 From: Justice Adams <107649528+justice-adams-apple@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:42:04 -0700 Subject: [PATCH 0194/1407] [cmake] Add check_linker_flag import (#85128) Fixing ``` CMake Error at cmake/llvm/AddLLVM.cmake:266 (check_linker_flag): Unknown CMake command "check_linker_flag". ``` --- llvm/cmake/modules/AddLLVM.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake index eb9e6101bdce..f6fb56eb51e8 100644 --- a/llvm/cmake/modules/AddLLVM.cmake +++ b/llvm/cmake/modules/AddLLVM.cmake @@ -263,6 +263,7 @@ if (NOT DEFINED LLVM_LINKER_DETECTED AND NOT WIN32) # -no_warn_duplicate_libraries, but only in versions of the linker that # support that flag. if(NOT LLVM_USE_LINKER AND ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + include(CheckLinkerFlag) check_linker_flag(C "-Wl,-no_warn_duplicate_libraries" LLVM_LINKER_SUPPORTS_NO_WARN_DUPLICATE_LIBRARIES) else() set(LLVM_LINKER_SUPPORTS_NO_WARN_DUPLICATE_LIBRARIES OFF CACHE INTERNAL "") -- GitLab From 5661188c5766c3136d1954d769825261715b1f9a Mon Sep 17 00:00:00 2001 From: Peter Klausler <35819229+klausler@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:42:40 -0700 Subject: [PATCH 0195/1407] =?UTF-8?q?[flang]=20Support=20multiple=20distin?= =?UTF-8?q?ct=20module=20files=20with=20same=20name=20in=20one=20=E2=80=A6?= =?UTF-8?q?=20(#84838)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …compilation Allow multiple module files with the same module name to exist in one compilation; distinct modules are distinguished by their hashes. --- .../flang/Semantics/module-dependences.h | 4 +- flang/include/flang/Semantics/symbol.h | 3 + flang/lib/Semantics/mod-file.cpp | 85 +++++++++---------- flang/test/Semantics/modfile63.f90 | 7 +- 4 files changed, 46 insertions(+), 53 deletions(-) diff --git a/flang/include/flang/Semantics/module-dependences.h b/flang/include/flang/Semantics/module-dependences.h index 29813a19a4b1..3401d64c9593 100644 --- a/flang/include/flang/Semantics/module-dependences.h +++ b/flang/include/flang/Semantics/module-dependences.h @@ -23,9 +23,9 @@ public: void AddDependence( std::string &&name, bool intrinsic, ModuleCheckSumType hash) { if (intrinsic) { - intrinsicMap_.emplace(std::move(name), hash); + intrinsicMap_.insert_or_assign(std::move(name), hash); } else { - nonIntrinsicMap_.emplace(std::move(name), hash); + nonIntrinsicMap_.insert_or_assign(std::move(name), hash); } } std::optional GetRequiredHash( diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h index c3175a5d1a11..67153ffb3be9 100644 --- a/flang/include/flang/Semantics/symbol.h +++ b/flang/include/flang/Semantics/symbol.h @@ -91,12 +91,15 @@ public: return moduleFileHash_; } void set_moduleFileHash(ModuleCheckSumType x) { moduleFileHash_ = x; } + const Symbol *previous() const { return previous_; } + void set_previous(const Symbol *p) { previous_ = p; } private: bool isSubmodule_; bool isDefaultPrivate_{false}; const Scope *scope_{nullptr}; std::optional moduleFileHash_; + const Symbol *previous_{nullptr}; // same name, different module file hash }; class MainProgramDetails : public WithOmpDeclarative { diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp index b4df7216a33e..5d0d210fa348 100644 --- a/flang/lib/Semantics/mod-file.cpp +++ b/flang/lib/Semantics/mod-file.cpp @@ -1242,7 +1242,7 @@ static void GetModuleDependences( std::size_t limit{content.size()}; std::string_view str{content.data(), limit}; for (std::size_t j{ModHeader::len}; - str.substr(j, ModHeader::needLen) == ModHeader::need;) { + str.substr(j, ModHeader::needLen) == ModHeader::need; ++j) { j += 7; auto checkSum{ExtractCheckSum(str.substr(j, ModHeader::sumLen))}; if (!checkSum) { @@ -1260,8 +1260,8 @@ static void GetModuleDependences( for (; j < limit && str.at(j) != '\n'; ++j) { } if (j > start && j < limit && str.at(j) == '\n') { - dependences.AddDependence( - std::string{str.substr(start, j - start)}, intrinsic, *checkSum); + std::string depModName{str.substr(start, j - start)}; + dependences.AddDependence(std::move(depModName), intrinsic, *checkSum); } else { break; } @@ -1271,7 +1271,7 @@ static void GetModuleDependences( Scope *ModFileReader::Read(SourceName name, std::optional isIntrinsic, Scope *ancestor, bool silent) { std::string ancestorName; // empty for module - Symbol *notAModule{nullptr}; + const Symbol *notAModule{nullptr}; bool fatalError{false}; if (ancestor) { if (auto *scope{ancestor->FindSubmodule(name)}) { @@ -1287,26 +1287,28 @@ Scope *ModFileReader::Read(SourceName name, std::optional isIntrinsic, if (it != context_.globalScope().end()) { Scope *scope{it->second->scope()}; if (scope->kind() == Scope::Kind::Module) { - if (requiredHash) { - if (const Symbol * foundModule{scope->symbol()}) { - if (const auto *module{foundModule->detailsIf()}; - module && module->moduleFileHash() && - *requiredHash != *module->moduleFileHash()) { - Say(name, ancestorName, - "Multiple versions of the module '%s' cannot be required by the same compilation"_err_en_US, - name.ToString()); - return nullptr; + for (const Symbol *found{scope->symbol()}; found;) { + if (const auto *module{found->detailsIf()}) { + if (!requiredHash || + *requiredHash == + module->moduleFileHash().value_or(*requiredHash)) { + return const_cast(found->scope()); } + found = module->previous(); // same name, distinct hash + } else { + notAModule = found; + break; } } - return scope; } else { notAModule = scope->symbol(); - // USE, NON_INTRINSIC global name isn't a module? - fatalError = isIntrinsic.has_value(); } } } + if (notAModule) { + // USE, NON_INTRINSIC global name isn't a module? + fatalError = isIntrinsic.has_value(); + } auto path{ModFileName(name, ancestorName, context_.moduleFileSuffix())}; parser::Parsing parsing{context_.allCookedSources()}; parser::Options options; @@ -1360,42 +1362,18 @@ Scope *ModFileReader::Read(SourceName name, std::optional isIntrinsic, // Look for the right module file if its hash is known if (requiredHash && !fatalError) { - std::vector misses; for (const std::string &maybe : parser::LocateSourceFileAll(path, options.searchDirectories)) { if (const auto *srcFile{context_.allCookedSources().allSources().OpenPath( maybe, llvm::errs())}) { - if (auto checkSum{VerifyHeader(srcFile->content())}) { - if (*checkSum == *requiredHash) { - path = maybe; - if (!misses.empty()) { - auto &msg{context_.Say(name, - "Module file for '%s' appears later in the module search path than conflicting modules with different checksums"_warn_en_US, - name.ToString())}; - for (const std::string &m : misses) { - msg.Attach( - name, "Module file with a conflicting name: '%s'"_en_US, m); - } - } - misses.clear(); - break; - } else { - misses.emplace_back(maybe); - } + if (auto checkSum{VerifyHeader(srcFile->content())}; + checkSum && *checkSum == *requiredHash) { + path = maybe; + break; } } } - if (!misses.empty()) { - auto &msg{Say(name, ancestorName, - "Could not find a module file for '%s' in the module search path with the expected checksum"_err_en_US, - name.ToString())}; - for (const std::string &m : misses) { - msg.Attach(name, "Module file with different checksum: '%s'"_en_US, m); - } - return nullptr; - } } - const auto *sourceFile{fatalError ? nullptr : parsing.Prescan(path, options)}; if (fatalError || parsing.messages().AnyFatalError()) { if (!silent) { @@ -1451,11 +1429,24 @@ Scope *ModFileReader::Read(SourceName name, std::optional isIntrinsic, Scope &topScope{isIntrinsic.value_or(false) ? context_.intrinsicModulesScope() : context_.globalScope()}; Symbol *moduleSymbol{nullptr}; + const Symbol *previousModuleSymbol{nullptr}; if (!ancestor) { // module, not submodule parentScope = &topScope; auto pair{parentScope->try_emplace(name, UnknownDetails{})}; if (!pair.second) { - return nullptr; + // There is already a global symbol or intrinsic module of the same name. + previousModuleSymbol = &*pair.first->second; + if (const auto *details{ + previousModuleSymbol->detailsIf()}) { + if (!details->moduleFileHash().has_value()) { + return nullptr; + } + } else { + return nullptr; + } + CHECK(parentScope->erase(name) != 0); + pair = parentScope->try_emplace(name, UnknownDetails{}); + CHECK(pair.second); } moduleSymbol = &*pair.first->second; moduleSymbol->set(Symbol::Flag::ModFile); @@ -1486,7 +1477,9 @@ Scope *ModFileReader::Read(SourceName name, std::optional isIntrinsic, } if (moduleSymbol) { CHECK(moduleSymbol->test(Symbol::Flag::ModFile)); - moduleSymbol->get().set_moduleFileHash(checkSum.value()); + auto &details{moduleSymbol->get()}; + details.set_moduleFileHash(checkSum.value()); + details.set_previous(previousModuleSymbol); if (isIntrinsic.value_or(false)) { moduleSymbol->attrs().set(Attr::INTRINSIC); } diff --git a/flang/test/Semantics/modfile63.f90 b/flang/test/Semantics/modfile63.f90 index aaf1f7beaa48..078312101724 100644 --- a/flang/test/Semantics/modfile63.f90 +++ b/flang/test/Semantics/modfile63.f90 @@ -1,12 +1,10 @@ ! RUN: %flang_fc1 -fsyntax-only -I%S/Inputs/dir1 %s ! RUN: not %flang_fc1 -fsyntax-only -I%S/Inputs/dir2 %s 2>&1 | FileCheck --check-prefix=ERROR %s ! RUN: %flang_fc1 -Werror -fsyntax-only -I%S/Inputs/dir1 -I%S/Inputs/dir2 %s -! RUN: not %flang_fc1 -Werror -fsyntax-only -I%S/Inputs/dir2 -I%S/Inputs/dir1 %s 2>&1 | FileCheck --check-prefix=WARNING %s ! Inputs/dir1 and Inputs/dir2 each have identical copies of modfile63b.mod. ! modfile63b.mod depends on Inputs/dir1/modfile63a.mod - the version in -! Inputs/dir2/modfile63a.mod has a distinct checksum and should be -! ignored with a warning. +! Inputs/dir2/modfile63a.mod has a distinct checksum. ! If it becomes necessary to recompile those modules, just use the ! module files as Fortran source. @@ -15,5 +13,4 @@ use modfile63b call s2 end -! ERROR: Could not find a module file for 'modfile63a' in the module search path with the expected checksum -! WARNING: Module file for 'modfile63a' appears later in the module search path than conflicting modules with different checksums +! ERROR: Cannot read module file for module 'modfile63a': File is not the right module file for 'modfile63a': -- GitLab From af964c7e31f0728e84c97b734933fcb9a1912bce Mon Sep 17 00:00:00 2001 From: Peter Klausler <35819229+klausler@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:52:25 -0700 Subject: [PATCH 0196/1407] =?UTF-8?q?[flang][runtime]=20Let=20FORT=5FCHECK?= =?UTF-8?q?=5FPOINTER=5FDEALLOCATION=3D0=20disable=20runtime=20=E2=80=A6?= =?UTF-8?q?=20(#84956)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …check Add an environment variable by which a user can disable the pointer validation check in DEALLOCATE statement handling. This is not safe, but it can help make a code work that allocates a pointer with an extended derived type, associates its target with a pointer to one of its ancestor types, and then deallocates that pointer. --- flang/docs/RuntimeEnvironment.md | 57 ++++++++++++++++++++++++++++++++ flang/docs/index.md | 1 + flang/runtime/environment.cpp | 13 ++++++++ flang/runtime/environment.h | 1 + flang/runtime/pointer.cpp | 26 +++++++++------ 5 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 flang/docs/RuntimeEnvironment.md diff --git a/flang/docs/RuntimeEnvironment.md b/flang/docs/RuntimeEnvironment.md new file mode 100644 index 000000000000..c7a3dfbb2af1 --- /dev/null +++ b/flang/docs/RuntimeEnvironment.md @@ -0,0 +1,57 @@ + + +```{contents} +--- +local: +--- +``` + +# Environment variables of significance to Fortran execution + +A few environment variables are queried by the Fortran runtime support +library. + +The following environment variables can affect the behavior of +Fortran programs during execution. + +## `DEFAULT_UTF8=1` + +Set `DEFAULT_UTF8` to cause formatted external input to assume UTF-8 +encoding on input and use UTF-8 encoding on formatted external output. + +## `FORT_CONVERT` + +Determines data conversions applied to unformatted I/O. + +* `NATIVE`: no conversions (default) +* `LITTLE_ENDIAN`: assume input is little-endian; emit little-endian output +* `BIG_ENDIAN`: assume input is big-endian; emit big-endian output +* `SWAP`: reverse endianness (always convert) + +## `FORT_CHECK_POINTER_DEALLOCATION` + +Fortran requires that a pointer that appears in a `DEALLOCATE` statement +must have been allocated in an `ALLOCATE` statement with the same declared +type. +The runtime support library validates this requirement by checking the +size of the allocated data, and will fail with an error message if +the deallocated pointer is not valid. +Set `FORT_CHECK_POINTER_DEALLOCATION=0` to disable this check. + +## `FORT_FMT_RECL` + +Set to an integer value to specify the record length for list-directed +and `NAMELIST` output. +The default is 72. + +## `NO_STOP_MESSAGE` + +Set `NO_STOP_MESSAGE=1` to disable the extra information about +IEEE floating-point exception flags that the Fortran language +standard requires for `STOP` and `ERROR STOP` statements. diff --git a/flang/docs/index.md b/flang/docs/index.md index b4dbdc87fdf6..ed749f565ff1 100644 --- a/flang/docs/index.md +++ b/flang/docs/index.md @@ -80,6 +80,7 @@ on how to get in touch with us and to learn more about the current status. Preprocessing ProcedurePointer RuntimeDescriptor + RuntimeEnvironment RuntimeTypeInfo Semantics f2018-grammar.md diff --git a/flang/runtime/environment.cpp b/flang/runtime/environment.cpp index 62d9ee2afd1c..29196ae8f310 100644 --- a/flang/runtime/environment.cpp +++ b/flang/runtime/environment.cpp @@ -123,6 +123,19 @@ void ExecutionEnvironment::Configure(int ac, const char *av[], } } + if (auto *x{std::getenv("FORT_CHECK_POINTER_DEALLOCATION")}) { + char *end; + auto n{std::strtol(x, &end, 10)}; + if (n >= 0 && n <= 1 && *end == '\0') { + checkPointerDeallocation = n != 0; + } else { + std::fprintf(stderr, + "Fortran runtime: FORT_CHECK_POINTER_DEALLOCATION=%s is invalid; " + "ignored\n", + x); + } + } + // TODO: Set RP/ROUND='PROCESSOR_DEFINED' from environment } diff --git a/flang/runtime/environment.h b/flang/runtime/environment.h index 82a5ec8f4ebf..6da2c7bb3cf7 100644 --- a/flang/runtime/environment.h +++ b/flang/runtime/environment.h @@ -48,6 +48,7 @@ struct ExecutionEnvironment { Convert conversion{Convert::Unknown}; // FORT_CONVERT bool noStopMessage{false}; // NO_STOP_MESSAGE=1 inhibits "Fortran STOP" bool defaultUTF8{false}; // DEFAULT_UTF8 + bool checkPointerDeallocation{true}; // FORT_CHECK_POINTER_DEALLOCATION }; extern ExecutionEnvironment executionEnvironment; diff --git a/flang/runtime/pointer.cpp b/flang/runtime/pointer.cpp index fc9e0eeb7dac..08a1223764f3 100644 --- a/flang/runtime/pointer.cpp +++ b/flang/runtime/pointer.cpp @@ -9,6 +9,7 @@ #include "flang/Runtime/pointer.h" #include "assign-impl.h" #include "derived.h" +#include "environment.h" #include "stat.h" #include "terminator.h" #include "tools.h" @@ -184,17 +185,20 @@ int RTDEF(PointerDeallocate)(Descriptor &pointer, bool hasStat, if (!pointer.IsAllocated()) { return ReturnError(terminator, StatBaseNull, errMsg, hasStat); } - // Validate the footer. This should fail if the pointer doesn't - // span the entire object, or the object was not allocated as a - // pointer. - std::size_t byteSize{pointer.Elements() * pointer.ElementBytes()}; - constexpr std::size_t align{sizeof(std::uintptr_t)}; - byteSize = ((byteSize + align - 1) / align) * align; - void *p{pointer.raw().base_addr}; - std::uintptr_t *footer{ - reinterpret_cast(static_cast(p) + byteSize)}; - if (*footer != ~reinterpret_cast(p)) { - return ReturnError(terminator, StatBadPointerDeallocation, errMsg, hasStat); + if (executionEnvironment.checkPointerDeallocation) { + // Validate the footer. This should fail if the pointer doesn't + // span the entire object, or the object was not allocated as a + // pointer. + std::size_t byteSize{pointer.Elements() * pointer.ElementBytes()}; + constexpr std::size_t align{sizeof(std::uintptr_t)}; + byteSize = ((byteSize + align - 1) / align) * align; + void *p{pointer.raw().base_addr}; + std::uintptr_t *footer{ + reinterpret_cast(static_cast(p) + byteSize)}; + if (*footer != ~reinterpret_cast(p)) { + return ReturnError( + terminator, StatBadPointerDeallocation, errMsg, hasStat); + } } return ReturnError(terminator, pointer.Destroy(/*finalize=*/true, /*destroyPointers=*/true, &terminator), -- GitLab From 003e292f9895a9cf4e30688269efa668d1fcbb09 Mon Sep 17 00:00:00 2001 From: Amy Huang Date: Wed, 13 Mar 2024 21:53:38 +0000 Subject: [PATCH 0197/1407] Revert "[Clang][C++23] Implement P2448R2 ..." (#85136) Revert "[Clang][C++23] Implement P2448R2: Relaxing some constexpr restrictions (#77753)" This reverts commit 99500e8c08a4d941acb8a7eb00523296fb2acf7a because it causes a behavior change for std=c++20. See https://github.com/llvm/llvm-project/pull/77753. --- clang/docs/ReleaseNotes.rst | 2 - .../clang/Basic/DiagnosticSemaKinds.td | 26 ++- clang/lib/AST/DeclCXX.cpp | 13 +- clang/lib/Sema/SemaDeclCXX.cpp | 93 +++++----- clang/test/AST/Interp/cxx23.cpp | 59 +++++-- .../class.compare.default/p3.cpp | 40 +++-- .../class.compare.default/p4.cpp | 20 +-- .../dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp | 8 +- .../dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp | 10 +- .../CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp | 18 +- .../CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp | 8 +- .../dcl.fct.def/dcl.fct.def.default/p2.cpp | 6 +- clang/test/CXX/drs/dr13xx.cpp | 22 +-- clang/test/CXX/drs/dr14xx.cpp | 6 +- clang/test/CXX/drs/dr15xx.cpp | 21 +-- clang/test/CXX/drs/dr16xx.cpp | 20 +-- clang/test/CXX/drs/dr6xx.cpp | 24 +-- clang/test/CXX/expr/expr.const/p5-26.cpp | 4 +- clang/test/CXX/special/class.copy/p13-0x.cpp | 2 +- .../SemaCXX/constant-expression-cxx11.cpp | 38 ++--- .../SemaCXX/constant-expression-cxx14.cpp | 33 ++-- .../SemaCXX/constant-expression-cxx2b.cpp | 24 +-- .../test/SemaCXX/cxx23-invalid-constexpr.cpp | 159 ------------------ clang/test/SemaCXX/cxx2a-consteval.cpp | 2 +- .../SemaCXX/deduced-return-type-cxx14.cpp | 8 +- .../addrspace-constructors.clcpp | 2 +- clang/www/cxx_status.html | 9 +- 27 files changed, 269 insertions(+), 408 deletions(-) delete mode 100644 clang/test/SemaCXX/cxx23-invalid-constexpr.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 5fe3fd066df2..e018d3835594 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -102,8 +102,6 @@ C++23 Feature Support materialize temporary object which is a prvalue in discarded-value expression. - Implemented `P1774R8: Portable assumptions `_. -- Implemented `P2448R2: Relaxing some constexpr restrictions `_. - C++2c Feature Support ^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index d7ab1635cf12..605fbc52701d 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9617,10 +9617,13 @@ def err_defaulted_copy_assign_not_ref : Error< "the parameter for an explicitly-defaulted copy assignment operator must be an " "lvalue reference type">; def err_incorrect_defaulted_constexpr : Error< - "defaulted definition of %sub{select_special_member_kind}0 cannot be marked %select{constexpr|consteval}1 " - "before C++23">; + "defaulted definition of %sub{select_special_member_kind}0 " + "is not constexpr">; def err_incorrect_defaulted_constexpr_with_vb: Error< "%sub{select_special_member_kind}0 cannot be 'constexpr' in a class with virtual base class">; +def err_incorrect_defaulted_consteval : Error< + "defaulted declaration of %sub{select_special_member_kind}0 " + "cannot be consteval because implicit definition is not constexpr">; def warn_defaulted_method_deleted : Warning< "explicitly defaulted %sub{select_special_member_kind}0 is implicitly " "deleted">, InGroup; @@ -9731,12 +9734,21 @@ def note_defaulted_comparison_cannot_deduce_undeduced_auto : Note< "%select{|member|base class}0 %1 declared here">; def note_defaulted_comparison_cannot_deduce_callee : Note< "selected 'operator<=>' for %select{|member|base class}0 %1 declared here">; -def err_defaulted_comparison_constexpr_mismatch : Error< +def ext_defaulted_comparison_constexpr_mismatch : Extension< "defaulted definition of %select{%sub{select_defaulted_comparison_kind}1|" - "three-way comparison operator}0 cannot be " - "declared %select{constexpr|consteval}2 because " - "%select{it|for which the corresponding implicit 'operator==' }0 " - "invokes a non-constexpr comparison function ">; + "three-way comparison operator}0 that is " + "declared %select{constexpr|consteval}2 but" + "%select{|for which the corresponding implicit 'operator==' }0 " + "invokes a non-constexpr comparison function is a C++23 extension">, + InGroup>; +def warn_cxx23_compat_defaulted_comparison_constexpr_mismatch : Warning< + "defaulted definition of %select{%sub{select_defaulted_comparison_kind}1|" + "three-way comparison operator}0 that is " + "declared %select{constexpr|consteval}2 but" + "%select{|for which the corresponding implicit 'operator==' }0 " + "invokes a non-constexpr comparison function is incompatible with C++ " + "standards before C++23">, + InGroup, DefaultIgnore; def note_defaulted_comparison_not_constexpr : Note< "non-constexpr comparison function would be used to compare " "%select{|member %1|base class %1}0">; diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp index 1c3dcf63465c..b4f2327d9c56 100644 --- a/clang/lib/AST/DeclCXX.cpp +++ b/clang/lib/AST/DeclCXX.cpp @@ -400,11 +400,10 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases, // C++11 [class.ctor]p6: // If that user-written default constructor would satisfy the - // requirements of a constexpr constructor/function(C++23), the - // implicitly-defined default constructor is constexpr. + // requirements of a constexpr constructor, the implicitly-defined + // default constructor is constexpr. if (!BaseClassDecl->hasConstexprDefaultConstructor()) - data().DefaultedDefaultConstructorIsConstexpr = - C.getLangOpts().CPlusPlus23; + data().DefaultedDefaultConstructorIsConstexpr = false; // C++1z [class.copy]p8: // The implicitly-declared copy constructor for a class X will have @@ -549,8 +548,7 @@ void CXXRecordDecl::addedClassSubobject(CXXRecordDecl *Subobj) { // -- for every subobject of class type or (possibly multi-dimensional) // array thereof, that class type shall have a constexpr destructor if (!Subobj->hasConstexprDestructor()) - data().DefaultedDestructorIsConstexpr = - getASTContext().getLangOpts().CPlusPlus23; + data().DefaultedDestructorIsConstexpr = false; // C++20 [temp.param]p7: // A structural type is [...] a literal class type [for which] the types @@ -1299,8 +1297,7 @@ void CXXRecordDecl::addedMember(Decl *D) { !FieldRec->hasConstexprDefaultConstructor() && !isUnion()) // The standard requires any in-class initializer to be a constant // expression. We consider this to be a defect. - data().DefaultedDefaultConstructorIsConstexpr = - Context.getLangOpts().CPlusPlus23; + data().DefaultedDefaultConstructorIsConstexpr = false; // C++11 [class.copy]p8: // The implicitly-declared copy constructor for a class X will have diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp index e258a4f7c894..199f2523cfb5 100644 --- a/clang/lib/Sema/SemaDeclCXX.cpp +++ b/clang/lib/Sema/SemaDeclCXX.cpp @@ -1715,8 +1715,6 @@ static bool CheckLiteralType(Sema &SemaRef, Sema::CheckConstexprKind Kind, static bool CheckConstexprDestructorSubobjects(Sema &SemaRef, const CXXDestructorDecl *DD, Sema::CheckConstexprKind Kind) { - assert(!SemaRef.getLangOpts().CPlusPlus23 && - "this check is obsolete for C++23"); auto Check = [&](SourceLocation Loc, QualType T, const FieldDecl *FD) { const CXXRecordDecl *RD = T->getBaseElementTypeUnsafe()->getAsCXXRecordDecl(); @@ -1748,8 +1746,6 @@ static bool CheckConstexprDestructorSubobjects(Sema &SemaRef, static bool CheckConstexprParameterTypes(Sema &SemaRef, const FunctionDecl *FD, Sema::CheckConstexprKind Kind) { - assert(!SemaRef.getLangOpts().CPlusPlus23 && - "this check is obsolete for C++23"); unsigned ArgIndex = 0; const auto *FT = FD->getType()->castAs(); for (FunctionProtoType::param_type_iterator i = FT->param_type_begin(), @@ -1771,8 +1767,6 @@ static bool CheckConstexprParameterTypes(Sema &SemaRef, /// true. If not, produce a suitable diagnostic and return false. static bool CheckConstexprReturnType(Sema &SemaRef, const FunctionDecl *FD, Sema::CheckConstexprKind Kind) { - assert(!SemaRef.getLangOpts().CPlusPlus23 && - "this check is obsolete for C++23"); if (CheckLiteralType(SemaRef, Kind, FD->getLocation(), FD->getReturnType(), diag::err_constexpr_non_literal_return, FD->isConsteval())) @@ -1862,18 +1856,16 @@ bool Sema::CheckConstexprFunctionDefinition(const FunctionDecl *NewFD, } } - // - its return type shall be a literal type; (removed in C++23) - if (!getLangOpts().CPlusPlus23 && - !CheckConstexprReturnType(*this, NewFD, Kind)) + // - its return type shall be a literal type; + if (!CheckConstexprReturnType(*this, NewFD, Kind)) return false; } if (auto *Dtor = dyn_cast(NewFD)) { // A destructor can be constexpr only if the defaulted destructor could be; // we don't need to check the members and bases if we already know they all - // have constexpr destructors. (removed in C++23) - if (!getLangOpts().CPlusPlus23 && - !Dtor->getParent()->defaultedDestructorIsConstexpr()) { + // have constexpr destructors. + if (!Dtor->getParent()->defaultedDestructorIsConstexpr()) { if (Kind == CheckConstexprKind::CheckValid) return false; if (!CheckConstexprDestructorSubobjects(*this, Dtor, Kind)) @@ -1881,9 +1873,8 @@ bool Sema::CheckConstexprFunctionDefinition(const FunctionDecl *NewFD, } } - // - each of its parameter types shall be a literal type; (removed in C++23) - if (!getLangOpts().CPlusPlus23 && - !CheckConstexprParameterTypes(*this, NewFD, Kind)) + // - each of its parameter types shall be a literal type; + if (!CheckConstexprParameterTypes(*this, NewFD, Kind)) return false; Stmt *Body = NewFD->getBody(); @@ -2466,8 +2457,7 @@ static bool CheckConstexprFunctionBody(Sema &SemaRef, const FunctionDecl *Dcl, // function", so is not checked in CheckValid mode. SmallVector Diags; if (Kind == Sema::CheckConstexprKind::Diagnose && - !Expr::isPotentialConstantExpr(Dcl, Diags) && - !SemaRef.getLangOpts().CPlusPlus23) { + !Expr::isPotentialConstantExpr(Dcl, Diags)) { SemaRef.Diag(Dcl->getLocation(), diag::ext_constexpr_function_never_constant_expr) << isa(Dcl) << Dcl->isConsteval() @@ -7545,23 +7535,21 @@ static bool defaultedSpecialMemberIsConstexpr( // C++1y [class.copy]p26: // -- [the class] is a literal type, and - if (!Ctor && !ClassDecl->isLiteral() && !S.getLangOpts().CPlusPlus23) + if (!Ctor && !ClassDecl->isLiteral()) return false; // -- every constructor involved in initializing [...] base class // sub-objects shall be a constexpr constructor; // -- the assignment operator selected to copy/move each direct base // class is a constexpr function, and - if (!S.getLangOpts().CPlusPlus23) { - for (const auto &B : ClassDecl->bases()) { - const RecordType *BaseType = B.getType()->getAs(); - if (!BaseType) - continue; - CXXRecordDecl *BaseClassDecl = cast(BaseType->getDecl()); - if (!specialMemberIsConstexpr(S, BaseClassDecl, CSM, 0, ConstArg, - InheritedCtor, Inherited)) - return false; - } + for (const auto &B : ClassDecl->bases()) { + const RecordType *BaseType = B.getType()->getAs(); + if (!BaseType) + continue; + CXXRecordDecl *BaseClassDecl = cast(BaseType->getDecl()); + if (!specialMemberIsConstexpr(S, BaseClassDecl, CSM, 0, ConstArg, + InheritedCtor, Inherited)) + return false; } // -- every constructor involved in initializing non-static data members @@ -7571,22 +7559,20 @@ static bool defaultedSpecialMemberIsConstexpr( // -- for each non-static data member of X that is of class type (or array // thereof), the assignment operator selected to copy/move that member is // a constexpr function - if (!S.getLangOpts().CPlusPlus23) { - for (const auto *F : ClassDecl->fields()) { - if (F->isInvalidDecl()) - continue; - if (CSM == Sema::CXXDefaultConstructor && F->hasInClassInitializer()) - continue; - QualType BaseType = S.Context.getBaseElementType(F->getType()); - if (const RecordType *RecordTy = BaseType->getAs()) { - CXXRecordDecl *FieldRecDecl = cast(RecordTy->getDecl()); - if (!specialMemberIsConstexpr(S, FieldRecDecl, CSM, - BaseType.getCVRQualifiers(), - ConstArg && !F->isMutable())) - return false; - } else if (CSM == Sema::CXXDefaultConstructor) { + for (const auto *F : ClassDecl->fields()) { + if (F->isInvalidDecl()) + continue; + if (CSM == Sema::CXXDefaultConstructor && F->hasInClassInitializer()) + continue; + QualType BaseType = S.Context.getBaseElementType(F->getType()); + if (const RecordType *RecordTy = BaseType->getAs()) { + CXXRecordDecl *FieldRecDecl = cast(RecordTy->getDecl()); + if (!specialMemberIsConstexpr(S, FieldRecDecl, CSM, + BaseType.getCVRQualifiers(), + ConstArg && !F->isMutable())) return false; - } + } else if (CSM == Sema::CXXDefaultConstructor) { + return false; } } @@ -7872,17 +7858,18 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD, MD->isConstexpr() && !Constexpr && MD->getTemplatedKind() == FunctionDecl::TK_NonTemplate) { if (!MD->isConsteval() && RD->getNumVBases()) { - Diag(MD->getBeginLoc(), - diag::err_incorrect_defaulted_constexpr_with_vb) + Diag(MD->getBeginLoc(), diag::err_incorrect_defaulted_constexpr_with_vb) << CSM; for (const auto &I : RD->vbases()) Diag(I.getBeginLoc(), diag::note_constexpr_virtual_base_here); } else { - Diag(MD->getBeginLoc(), diag::err_incorrect_defaulted_constexpr) - << CSM << MD->isConsteval(); + Diag(MD->getBeginLoc(), MD->isConsteval() + ? diag::err_incorrect_defaulted_consteval + : diag::err_incorrect_defaulted_constexpr) + << CSM; } - HadError = true; - // FIXME: Explain why the special member can't be constexpr. + // FIXME: Explain why the special member can't be constexpr. + HadError = true; } if (First) { @@ -9114,11 +9101,13 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD, // - if the function is a constructor or destructor, its class does not // have any virtual base classes. if (FD->isConstexpr()) { - if (!getLangOpts().CPlusPlus23 && - CheckConstexprReturnType(*this, FD, CheckConstexprKind::Diagnose) && + if (CheckConstexprReturnType(*this, FD, CheckConstexprKind::Diagnose) && CheckConstexprParameterTypes(*this, FD, CheckConstexprKind::Diagnose) && !Info.Constexpr) { - Diag(FD->getBeginLoc(), diag::err_defaulted_comparison_constexpr_mismatch) + Diag(FD->getBeginLoc(), + getLangOpts().CPlusPlus23 + ? diag::warn_cxx23_compat_defaulted_comparison_constexpr_mismatch + : diag::ext_defaulted_comparison_constexpr_mismatch) << FD->isImplicit() << (int)DCK << FD->isConsteval(); DefaultedComparisonAnalyzer(*this, RD, FD, DCK, DefaultedComparisonAnalyzer::ExplainConstexpr) diff --git a/clang/test/AST/Interp/cxx23.cpp b/clang/test/AST/Interp/cxx23.cpp index 127b58915127..f1df936a5abe 100644 --- a/clang/test/AST/Interp/cxx23.cpp +++ b/clang/test/AST/Interp/cxx23.cpp @@ -1,58 +1,82 @@ -// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify=ref20,all,all-20 %s +// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify=ref20,all %s // RUN: %clang_cc1 -std=c++23 -fsyntax-only -fcxx-exceptions -verify=ref23,all %s -// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify=expected20,all,all-20 %s -fexperimental-new-constant-interpreter +// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify=expected20,all %s -fexperimental-new-constant-interpreter // RUN: %clang_cc1 -std=c++23 -fsyntax-only -fcxx-exceptions -verify=expected23,all %s -fexperimental-new-constant-interpreter /// FIXME: The new interpreter is missing all the 'control flows through...' diagnostics. constexpr int f(int n) { // ref20-error {{constexpr function never produces a constant expression}} \ - // expected20-error {{constexpr function never produces a constant expression}} + // ref23-error {{constexpr function never produces a constant expression}} \ + // expected20-error {{constexpr function never produces a constant expression}} \ + // expected23-error {{constexpr function never produces a constant expression}} static const int m = n; // ref20-note {{control flows through the definition of a static variable}} \ // ref20-warning {{is a C++23 extension}} \ + // ref23-note {{control flows through the definition of a static variable}} \ // expected20-warning {{is a C++23 extension}} \ // expected20-note {{declared here}} \ + // expected23-note {{declared here}} - return m; // expected20-note {{initializer of 'm' is not a constant expression}} + return m; // expected20-note {{initializer of 'm' is not a constant expression}} \ + // expected23-note {{initializer of 'm' is not a constant expression}} } constexpr int g(int n) { // ref20-error {{constexpr function never produces a constant expression}} \ - // expected20-error {{constexpr function never produces a constant expression}} + // ref23-error {{constexpr function never produces a constant expression}} \ + // expected20-error {{constexpr function never produces a constant expression}} \ + // expected23-error {{constexpr function never produces a constant expression}} thread_local const int m = n; // ref20-note {{control flows through the definition of a thread_local variable}} \ // ref20-warning {{is a C++23 extension}} \ + // ref23-note {{control flows through the definition of a thread_local variable}} \ // expected20-warning {{is a C++23 extension}} \ - // expected20-note {{declared here}} - return m; // expected20-note {{initializer of 'm' is not a constant expression}} + // expected20-note {{declared here}} \ + // expected23-note {{declared here}} + return m; // expected20-note {{initializer of 'm' is not a constant expression}} \ + // expected23-note {{initializer of 'm' is not a constant expression}} } constexpr int c_thread_local(int n) { // ref20-error {{constexpr function never produces a constant expression}} \ - // expected20-error {{constexpr function never produces a constant expression}} + // ref23-error {{constexpr function never produces a constant expression}} \ + // expected20-error {{constexpr function never produces a constant expression}} \ + // expected23-error {{constexpr function never produces a constant expression}} static _Thread_local int m = 0; // ref20-note {{control flows through the definition of a thread_local variable}} \ // ref20-warning {{is a C++23 extension}} \ + // ref23-note {{control flows through the definition of a thread_local variable}} \ // expected20-warning {{is a C++23 extension}} \ - // expected20-note {{declared here}} - return m; // expected20-note {{read of non-const variable}} + // expected20-note {{declared here}} \ + // expected23-note {{declared here}} + return m; // expected20-note {{read of non-const variable}} \ + // expected23-note {{read of non-const variable}} } constexpr int gnu_thread_local(int n) { // ref20-error {{constexpr function never produces a constant expression}} \ - // expected20-error {{constexpr function never produces a constant expression}} + // ref23-error {{constexpr function never produces a constant expression}} \ + // expected20-error {{constexpr function never produces a constant expression}} \ + // expected23-error {{constexpr function never produces a constant expression}} static __thread int m = 0; // ref20-note {{control flows through the definition of a thread_local variable}} \ // ref20-warning {{is a C++23 extension}} \ + // ref23-note {{control flows through the definition of a thread_local variable}} \ // expected20-warning {{is a C++23 extension}} \ - // expected20-note {{declared here}} - return m; // expected20-note {{read of non-const variable}} + // expected20-note {{declared here}} \ + // expected23-note {{declared here}} + return m; // expected20-note {{read of non-const variable}} \ + // expected23-note {{read of non-const variable}} } -constexpr int h(int n) { // ref20-error {{constexpr function never produces a constant expression}} +constexpr int h(int n) { // ref20-error {{constexpr function never produces a constant expression}} \ + // ref23-error {{constexpr function never produces a constant expression}} static const int m = n; // ref20-note {{control flows through the definition of a static variable}} \ // ref20-warning {{is a C++23 extension}} \ + // ref23-note {{control flows through the definition of a static variable}} \ // expected20-warning {{is a C++23 extension}} return &m - &m; } -constexpr int i(int n) { // ref20-error {{constexpr function never produces a constant expression}} +constexpr int i(int n) { // ref20-error {{constexpr function never produces a constant expression}} \ + // ref23-error {{constexpr function never produces a constant expression}} thread_local const int m = n; // ref20-note {{control flows through the definition of a thread_local variable}} \ // ref20-warning {{is a C++23 extension}} \ + // ref23-note {{control flows through the definition of a thread_local variable}} \ // expected20-warning {{is a C++23 extension}} return &m - &m; } @@ -108,9 +132,8 @@ namespace StaticOperators { static_assert(f2() == 3); struct S1 { - constexpr S1() { // all-20-error {{never produces a constant expression}} - throw; // all-note {{not valid in a constant expression}} \ - // all-20-note {{not valid in a constant expression}} + constexpr S1() { // all-error {{never produces a constant expression}} + throw; // all-note 2{{not valid in a constant expression}} } static constexpr int operator()() { return 3; } // ref20-warning {{C++23 extension}} \ // expected20-warning {{C++23 extension}} diff --git a/clang/test/CXX/class/class.compare/class.compare.default/p3.cpp b/clang/test/CXX/class/class.compare/class.compare.default/p3.cpp index c73eb0dee995..166bd97e2731 100644 --- a/clang/test/CXX/class/class.compare/class.compare.default/p3.cpp +++ b/clang/test/CXX/class/class.compare/class.compare.default/p3.cpp @@ -1,8 +1,8 @@ // This test is for the [class.compare.default]p3 added by P2002R0 -// Also covers modifications made by P2448R2 +// Also covers modifications made by P2448R2 and extension warnings -// RUN: %clang_cc1 -std=c++2a -verify=expected,cxx2a %s -// RUN: %clang_cc1 -std=c++23 -verify=expected %s +// RUN: %clang_cc1 -std=c++2a -verify %s +// RUN: %clang_cc1 -std=c++2a -Wc++23-default-comp-relaxed-constexpr -verify=expected,extension %s namespace std { struct strong_ordering { @@ -82,12 +82,10 @@ struct TestB { }; struct C { - friend bool operator==(const C&, const C&); // expected-note {{previous}} \ - // cxx2a-note 2{{declared here}} + friend bool operator==(const C&, const C&); // expected-note {{previous}} extension-note 2{{non-constexpr comparison function declared here}} friend bool operator!=(const C&, const C&) = default; // expected-note {{previous}} - friend std::strong_ordering operator<=>(const C&, const C&); // expected-note {{previous}} \ - // cxx2a-note 2{{declared here}} + friend std::strong_ordering operator<=>(const C&, const C&); // expected-note {{previous}} extension-note 2{{non-constexpr comparison function declared here}} friend bool operator<(const C&, const C&) = default; // expected-note {{previous}} friend bool operator<=(const C&, const C&) = default; // expected-note {{previous}} friend bool operator>(const C&, const C&) = default; // expected-note {{previous}} @@ -131,23 +129,23 @@ struct TestD { struct E { A a; - C c; // cxx2a-note 2{{non-constexpr comparison function would be used to compare member 'c'}} + C c; // extension-note 2{{non-constexpr comparison function would be used to compare member 'c'}} A b; - friend constexpr bool operator==(const E&, const E&) = default; // cxx2a-error {{cannot be declared constexpr}} + friend constexpr bool operator==(const E&, const E&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} friend constexpr bool operator!=(const E&, const E&) = default; - friend constexpr std::strong_ordering operator<=>(const E&, const E&) = default; // cxx2a-error {{cannot be declared constexpr}} + friend constexpr std::strong_ordering operator<=>(const E&, const E&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} friend constexpr bool operator<(const E&, const E&) = default; friend constexpr bool operator<=(const E&, const E&) = default; friend constexpr bool operator>(const E&, const E&) = default; friend constexpr bool operator>=(const E&, const E&) = default; }; -struct E2 : A, C { // cxx2a-note 2{{non-constexpr comparison function would be used to compare base class 'C'}} - friend constexpr bool operator==(const E2&, const E2&) = default; // cxx2a-error {{cannot be declared constexpr}} +struct E2 : A, C { // extension-note 2{{non-constexpr comparison function would be used to compare base class 'C'}} + friend constexpr bool operator==(const E2&, const E2&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} friend constexpr bool operator!=(const E2&, const E2&) = default; - friend constexpr std::strong_ordering operator<=>(const E2&, const E2&) = default; // cxx2a-error {{cannot be declared constexpr}} + friend constexpr std::strong_ordering operator<=>(const E2&, const E2&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} friend constexpr bool operator<(const E2&, const E2&) = default; friend constexpr bool operator<=(const E2&, const E2&) = default; friend constexpr bool operator>(const E2&, const E2&) = default; @@ -155,14 +153,14 @@ struct E2 : A, C { // cxx2a-note 2{{non-constexpr comparison function would be u }; struct F { - friend bool operator==(const F&, const F&); // cxx2a-note {{declared here}} - friend constexpr bool operator!=(const F&, const F&) = default; // cxx2a-error {{cannot be declared constexpr}} - - friend std::strong_ordering operator<=>(const F&, const F&); // cxx2a-note 4{{non-constexpr comparison function declared here}} - friend constexpr bool operator<(const F&, const F&) = default; // cxx2a-error {{cannot be declared constexpr}} - friend constexpr bool operator<=(const F&, const F&) = default; // cxx2a-error {{cannot be declared constexpr}} - friend constexpr bool operator>(const F&, const F&) = default; // cxx2a-error {{cannot be declared constexpr}} - friend constexpr bool operator>=(const F&, const F&) = default; // cxx2a-error {{cannot be declared constexpr}} + friend bool operator==(const F&, const F&); // extension-note {{non-constexpr comparison function declared here}} + friend constexpr bool operator!=(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} + + friend std::strong_ordering operator<=>(const F&, const F&); // extension-note 4{{non-constexpr comparison function declared here}} + friend constexpr bool operator<(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} + friend constexpr bool operator<=(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} + friend constexpr bool operator>(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} + friend constexpr bool operator>=(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} }; // No implicit 'constexpr' if it's not the first declaration. diff --git a/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp b/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp index 534c3b34d883..02cdd7f85aeb 100644 --- a/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp +++ b/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp @@ -1,9 +1,9 @@ -// RUN: %clang_cc1 -std=c++2a -verify=expected,cxx2a %s -// RUN: %clang_cc1 -std=c++23 -verify=expected %s +// RUN: %clang_cc1 -std=c++2a -verify %s +// RUN: %clang_cc1 -std=c++2a -Wc++23-default-comp-relaxed-constexpr -verify=expected,extension %s // This test is for [class.compare.default]p3 as modified and renumbered to p4 // by P2002R0. -// Also covers modifications made by P2448R2 +// Also covers modifications made by P2448R2 and extension warnings namespace std { struct strong_ordering { @@ -78,13 +78,13 @@ void use_g(G g) { } struct H { - bool operator==(const H&) const; // cxx2a-note {{non-constexpr comparison function declared here}} + bool operator==(const H&) const; // extension-note {{non-constexpr comparison function declared here}} constexpr std::strong_ordering operator<=>(const H&) const { return std::strong_ordering::equal; } }; struct I { - H h; // cxx2a-note {{non-constexpr comparison function would be used to compare member 'h'}} - constexpr std::strong_ordering operator<=>(const I&) const = default; // cxx2a-error {{cannot be declared constexpr}} + H h; // extension-note {{non-constexpr comparison function would be used to compare member 'h'}} + constexpr std::strong_ordering operator<=>(const I&) const = default; // extension-warning {{implicit 'operator==' invokes a non-constexpr comparison function is a C++23 extension}} }; struct J { @@ -148,16 +148,16 @@ namespace NoInjectionIfOperatorEqualsDeclared { namespace GH61238 { template struct my_struct { - A value; // cxx2a-note {{non-constexpr comparison function would be used to compare member 'value'}} + A value; // extension-note {{non-constexpr comparison function would be used to compare member 'value'}} - constexpr friend bool operator==(const my_struct &, const my_struct &) noexcept = default; // cxx2a-error {{cannot be declared constexpr}} + constexpr friend bool operator==(const my_struct &, const my_struct &) noexcept = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} }; struct non_constexpr_type { - friend bool operator==(non_constexpr_type, non_constexpr_type) noexcept { // cxx2a-note {{non-constexpr comparison function declared here}} + friend bool operator==(non_constexpr_type, non_constexpr_type) noexcept { // extension-note {{non-constexpr comparison function declared here}} return false; } }; -my_struct obj; // cxx2a-note {{in instantiation of template class 'GH61238::my_struct' requested here}} +my_struct obj; // extension-note {{in instantiation of template class 'GH61238::my_struct' requested here}} } diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp index 48bc8fb426bc..7ad2e582a812 100644 --- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp +++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp @@ -58,12 +58,12 @@ namespace subobject { struct A { ~A(); }; - struct B : A { // cxx2a-note {{here}} - constexpr ~B() {} // cxx2a-error {{destructor cannot be declared constexpr because base class 'A' does not have a constexpr destructor}} + struct B : A { // expected-note {{here}} + constexpr ~B() {} // expected-error {{destructor cannot be declared constexpr because base class 'A' does not have a constexpr destructor}} }; struct C { - A a; // cxx2a-note {{here}} - constexpr ~C() {} // cxx2a-error {{destructor cannot be declared constexpr because data member 'a' does not have a constexpr destructor}} + A a; // expected-note {{here}} + constexpr ~C() {} // expected-error {{destructor cannot be declared constexpr because data member 'a' does not have a constexpr destructor}} }; struct D : A { A a; diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp index 8cb37ae6d1cd..c07502c0555b 100644 --- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp +++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp @@ -14,8 +14,9 @@ constexpr int i(int n) { return m; } -constexpr int g() { - goto test; // expected-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++23}} +constexpr int g() { // expected-error {{constexpr function never produces a constant expression}} + goto test; // expected-note {{subexpression not valid in a constant expression}} \ + // expected-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++23}} test: return 0; } @@ -28,8 +29,9 @@ struct NonLiteral { // expected-note 2 {{'NonLiteral' is not literal}} NonLiteral() {} }; -constexpr void non_literal() { - NonLiteral n; // expected-warning {{definition of a variable of non-literal type in a constexpr function is incompatible with C++ standards before C++23}} +constexpr void non_literal() { // expected-error {{constexpr function never produces a constant expression}} + NonLiteral n; // expected-note {{non-literal type 'NonLiteral' cannot be used in a constant expression}} \ + // expected-warning {{definition of a variable of non-literal type in a constexpr function is incompatible with C++ standards before C++23}} } constexpr void non_literal2(bool b) { diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp index 4416c8252264..6214ff8006d6 100644 --- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp +++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp @@ -1,6 +1,6 @@ // RUN: %clang_cc1 -fcxx-exceptions -verify=expected,beforecxx14,beforecxx20,beforecxx23 -std=c++11 %s -// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,beforecxx20,beforecxx23,cxx14_20 -std=c++14 %s -// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,aftercxx20,beforecxx23,cxx14_20 -std=c++20 %s +// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,beforecxx20,beforecxx23 -std=c++14 %s +// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,aftercxx20,beforecxx23 -std=c++20 %s // RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,aftercxx20 -std=c++23 %s namespace N { @@ -11,7 +11,7 @@ namespace M { typedef double D; } -struct NonLiteral { // beforecxx23-note 2{{no constexpr constructors}} +struct NonLiteral { // expected-note 2{{no constexpr constructors}} NonLiteral() {} NonLiteral(int) {} }; @@ -43,7 +43,7 @@ struct T : SS, NonLiteral { // - its return type shall be a literal type; // Once we support P2448R2 constexpr functions will be allowd to return non-literal types // The destructor will also be allowed - constexpr NonLiteral NonLiteralReturn() const { return {}; } // beforecxx23-error {{constexpr function's return type 'NonLiteral' is not a literal type}} + constexpr NonLiteral NonLiteralReturn() const { return {}; } // expected-error {{constexpr function's return type 'NonLiteral' is not a literal type}} constexpr void VoidReturn() const { return; } // beforecxx14-error {{constexpr function's return type 'void' is not a literal type}} constexpr ~T(); // beforecxx20-error {{destructor cannot be declared constexpr}} @@ -52,7 +52,7 @@ struct T : SS, NonLiteral { // - each of its parameter types shall be a literal type; // Once we support P2448R2 constexpr functions will be allowd to have parameters of non-literal types - constexpr int NonLiteralParam(NonLiteral) const { return 0; } // beforecxx23-error {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}} + constexpr int NonLiteralParam(NonLiteral) const { return 0; } // expected-error {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}} typedef int G(NonLiteral) const; constexpr G NonLiteralParam2; // ok until definition @@ -66,7 +66,7 @@ struct T : SS, NonLiteral { // constexpr since they can't be const. constexpr T &operator=(const T &) = default; // beforecxx14-error {{an explicitly-defaulted copy assignment operator may not have 'const', 'constexpr' or 'volatile' qualifiers}} \ // beforecxx14-warning {{C++14}} \ - // cxx14_20-error{{defaulted definition of copy assignment operator cannot be marked constexpr}} + // aftercxx14-error{{defaulted definition of copy assignment operator is not constexpr}} }; constexpr int T::OutOfLineVirtual() const { return 0; } @@ -229,9 +229,9 @@ namespace DR1364 { return k; // ok, even though lvalue-to-rvalue conversion of a function // parameter is not allowed in a constant expression. } - int kGlobal; // beforecxx23-note {{here}} - constexpr int f() { // beforecxx23-error {{constexpr function never produces a constant expression}} - return kGlobal; // beforecxx23-note {{read of non-const}} + int kGlobal; // expected-note {{here}} + constexpr int f() { // expected-error {{constexpr function never produces a constant expression}} + return kGlobal; // expected-note {{read of non-const}} } } diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp index 92698ec1c738..f1f677ebfcd3 100644 --- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp +++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp @@ -272,7 +272,7 @@ struct X { union XU1 { int a; constexpr XU1() = default; }; #ifndef CXX2A -// expected-error@-2{{cannot be marked constexpr}} +// expected-error@-2{{not constexpr}} #endif union XU2 { int a = 1; constexpr XU2() = default; }; @@ -282,7 +282,7 @@ struct XU3 { }; constexpr XU3() = default; #ifndef CXX2A - // expected-error@-2{{cannot be marked constexpr}} + // expected-error@-2{{not constexpr}} #endif }; struct XU4 { @@ -333,7 +333,7 @@ namespace CtorLookup { constexpr B(B&); }; constexpr B::B(const B&) = default; - constexpr B::B(B&) = default; // expected-error {{cannot be marked constexpr}} + constexpr B::B(B&) = default; // expected-error {{not constexpr}} struct C { A a; @@ -342,7 +342,7 @@ namespace CtorLookup { constexpr C(C&); }; constexpr C::C(const C&) = default; - constexpr C::C(C&) = default; // expected-error {{cannot be marked constexpr}} + constexpr C::C(C&) = default; // expected-error {{not constexpr}} } namespace PR14503 { diff --git a/clang/test/CXX/dcl.decl/dcl.fct.def/dcl.fct.def.default/p2.cpp b/clang/test/CXX/dcl.decl/dcl.fct.def/dcl.fct.def.default/p2.cpp index 849594307390..5b525fc91aba 100644 --- a/clang/test/CXX/dcl.decl/dcl.fct.def/dcl.fct.def.default/p2.cpp +++ b/clang/test/CXX/dcl.decl/dcl.fct.def/dcl.fct.def.default/p2.cpp @@ -3,7 +3,7 @@ // An explicitly-defaulted function may be declared constexpr only if it would // have been implicitly declared as constexpr. struct S1 { - constexpr S1() = default; // expected-error {{defaulted definition of default constructor cannot be marked constexpr}} + constexpr S1() = default; // expected-error {{defaulted definition of default constructor is not constexpr}} constexpr S1(const S1&) = default; constexpr S1(S1&&) = default; constexpr S1 &operator=(const S1&) const = default; // expected-error {{explicitly-defaulted copy assignment operator may not have}} @@ -18,8 +18,8 @@ struct NoCopyMove { }; struct S2 { constexpr S2() = default; - constexpr S2(const S2&) = default; // expected-error {{defaulted definition of copy constructor cannot be marked constexpr}} - constexpr S2(S2&&) = default; // expected-error {{defaulted definition of move constructor cannot be marked}} + constexpr S2(const S2&) = default; // expected-error {{defaulted definition of copy constructor is not constexpr}} + constexpr S2(S2&&) = default; // expected-error {{defaulted definition of move constructor is not constexpr}} NoCopyMove ncm; }; diff --git a/clang/test/CXX/drs/dr13xx.cpp b/clang/test/CXX/drs/dr13xx.cpp index d8e3b5d87bd1..effdc53040d0 100644 --- a/clang/test/CXX/drs/dr13xx.cpp +++ b/clang/test/CXX/drs/dr13xx.cpp @@ -1,8 +1,8 @@ // RUN: %clang_cc1 -std=c++98 %s -verify=expected,cxx98-14,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11-20,cxx11-17,cxx11-14,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx11-20,cxx11-17,cxx11-14,since-cxx14,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify=expected,cxx11-20,cxx11-17,since-cxx14,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 %s -verify=expected,cxx11-20,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11-17,cxx11-14,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx11-17,cxx11-14,since-cxx14,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,cxx11-17,since-cxx14,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors // RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors // RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors @@ -485,11 +485,11 @@ namespace dr1358 { // dr1358: 3.1 struct B : Virt { int member; constexpr B(NonLit u) : member(u) {} - // cxx11-20-error@-1 {{constexpr constructor's 1st parameter type 'NonLit' is not a literal type}} - // cxx11-20-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} + // since-cxx11-error@-1 {{constexpr constructor's 1st parameter type 'NonLit' is not a literal type}} + // since-cxx11-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} constexpr NonLit f(NonLit u) const { return NonLit(); } - // cxx11-20-error@-1 {{constexpr function's return type 'NonLit' is not a literal type}} - // cxx11-20-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} + // since-cxx11-error@-1 {{constexpr function's return type 'NonLit' is not a literal type}} + // since-cxx11-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} }; #endif } @@ -498,13 +498,13 @@ namespace dr1359 { // dr1359: 3.5 #if __cplusplus >= 201103L union A { constexpr A() = default; }; union B { constexpr B() = default; int a; }; // #dr1359-B - // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr before C++23}} + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} union C { constexpr C() = default; int a, b; }; // #dr1359-C - // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}} + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} struct X { constexpr X() = default; union {}; }; // since-cxx11-error@-1 {{declaration does not declare anything}} struct Y { constexpr Y() = default; union { int a; }; }; // #dr1359-Y - // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}} + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} constexpr A a = A(); constexpr B b = B(); diff --git a/clang/test/CXX/drs/dr14xx.cpp b/clang/test/CXX/drs/dr14xx.cpp index ed6dda731fd5..58a2b3a0d027 100644 --- a/clang/test/CXX/drs/dr14xx.cpp +++ b/clang/test/CXX/drs/dr14xx.cpp @@ -153,16 +153,16 @@ namespace dr1460 { // dr1460: 3.5 namespace Defaulted { union A { constexpr A() = default; }; union B { int n; constexpr B() = default; }; - // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}} + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} union C { int n = 0; constexpr C() = default; }; struct D { union {}; constexpr D() = default; }; // expected-error@-1 {{declaration does not declare anything}} struct E { union { int n; }; constexpr E() = default; }; - // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}} + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} struct F { union { int n = 0; }; constexpr F() = default; }; struct G { union { int n = 0; }; union { int m; }; constexpr G() = default; }; - // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}} + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} struct H { union { int n = 0; diff --git a/clang/test/CXX/drs/dr15xx.cpp b/clang/test/CXX/drs/dr15xx.cpp index 195c0fa610d5..ac503db625ba 100644 --- a/clang/test/CXX/drs/dr15xx.cpp +++ b/clang/test/CXX/drs/dr15xx.cpp @@ -1,10 +1,10 @@ // RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx11,cxx11-14 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx11,cxx11-14,cxx14-17 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx23,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx23,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,cxx11-14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,cxx11-14,cxx14-17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors namespace dr1512 { // dr1512: 4 void f(char *p) { @@ -407,7 +407,7 @@ namespace dr1573 { // dr1573: 3.9 B b(1, 'x', 4.0, "hello"); // ok // inherited constructor is effectively constexpr if the user-written constructor would be - struct C { C(); constexpr C(int) {} }; // #dr1573-C + struct C { C(); constexpr C(int) {} }; struct D : C { using C::C; }; constexpr D d = D(0); // ok struct E : C { using C::C; A a; }; // #dr1573-E @@ -420,11 +420,8 @@ namespace dr1573 { // dr1573: 3.9 struct F : C { using C::C; C c; }; // #dr1573-F constexpr F f = F(0); // since-cxx11-error@-1 {{constexpr variable 'f' must be initialized by a constant expression}} - // cxx11-20-note@-2 {{constructor inherited from base class 'C' cannot be used in a constant expression; derived class cannot be implicitly initialized}} - // since-cxx23-note@-3 {{in implicit initialization for inherited constructor of 'F'}} - // since-cxx23-note@#dr1573-F {{non-constexpr constructor 'C' cannot be used in a constant expression}} - // cxx11-20-note@#dr1573-F {{declared here}} - // since-cxx23-note@#dr1573-C {{declared here}} + // since-cxx11-note@-2 {{constructor inherited from base class 'C' cannot be used in a constant expression; derived class cannot be implicitly initialized}} + // since-cxx11-note@#dr1573-F {{declared here}} // inherited constructor is effectively deleted if the user-written constructor would be struct G { G(int); }; diff --git a/clang/test/CXX/drs/dr16xx.cpp b/clang/test/CXX/drs/dr16xx.cpp index 766c90d3bc7b..2dd7d1502e59 100644 --- a/clang/test/CXX/drs/dr16xx.cpp +++ b/clang/test/CXX/drs/dr16xx.cpp @@ -1,10 +1,10 @@ // RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx14,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx14,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx23,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx23,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors #if __cplusplus == 199711L #define static_assert(...) __extension__ _Static_assert(__VA_ARGS__) @@ -256,12 +256,12 @@ namespace dr1658 { // dr1658: 5 struct A { A(A&); }; struct B : virtual A { virtual void f() = 0; }; struct C : virtual A { virtual void f(); }; - struct D : A { virtual void f() = 0; }; // since-cxx23-note {{previous declaration is here}} + struct D : A { virtual void f() = 0; }; struct X { friend B::B(const B&) throw(); friend C::C(C&); - friend D::D(D&); // since-cxx23-error {{non-constexpr declaration of 'D' follows constexpr declaration}} + friend D::D(D&); }; } @@ -350,8 +350,8 @@ namespace dr1684 { // dr1684: 3.6 }; constexpr int f(NonLiteral &) { return 0; } constexpr int f(NonLiteral) { return 0; } - // cxx11-20-error@-1 {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}} - // cxx11-20-note@#dr1684-struct {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} + // since-cxx11-error@-1 {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}} + // since-cxx11-note@#dr1684-struct {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} #endif } diff --git a/clang/test/CXX/drs/dr6xx.cpp b/clang/test/CXX/drs/dr6xx.cpp index 190e05784f32..b35d3051ab55 100644 --- a/clang/test/CXX/drs/dr6xx.cpp +++ b/clang/test/CXX/drs/dr6xx.cpp @@ -1,8 +1,8 @@ // RUN: %clang_cc1 -std=c++98 %s -verify=expected,cxx98-17,cxx98-14,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking -// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11-20,cxx98-17,cxx11-17,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking -// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx11-20,cxx98-17,cxx11-17,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking -// RUN: %clang_cc1 -std=c++17 %s -verify=expected,cxx11-20,cxx98-17,cxx11-17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking -// RUN: %clang_cc1 -std=c++20 %s -verify=expected,cxx11-20,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx98-17,cxx11-17,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx98-17,cxx11-17,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,cxx98-17,cxx11-17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking // RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking namespace dr600 { // dr600: 2.8 @@ -584,8 +584,8 @@ namespace dr647 { // dr647: 3.1 struct C { constexpr C(NonLiteral); constexpr C(NonLiteral, int) {} - // cxx11-20-error@-1 {{constexpr constructor's 1st parameter type 'NonLiteral' is not a literal type}} - // cxx11-20-note@#dr647-NonLiteral {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} + // since-cxx11-error@-1 {{constexpr constructor's 1st parameter type 'NonLiteral' is not a literal type}} + // since-cxx11-note@#dr647-NonLiteral {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} constexpr C() try {} catch (...) {} // cxx11-17-error@-1 {{function try block in constexpr constructor is a C++20 extension}} // cxx11-error@-2 {{use of this statement in a constexpr constructor is a C++14 extension}} @@ -609,15 +609,15 @@ namespace dr647 { // dr647: 3.1 d(0) {} constexpr E(int) - // cxx11-20-error@-1 {{constexpr constructor never produces a constant expression}} - // cxx11-20-note@#dr647-int-d {{non-constexpr constructor 'D' cannot be used in a constant expression}} - // cxx11-20-note@#dr647-D-float-ctor {{declared here}} + // since-cxx11-error@-1 {{constexpr constructor never produces a constant expression}} + // since-cxx11-note@#dr647-int-d {{non-constexpr constructor 'D' cannot be used in a constant expression}} + // since-cxx11-note@#dr647-D-float-ctor {{declared here}} : n(0), d(0.0f) {} // #dr647-int-d constexpr E(float f) - // cxx11-20-error@-1 {{never produces a constant expression}} - // cxx11-20-note@#dr647-float-d {{non-constexpr constructor}} - // cxx11-20-note@#dr647-D-float-ctor {{declared here}} + // since-cxx11-error@-1 {{never produces a constant expression}} + // since-cxx11-note@#dr647-float-d {{non-constexpr constructor}} + // since-cxx11-note@#dr647-D-float-ctor {{declared here}} : n(get()), d(D(0) + f) {} // #dr647-float-d }; diff --git a/clang/test/CXX/expr/expr.const/p5-26.cpp b/clang/test/CXX/expr/expr.const/p5-26.cpp index 3624b1e5a3e3..de2afa71b426 100644 --- a/clang/test/CXX/expr/expr.const/p5-26.cpp +++ b/clang/test/CXX/expr/expr.const/p5-26.cpp @@ -5,11 +5,11 @@ struct S {}; struct T : S {} t; -consteval void test() { +consteval void test() { // cxx23-error{{consteval function never produces a constant expression}} void* a = &t; const void* b = &t; volatile void* c = &t; - (void)static_cast(a); + (void)static_cast(a); //cxx23-note {{cast from 'void *' is not allowed in a constant expression in C++ standards before C++2c}} (void)static_cast(a); (void)static_cast(a); diff --git a/clang/test/CXX/special/class.copy/p13-0x.cpp b/clang/test/CXX/special/class.copy/p13-0x.cpp index 013d5b565823..16c8a4029cba 100644 --- a/clang/test/CXX/special/class.copy/p13-0x.cpp +++ b/clang/test/CXX/special/class.copy/p13-0x.cpp @@ -125,7 +125,7 @@ namespace Mutable { mutable A a; }; struct C { - constexpr C(const C &) = default; // expected-error {{cannot be marked constexpr}} + constexpr C(const C &) = default; // expected-error {{not constexpr}} A a; }; } diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp index efb391ba0922..9e2ae07cbe4c 100644 --- a/clang/test/SemaCXX/constant-expression-cxx11.cpp +++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp @@ -1273,8 +1273,8 @@ namespace PR11595 { struct B { B(); A& x; }; static_assert(B().x == 3, ""); // expected-error {{constant expression}} expected-note {{non-literal type 'B' cannot be used in a constant expression}} - constexpr bool f(int k) { // cxx11_20-error {{constexpr function never produces a constant expression}} - return B().x == k; // cxx11_20-note {{non-literal type 'B' cannot be used in a constant expression}} + constexpr bool f(int k) { // expected-error {{constexpr function never produces a constant expression}} + return B().x == k; // expected-note {{non-literal type 'B' cannot be used in a constant expression}} } } @@ -1326,8 +1326,8 @@ namespace ExternConstexpr { constexpr int g() { return q; } // expected-note {{outside its lifetime}} constexpr int q = g(); // expected-error {{constant expression}} expected-note {{in call}} - extern int r; // cxx11_20-note {{here}} - constexpr int h() { return r; } // cxx11_20-error {{never produces a constant}} cxx11_20-note {{read of non-const}} + extern int r; // expected-note {{here}} + constexpr int h() { return r; } // expected-error {{never produces a constant}} expected-note {{read of non-const}} struct S { int n; }; extern const S s; @@ -1678,7 +1678,7 @@ namespace ImplicitConstexpr { struct R { constexpr R() noexcept; constexpr R(const R&) noexcept; constexpr R(R&&) noexcept; ~R() noexcept; }; struct S { R r; }; // expected-note 3{{here}} struct T { T(const T&) noexcept; T(T &&) noexcept; ~T() noexcept; }; - struct U { T t; }; // cxx11_20-note 3{{here}} + struct U { T t; }; // expected-note 3{{here}} static_assert(!__is_literal_type(Q), ""); static_assert(!__is_literal_type(R), ""); static_assert(!__is_literal_type(S), ""); @@ -1691,9 +1691,9 @@ namespace ImplicitConstexpr { friend S::S() noexcept; // expected-error {{follows constexpr}} friend S::S(S&&) noexcept; // expected-error {{follows constexpr}} friend S::S(const S&) noexcept; // expected-error {{follows constexpr}} - friend constexpr U::U() noexcept; // cxx11_20-error {{follows non-constexpr}} - friend constexpr U::U(U&&) noexcept; // cxx11_20-error {{follows non-constexpr}} - friend constexpr U::U(const U&) noexcept; // cxx11_20-error {{follows non-constexpr}} + friend constexpr U::U() noexcept; // expected-error {{follows non-constexpr}} + friend constexpr U::U(U&&) noexcept; // expected-error {{follows non-constexpr}} + friend constexpr U::U(const U&) noexcept; // expected-error {{follows non-constexpr}} }; } @@ -1906,9 +1906,9 @@ namespace StmtExpr { }); } static_assert(g(123) == 15129, ""); - constexpr int h() { // cxx11_20-error {{never produces a constant}} + constexpr int h() { // expected-error {{never produces a constant}} return ({ // expected-warning {{extension}} - return 0; // cxx11_20-note {{not supported}} + return 0; // expected-note {{not supported}} 1; }); } @@ -2093,8 +2093,8 @@ namespace ZeroSizeTypes { // expected-note@-2 {{subtraction of pointers to type 'int[0]' of zero size}} int arr[5][0]; - constexpr int f() { // cxx11_20-error {{never produces a constant expression}} - return &arr[3] - &arr[0]; // cxx11_20-note {{subtraction of pointers to type 'int[0]' of zero size}} + constexpr int f() { // expected-error {{never produces a constant expression}} + return &arr[3] - &arr[0]; // expected-note {{subtraction of pointers to type 'int[0]' of zero size}} } } @@ -2118,8 +2118,8 @@ namespace NeverConstantTwoWays { // If we see something non-constant but foldable followed by something // non-constant and not foldable, we want the first diagnostic, not the // second. - constexpr int f(int n) { // cxx11_20-error {{never produces a constant expression}} - return (int *)(long)&n == &n ? // cxx11_20-note {{reinterpret_cast}} + constexpr int f(int n) { // expected-error {{never produces a constant expression}} + return (int *)(long)&n == &n ? // expected-note {{reinterpret_cast}} 1 / 0 : // expected-warning {{division by zero}} 0; } @@ -2277,8 +2277,7 @@ namespace InheritedCtor { struct A { constexpr A(int) {} }; struct B : A { int n; using A::A; }; // expected-note {{here}} - constexpr B b(0); // expected-error {{constant expression}} cxx11_20-note {{derived class}}\ - // cxx23-note {{not initialized}} + constexpr B b(0); // expected-error {{constant expression}} expected-note {{derived class}} struct C : A { using A::A; struct { union { int n, m = 0; }; union { int a = 0; }; int k = 0; }; struct {}; union {}; }; // expected-warning 6{{}} constexpr C c(0); @@ -2317,11 +2316,10 @@ namespace InheritedCtor { namespace PR28366 { namespace ns1 { -void f(char c) { //expected-note{{declared here}} - //cxx11_20-note@-1{{declared here}} +void f(char c) { //expected-note2{{declared here}} struct X { - static constexpr char f() { // cxx11_20-error {{never produces a constant expression}} - return c; //expected-error{{reference to local}} cxx11_20-note{{function parameter}} + static constexpr char f() { //expected-error{{never produces a constant expression}} + return c; //expected-error{{reference to local}} expected-note{{function parameter}} } }; int I = X::f(); diff --git a/clang/test/SemaCXX/constant-expression-cxx14.cpp b/clang/test/SemaCXX/constant-expression-cxx14.cpp index 80a7a2dd3153..273d7ff3a208 100644 --- a/clang/test/SemaCXX/constant-expression-cxx14.cpp +++ b/clang/test/SemaCXX/constant-expression-cxx14.cpp @@ -44,13 +44,13 @@ constexpr int g(int k) { return 3 * k3 + 5 * k2 + n * k - 20; } static_assert(g(2) == 42, ""); -constexpr int h(int n) { // cxx14_20-error {{constexpr function never produces a constant expression}} - static const int m = n; // cxx14_20-note {{control flows through the definition of a static variable}} \ +constexpr int h(int n) { // expected-error {{constexpr function never produces a constant expression}} + static const int m = n; // expected-note {{control flows through the definition of a static variable}} \ // cxx14_20-warning {{definition of a static variable in a constexpr function is a C++23 extension}} return m; } -constexpr int i(int n) { // cxx14_20-error {{constexpr function never produces a constant expression}} - thread_local const int m = n; // cxx14_20-note {{control flows through the definition of a thread_local variable}} \ +constexpr int i(int n) { // expected-error {{constexpr function never produces a constant expression}} + thread_local const int m = n; // expected-note {{control flows through the definition of a thread_local variable}} \ // cxx14_20-warning {{definition of a thread_local variable in a constexpr function is a C++23 extension}} return m; } @@ -68,7 +68,6 @@ constexpr int j(int k) { } } } // expected-note 2{{control reached end of constexpr function}} - // cxx23-warning@-1 {{does not return a value in all control paths}} static_assert(j(0) == -3, ""); static_assert(j(1) == 5, ""); static_assert(j(2), ""); // expected-error {{constant expression}} expected-note {{in call to 'j(2)'}} @@ -105,10 +104,10 @@ static_assert(l(false) == 5, ""); static_assert(l(true), ""); // expected-error {{constant expression}} expected-note {{in call to 'l(true)'}} // Potential constant expression checking is still applied where possible. -constexpr int htonl(int x) { // cxx14_20-error {{never produces a constant expression}} +constexpr int htonl(int x) { // expected-error {{never produces a constant expression}} typedef unsigned char uchar; uchar arr[4] = { uchar(x >> 24), uchar(x >> 16), uchar(x >> 8), uchar(x) }; - return *reinterpret_cast(arr); // cxx14_20-note {{reinterpret_cast is not allowed in a constant expression}} + return *reinterpret_cast(arr); // expected-note {{reinterpret_cast is not allowed in a constant expression}} } constexpr int maybe_htonl(bool isBigEndian, int x) { @@ -184,7 +183,7 @@ namespace string_assign { static_assert(!test1(100), ""); static_assert(!test1(101), ""); // expected-error {{constant expression}} expected-note {{in call to 'test1(101)'}} - constexpr void f() { // cxx14_20-error{{constexpr function never produces a constant expression}} cxx14_20-note@+2{{assignment to dereferenced one-past-the-end pointer is not allowed in a constant expression}} + constexpr void f() { // expected-error{{constexpr function never produces a constant expression}} expected-note@+2{{assignment to dereferenced one-past-the-end pointer is not allowed in a constant expression}} char foo[10] = { "z" }; // expected-note {{here}} foo[10] = 'x'; // expected-warning {{past the end}} } @@ -208,14 +207,14 @@ namespace array_resize { namespace potential_const_expr { constexpr void set(int &n) { n = 1; } constexpr int div_zero_1() { int z = 0; set(z); return 100 / z; } // no error - constexpr int div_zero_2() { // cxx14_20-error {{never produces a constant expression}} + constexpr int div_zero_2() { // expected-error {{never produces a constant expression}} int z = 0; - return 100 / (set(z), 0); // cxx14_20-note {{division by zero}} + return 100 / (set(z), 0); // expected-note {{division by zero}} } - int n; // cxx14_20-note {{declared here}} - constexpr int ref() { // cxx14_20-error {{never produces a constant expression}} + int n; // expected-note {{declared here}} + constexpr int ref() { // expected-error {{never produces a constant expression}} int &r = n; - return r; // cxx14_20-note {{read of non-const variable 'n'}} + return r; // expected-note {{read of non-const variable 'n'}} } } @@ -847,8 +846,8 @@ namespace StmtExpr { static_assert(g() == 0, ""); // expected-error {{constant expression}} expected-note {{in call}} // FIXME: We should handle the void statement expression case. - constexpr int h() { // cxx14_20-error {{never produces a constant}} - ({ if (true) {} }); // cxx14_20-note {{not supported}} + constexpr int h() { // expected-error {{never produces a constant}} + ({ if (true) {} }); // expected-note {{not supported}} return 0; } } @@ -1044,9 +1043,9 @@ static_assert(sum(Cs) == 'a' + 'b', ""); // expected-error{{not an integral cons constexpr int S = sum(Cs); // expected-error{{must be initialized by a constant expression}} expected-note{{in call}} } -constexpr void PR28739(int n) { // cxx14_20-error {{never produces a constant}} +constexpr void PR28739(int n) { // expected-error {{never produces a constant}} int *p = &n; // expected-note {{array 'p' declared here}} - p += (__int128)(unsigned long)-1; // cxx14_20-note {{cannot refer to element 18446744073709551615 of non-array object in a constant expression}} + p += (__int128)(unsigned long)-1; // expected-note {{cannot refer to element 18446744073709551615 of non-array object in a constant expression}} // expected-warning@-1 {{the pointer incremented by 18446744073709551615 refers past the last possible element for an array in 64-bit address space containing 32-bit (4-byte) elements (max possible 4611686018427387904 elements)}} } diff --git a/clang/test/SemaCXX/constant-expression-cxx2b.cpp b/clang/test/SemaCXX/constant-expression-cxx2b.cpp index 2519839b7ac5..2ee1d48d1cd6 100644 --- a/clang/test/SemaCXX/constant-expression-cxx2b.cpp +++ b/clang/test/SemaCXX/constant-expression-cxx2b.cpp @@ -10,36 +10,36 @@ struct Constexpr{}; #if __cplusplus > 202002L -constexpr int f(int n) { // cxx2a-error {{constexpr function never produces a constant expression}} - static const int m = n; // cxx2a-note {{control flows through the definition of a static variable}} \ +constexpr int f(int n) { // expected-error {{constexpr function never produces a constant expression}} + static const int m = n; // expected-note {{control flows through the definition of a static variable}} \ // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}} return m; } -constexpr int g(int n) { // cxx2a-error {{constexpr function never produces a constant expression}} - thread_local const int m = n; // cxx2a-note {{control flows through the definition of a thread_local variable}} \ +constexpr int g(int n) { // expected-error {{constexpr function never produces a constant expression}} + thread_local const int m = n; // expected-note {{control flows through the definition of a thread_local variable}} \ // cxx23-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++23}} return m; } -constexpr int c_thread_local(int n) { // cxx2a-error {{constexpr function never produces a constant expression}} - static _Thread_local int m = 0; // cxx2a-note {{control flows through the definition of a thread_local variable}} \ +constexpr int c_thread_local(int n) { // expected-error {{constexpr function never produces a constant expression}} + static _Thread_local int m = 0; // expected-note {{control flows through the definition of a thread_local variable}} \ // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}} return m; } -constexpr int gnu_thread_local(int n) { // cxx2a-error {{constexpr function never produces a constant expression}} - static __thread int m = 0; // cxx2a-note {{control flows through the definition of a thread_local variable}} \ +constexpr int gnu_thread_local(int n) { // expected-error {{constexpr function never produces a constant expression}} + static __thread int m = 0; // expected-note {{control flows through the definition of a thread_local variable}} \ // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}} return m; } -constexpr int h(int n) { // cxx2a-error {{constexpr function never produces a constant expression}} - static const int m = n; // cxx2a-note {{control flows through the definition of a static variable}} \ +constexpr int h(int n) { // expected-error {{constexpr function never produces a constant expression}} + static const int m = n; // expected-note {{control flows through the definition of a static variable}} \ // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}} return &m - &m; } -constexpr int i(int n) { // cxx2a-error {{constexpr function never produces a constant expression}} - thread_local const int m = n; // cxx2a-note {{control flows through the definition of a thread_local variable}} \ +constexpr int i(int n) { // expected-error {{constexpr function never produces a constant expression}} + thread_local const int m = n; // expected-note {{control flows through the definition of a thread_local variable}} \ // cxx23-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++23}} return &m - &m; } diff --git a/clang/test/SemaCXX/cxx23-invalid-constexpr.cpp b/clang/test/SemaCXX/cxx23-invalid-constexpr.cpp deleted file mode 100644 index 4dc16c59d805..000000000000 --- a/clang/test/SemaCXX/cxx23-invalid-constexpr.cpp +++ /dev/null @@ -1,159 +0,0 @@ -// RUN: %clang_cc1 -fsyntax-only -verify=expected -std=c++23 %s - -// This test covers modifications made by P2448R2. - -// Check that there is no error when a constexpr function that never produces a -// constant expression, but still an error if such function is called from -// constexpr context. -constexpr int F(int N) { - double D = 2.0 / 0.0; // expected-note {{division by zero}} - return 1; -} - -constexpr int F0(int N) { - if (N == 0) - double d2 = 2.0 / 0.0; // expected-note {{division by zero}} - return 1; -} - -template -constexpr int FT(T N) { - double D = 2.0 / 0.0; // expected-note {{division by zero}} - return 1; -} - -class NonLiteral { // expected-note {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors}} -public: - NonLiteral() {} - ~NonLiteral() {} -}; - -constexpr NonLiteral F1() { - return NonLiteral{}; -} - -constexpr int F2(NonLiteral N) { - return 8; -} - -class Derived : public NonLiteral { - constexpr ~Derived() {}; -}; - -class Derived1 : public NonLiteral { - constexpr Derived1() : NonLiteral () {} -}; - - -struct X { - X(); - X(const X&); - X(X&&); - X& operator=(X&); - X& operator=(X&& other); - bool operator==(X const&) const; -}; - -template -struct Wrapper { - constexpr Wrapper() = default; - constexpr Wrapper(Wrapper const&) = default; - constexpr Wrapper(T const& t) : t(t) { } - constexpr Wrapper(Wrapper &&) = default; - constexpr X get() const { return t; } - constexpr bool operator==(Wrapper const&) const = default; - private: - T t; -}; - -struct WrapperNonT { - constexpr WrapperNonT() = default; - constexpr WrapperNonT(WrapperNonT const&) = default; - constexpr WrapperNonT(X const& t) : t(t) { } - constexpr WrapperNonT(WrapperNonT &&) = default; - constexpr WrapperNonT& operator=(WrapperNonT &) = default; - constexpr WrapperNonT& operator=(WrapperNonT&& other) = default; - constexpr X get() const { return t; } - constexpr bool operator==(WrapperNonT const&) const = default; - private: - X t; -}; - -struct NonDefaultMembers { - constexpr NonDefaultMembers() {}; // expected-note {{non-literal type 'X' cannot be used in a constant expression}} - constexpr NonDefaultMembers(NonDefaultMembers const&) {}; - constexpr NonDefaultMembers(NonDefaultMembers &&) {}; - constexpr NonDefaultMembers& operator=(NonDefaultMembers &other) {this->t = other.t; return *this;} - constexpr NonDefaultMembers& operator=(NonDefaultMembers&& other) {this->t = other.t; return *this;} - constexpr bool operator==(NonDefaultMembers const& other) const {return this->t == other.t;} - X t; -}; - -int Glob = 0; -class C1 { -public: - constexpr C1() : D(Glob) {}; -private: - int D; -}; - -void test() { - - constexpr int A = F(3); // expected-error {{constexpr variable 'A' must be initialized by a constant expression}} - // expected-note@-1 {{in call}} - F(3); - constexpr int B = F0(0); // expected-error {{constexpr variable 'B' must be initialized by a constant expression}} - // expected-note@-1 {{in call}} - F0(0); - constexpr auto C = F1(); // expected-error {{constexpr variable cannot have non-literal type 'const NonLiteral'}} - F1(); - NonLiteral L; - constexpr auto D = F2(L); // expected-error {{constexpr variable 'D' must be initialized by a constant expression}} - // expected-note@-1 {{non-literal type 'NonLiteral' cannot be used in a constant expression}} - - constexpr auto E = FT(1); // expected-error {{constexpr variable 'E' must be initialized by a constant expression}} - // expected-note@-1 {{in call}} - F2(L); - - Wrapper x; - WrapperNonT x1; - NonDefaultMembers x2; - - // TODO these produce notes with an invalid source location. - // static_assert((Wrapper(), true)); - // static_assert((WrapperNonT(), true),""); - - static_assert((NonDefaultMembers(), true),""); // expected-error{{expression is not an integral constant expression}} \ - // expected-note {{in call to}} - constexpr bool FFF = (NonDefaultMembers() == NonDefaultMembers()); // expected-error{{must be initialized by a constant expression}} \ - // expected-note{{non-literal}} -} - -struct A { - A (); - ~A(); -}; - -template -struct opt -{ - union { - char c; - T data; - }; - - constexpr opt() {} - - constexpr ~opt() { - if (engaged) - data.~T(); - } - - bool engaged = false; -}; - -consteval void foo() { - opt a; -} - -void bar() { foo(); } diff --git a/clang/test/SemaCXX/cxx2a-consteval.cpp b/clang/test/SemaCXX/cxx2a-consteval.cpp index 192621225a54..d8482ec53f0e 100644 --- a/clang/test/SemaCXX/cxx2a-consteval.cpp +++ b/clang/test/SemaCXX/cxx2a-consteval.cpp @@ -54,7 +54,7 @@ struct C { struct D { C c; - consteval D() = default; // expected-error {{cannot be marked consteval}} + consteval D() = default; // expected-error {{cannot be consteval}} consteval ~D() = default; // expected-error {{destructor cannot be declared consteval}} }; diff --git a/clang/test/SemaCXX/deduced-return-type-cxx14.cpp b/clang/test/SemaCXX/deduced-return-type-cxx14.cpp index 431d77ca785b..415bbbf1a0bc 100644 --- a/clang/test/SemaCXX/deduced-return-type-cxx14.cpp +++ b/clang/test/SemaCXX/deduced-return-type-cxx14.cpp @@ -1,8 +1,8 @@ // RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,since-cxx20,since-cxx14,cxx20_23,cxx23 %s // RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,since-cxx20,since-cxx14,cxx20_23,cxx23 %s -fdelayed-template-parsing -DDELAYED_TEMPLATE_PARSING -// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx20,since-cxx20,since-cxx14,cxx14_20,cxx20_23 %s -// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx20,since-cxx20,since-cxx14,cxx14_20,cxx20_23 %s -fdelayed-template-parsing -DDELAYED_TEMPLATE_PARSING +// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,since-cxx20,since-cxx14,cxx14_20,cxx20_23 %s +// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,since-cxx20,since-cxx14,cxx14_20,cxx20_23 %s -fdelayed-template-parsing -DDELAYED_TEMPLATE_PARSING // RUN: %clang_cc1 -std=c++14 -fsyntax-only -verify=expected,since-cxx14,cxx14_20,cxx14 %s // RUN: %clang_cc1 -std=c++14 -fsyntax-only -verify=expected,since-cxx14,cxx14_20,cxx14 %s -fdelayed-template-parsing -DDELAYED_TEMPLATE_PARSING @@ -299,8 +299,8 @@ namespace Constexpr { constexpr int q = Y().f(); // expected-error {{must be initialized by a constant expression}} expected-note {{in call to 'Y().f()'}} } struct NonLiteral { ~NonLiteral(); } nl; // cxx14-note {{user-provided destructor}} - // cxx20-note@-1 {{'NonLiteral' is not literal because its destructor is not constexpr}} - constexpr auto f2(int n) { return nl; } // cxx14_20-error {{constexpr function's return type 'struct NonLiteral' is not a literal type}} + // cxx20_23-note@-1 {{'NonLiteral' is not literal because its destructor is not constexpr}} + constexpr auto f2(int n) { return nl; } // expected-error {{return type 'struct NonLiteral' is not a literal type}} } // It's not really clear whether these are valid, but this matches g++. diff --git a/clang/test/SemaOpenCLCXX/addrspace-constructors.clcpp b/clang/test/SemaOpenCLCXX/addrspace-constructors.clcpp index 067a404c489a..1b97484767b1 100644 --- a/clang/test/SemaOpenCLCXX/addrspace-constructors.clcpp +++ b/clang/test/SemaOpenCLCXX/addrspace-constructors.clcpp @@ -54,5 +54,5 @@ struct Z { struct W { int w; - constexpr W() __constant = default; // expected-error {{defaulted definition of default constructor cannot be marked constexpr}} + constexpr W() __constant = default; // expected-error {{defaulted definition of default constructor is not constexpr}} }; diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html index 1e36b90356c3..fe3fc0926e49 100755 --- a/clang/www/cxx_status.html +++ b/clang/www/cxx_status.html @@ -356,7 +356,14 @@ C++23, informally referred to as C++26.

Relaxing some constexpr restrictions P2448R2 - Clang 19 + +
Clang 17 (Partial) + We do not support outside of defaulted special memeber functions the change that constexpr functions no + longer have to be constexpr compatible but rather support a less restricted requirements for constexpr + functions. Which include allowing non-literal types as return values and parameters, allow calling of + non-constexpr functions and constructors. +
+ Using unknown pointers and references in constant expressions -- GitLab From 207e45fb67ee3dbec9590d9303eebf4f720c8a40 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 13 Mar 2024 14:56:25 -0700 Subject: [PATCH 0198/1407] [RISCV] Add back SiFive's cdiscard.d.l1, cflush.d.l1, and cease instructions. (#83896) These were in LLVM 17 but removed from LLVM 18 due to an incorrect extension name being used. This restores them with new extension names that match SiFive's downstream compiler. The extension name has been used internally for some time. It uses XSiFive instead of XSf like the newer extensions. `cease` did not have an internal extension name so its using the `XSf` convention. The spec for the instructions is here https://sifive.cdn.prismic.io/sifive/767804da-53b2-4893-97d5-b7c030ae0a94_s76mc_core_complex_manual_21G3.pdf though the extension name is not listed. Column width in the extension printing had to be changed to accommodate a longer extension name. --- .../test/Preprocessor/riscv-target-features.c | 27 ++ llvm/docs/RISCVUsage.rst | 9 + llvm/lib/Support/RISCVISAInfo.cpp | 5 +- .../RISCV/Disassembler/RISCVDisassembler.cpp | 8 + llvm/lib/Target/RISCV/RISCVFeatures.td | 24 ++ llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td | 32 ++ llvm/test/MC/RISCV/xsifive-invalid.s | 20 ++ llvm/test/MC/RISCV/xsifive-valid.s | 36 ++ llvm/unittests/Support/RISCVISAInfoTest.cpp | 327 +++++++++--------- 9 files changed, 325 insertions(+), 163 deletions(-) create mode 100644 llvm/test/MC/RISCV/xsifive-invalid.s create mode 100644 llvm/test/MC/RISCV/xsifive-valid.s diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c index 1a15be1c6e4d..b2cad622610b 100644 --- a/clang/test/Preprocessor/riscv-target-features.c +++ b/clang/test/Preprocessor/riscv-target-features.c @@ -56,11 +56,14 @@ // CHECK-NOT: __riscv_xcvmac {{.*$}} // CHECK-NOT: __riscv_xcvmem {{.*$}} // CHECK-NOT: __riscv_xcvsimd {{.*$}} +// CHECK-NOT: __riscv_xsfcease {{.*$}} // CHECK-NOT: __riscv_xsfvcp {{.*$}} // CHECK-NOT: __riscv_xsfvfnrclipxfqf {{.*$}} // CHECK-NOT: __riscv_xsfvfwmaccqqq {{.*$}} // CHECK-NOT: __riscv_xsfqmaccdod {{.*$}} // CHECK-NOT: __riscv_xsfvqmaccqoq {{.*$}} +// CHECK-NOT: __riscv_xsifivecdiscarddlone {{.*$}} +// CHECK-NOT: __riscv_xsifivecflushdlone {{.*$}} // CHECK-NOT: __riscv_xtheadba {{.*$}} // CHECK-NOT: __riscv_xtheadbb {{.*$}} // CHECK-NOT: __riscv_xtheadbs {{.*$}} @@ -517,6 +520,14 @@ // RUN: -o - | FileCheck --check-prefix=CHECK-XCVSIMD-EXT %s // CHECK-XCVSIMD-EXT: __riscv_xcvsimd 1000000{{$}} +// RUN: %clang --target=riscv32-unknown-linux-gnu \ +// RUN: -march=rv32ixsfcease -E -dM %s \ +// RUN: -o - | FileCheck --check-prefix=CHECK-XSFCEASE-EXT %s +// RUN: %clang --target=riscv64-unknown-linux-gnu \ +// RUN: -march=rv64ixsfcease -E -dM %s \ +// RUN: -o - | FileCheck --check-prefix=CHECK-XSFCEASE-EXT %s +// CHECK-XSFCEASE-EXT: __riscv_xsfcease 1000000{{$}} + // RUN: %clang --target=riscv32-unknown-linux-gnu \ // RUN: -march=rv32ixsfvcp -E -dM %s \ // RUN: -o - | FileCheck --check-prefix=CHECK-XSFVCP-EXT %s @@ -557,6 +568,22 @@ // RUN: -o - | FileCheck --check-prefix=CHECK-XSFVQMACCQOQ-EXT %s // CHECK-XSFVQMACCQOQ-EXT: __riscv_xsfvqmaccqoq 1000000{{$}} +// RUN: %clang --target=riscv32-unknown-linux-gnu \ +// RUN: -march=rv32ixsifivecdiscarddlone -E -dM %s \ +// RUN: -o - | FileCheck --check-prefix=CHECK-XSIFIVECDISCARDDLONE-EXT %s +// RUN: %clang --target=riscv64-unknown-linux-gnu \ +// RUN: -march=rv64ixsifivecdiscarddlone -E -dM %s \ +// RUN: -o - | FileCheck --check-prefix=CHECK-XSIFIVECDISCARDDLONE-EXT %s +// CHECK-XSIFIVECDISCARDDLONE-EXT: __riscv_xsifivecdiscarddlone 1000000{{$}} + +// RUN: %clang --target=riscv32-unknown-linux-gnu \ +// RUN: -march=rv32ixsifivecflushdlone -E -dM %s \ +// RUN: -o - | FileCheck --check-prefix=CHECK-XSIFIVECFLUSHDLONE-EXT %s +// RUN: %clang --target=riscv64-unknown-linux-gnu \ +// RUN: -march=rv64ixsifivecflushdlone -E -dM %s \ +// RUN: -o - | FileCheck --check-prefix=CHECK-XSIFIVECFLUSHDLONE-EXT %s +// CHECK-XSIFIVECFLUSHDLONE-EXT: __riscv_xsifivecflushdlone 1000000{{$}} + // RUN: %clang --target=riscv32-unknown-linux-gnu \ // RUN: -march=rv32ixtheadba -E -dM %s \ // RUN: -o - | FileCheck --check-prefix=CHECK-XTHEADBA-EXT %s diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst index a1de8596480d..2f17c9d7dda0 100644 --- a/llvm/docs/RISCVUsage.rst +++ b/llvm/docs/RISCVUsage.rst @@ -362,6 +362,15 @@ The current vendor extensions supported are: ``XCVbi`` LLVM implements `version 1.0.0 of the CORE-V immediate branching custom instructions specification `__ by OpenHW Group. All instructions are prefixed with `cv.` as described in the specification. These instructions are only available for riscv32 at this time. +``XSiFivecdiscarddlone`` + LLVM implements `the SiFive sf.cdiscard.d.l1 instruction specified in `_ by SiFive. + +``XSiFivecflushdlone`` + LLVM implements `the SiFive sf.cflush.d.l1 instruction specified in `_ by SiFive. + +``XSfcease`` + LLVM implements `the SiFive sf.cease instruction specified in `_ by SiFive. + Experimental C Intrinsics ========================= diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 6eec03fd6f70..39235ace4724 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -90,11 +90,14 @@ static const RISCVSupportedExtension SupportedExtensions[] = { {"xcvmac", {1, 0}}, {"xcvmem", {1, 0}}, {"xcvsimd", {1, 0}}, + {"xsfcease", {1, 0}}, {"xsfvcp", {1, 0}}, {"xsfvfnrclipxfqf", {1, 0}}, {"xsfvfwmaccqqq", {1, 0}}, {"xsfvqmaccdod", {1, 0}}, {"xsfvqmaccqoq", {1, 0}}, + {"xsifivecdiscarddlone", {1, 0}}, + {"xsifivecflushdlone", {1, 0}}, {"xtheadba", {1, 0}}, {"xtheadbb", {1, 0}}, {"xtheadbs", {1, 0}}, @@ -258,7 +261,7 @@ static void PrintExtension(StringRef Name, StringRef Version, StringRef Description) { outs().indent(4); unsigned VersionWidth = Description.empty() ? 0 : 10; - outs() << left_justify(Name, 20) << left_justify(Version, VersionWidth) + outs() << left_justify(Name, 21) << left_justify(Version, VersionWidth) << Description << "\n"; } diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp index f1ca1212ec37..6aadabdf1bc6 100644 --- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp +++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp @@ -595,6 +595,14 @@ DecodeStatus RISCVDisassembler::getInstruction(MCInst &MI, uint64_t &Size, TRY_TO_DECODE_FEATURE( RISCV::FeatureVendorXSfvfnrclipxfqf, DecoderTableXSfvfnrclipxfqf32, "SiFive FP32-to-int8 Ranged Clip Instructions opcode table"); + TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXSiFivecdiscarddlone, + DecoderTableXSiFivecdiscarddlone32, + "SiFive sf.cdiscard.d.l1 custom opcode table"); + TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXSiFivecflushdlone, + DecoderTableXSiFivecflushdlone32, + "SiFive sf.cflush.d.l1 custom opcode table"); + TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXSfcease, DecoderTableXSfcease32, + "SiFive sf.cease custom opcode table"); TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXCVbitmanip, DecoderTableXCVbitmanip32, "CORE-V Bit Manipulation custom opcode table"); diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index 83619ccb24ba..f3e641e25018 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -1058,6 +1058,30 @@ def HasVendorXSfvfnrclipxfqf AssemblerPredicate<(all_of FeatureVendorXSfvfnrclipxfqf), "'XSfvfnrclipxfqf' (SiFive FP32-to-int8 Ranged Clip Instructions)">; +def FeatureVendorXSiFivecdiscarddlone + : SubtargetFeature<"xsifivecdiscarddlone", "HasVendorXSiFivecdiscarddlone", "true", + "'XSiFivecdiscarddlone' (SiFive sf.cdiscard.d.l1 Instruction)", []>; +def HasVendorXSiFivecdiscarddlone + : Predicate<"Subtarget->hasVendorXSiFivecdiscarddlone()">, + AssemblerPredicate<(all_of FeatureVendorXSiFivecdiscarddlone), + "'XSiFivecdiscarddlone' (SiFive sf.cdiscard.d.l1 Instruction)">; + +def FeatureVendorXSiFivecflushdlone + : SubtargetFeature<"xsifivecflushdlone", "HasVendorXSiFivecflushdlone", "true", + "'XSiFivecflushdlone' (SiFive sf.cflush.d.l1 Instruction)", []>; +def HasVendorXSiFivecflushdlone + : Predicate<"Subtarget->hasVendorXSiFivecflushdlone()">, + AssemblerPredicate<(all_of FeatureVendorXSiFivecflushdlone), + "'XSiFivecflushdlone' (SiFive sf.cflush.d.l1 Instruction)">; + +def FeatureVendorXSfcease + : SubtargetFeature<"xsfcease", "HasVendorXSfcease", "true", + "'XSfcease' (SiFive sf.cease Instruction)", []>; +def HasVendorXSfcease + : Predicate<"Subtarget->hasVendorXSfcease()">, + AssemblerPredicate<(all_of FeatureVendorXSfcease), + "'XSfcease' (SiFive sf.cease Instruction)">; + // Core-V Extensions def FeatureVendorXCVelw diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td index b4130e3805a1..9a6818c99af2 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td @@ -808,3 +808,35 @@ let Predicates = [HasVendorXSfvfnrclipxfqf] in { defm : VPatVFNRCLIP<"vfnrclip_xu_f_qf", "VFNRCLIP_XU_F_QF">; defm : VPatVFNRCLIP<"vfnrclip_x_f_qf", "VFNRCLIP_X_F_QF">; } + +let Predicates = [HasVendorXSiFivecdiscarddlone] in { + let hasNoSchedulingInfo = 1, hasSideEffects = 1, mayLoad = 0, mayStore = 0, + DecoderNamespace = "XSiFivecdiscarddlone" in + def SF_CDISCARD_D_L1 + : RVInstIUnary<0b111111000010, 0b000, OPC_SYSTEM, (outs), (ins GPR:$rs1), + "sf.cdiscard.d.l1", "$rs1">, Sched<[]> { + let rd = 0; + } + def : InstAlias<"sf.cdiscard.d.l1", (SF_CDISCARD_D_L1 X0)>; +} // Predicates = [HasVendorXSifivecdiscarddlone] + +let Predicates = [HasVendorXSiFivecflushdlone] in { + let hasNoSchedulingInfo = 1, hasSideEffects = 1, mayLoad = 0, mayStore = 0, + DecoderNamespace = "XSiFivecflushdlone" in + def SF_CFLUSH_D_L1 + : RVInstIUnary<0b111111000000, 0b000, OPC_SYSTEM, (outs), (ins GPR:$rs1), + "sf.cflush.d.l1", "$rs1">, Sched<[]> { + let rd = 0; + } + def : InstAlias<"sf.cflush.d.l1", (SF_CFLUSH_D_L1 X0)>; +} // Predicates = [HasVendorXSifivecflushdlone] + +let Predicates = [HasVendorXSfcease] in { + let hasNoSchedulingInfo = 1, hasSideEffects = 1, mayLoad = 0, mayStore = 0, + DecoderNamespace = "XSfcease" in + def SF_CEASE : RVInstIUnary<0b001100000101, 0b000, OPC_SYSTEM, (outs), (ins), + "sf.cease", "">, Sched<[]> { + let rs1 = 0b00000; + let rd = 0b00000; +} +} diff --git a/llvm/test/MC/RISCV/xsifive-invalid.s b/llvm/test/MC/RISCV/xsifive-invalid.s new file mode 100644 index 000000000000..5210d29f9d36 --- /dev/null +++ b/llvm/test/MC/RISCV/xsifive-invalid.s @@ -0,0 +1,20 @@ +# RUN: not llvm-mc -triple riscv32 < %s 2>&1 | FileCheck %s +# RUN: not llvm-mc -triple riscv64 < %s 2>&1 | FileCheck %s + +sf.cflush.d.l1 0x10 # CHECK: :[[@LINE]]:16: error: invalid operand for instruction + +sf.cdiscard.d.l1 0x10 # CHECK: :[[@LINE]]:18: error: invalid operand for instruction + +sf.cflush.d.l1 x0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: 'XSiFivecflushdlone' (SiFive sf.cflush.d.l1 Instruction){{$}} + +sf.cflush.d.l1 x7 # CHECK: :[[@LINE]]:1: error: instruction requires the following: 'XSiFivecflushdlone' (SiFive sf.cflush.d.l1 Instruction){{$}} + +sf.cdiscard.d.l1 x0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: 'XSiFivecdiscarddlone' (SiFive sf.cdiscard.d.l1 Instruction){{$}} + +sf.cdiscard.d.l1 x7 # CHECK: :[[@LINE]]:1: error: instruction requires the following: 'XSiFivecdiscarddlone' (SiFive sf.cdiscard.d.l1 Instruction){{$}} + +sf.cease x1 # CHECK: :[[@LINE]]:10: error: invalid operand for instruction + +sf.cease 0x10 # CHECK: :[[@LINE]]:10: error: invalid operand for instruction + +sf.cease # CHECK: :[[@LINE]]:1: error: instruction requires the following: 'XSfcease' (SiFive sf.cease Instruction){{$}} diff --git a/llvm/test/MC/RISCV/xsifive-valid.s b/llvm/test/MC/RISCV/xsifive-valid.s new file mode 100644 index 000000000000..8aa0ab1bd8ba --- /dev/null +++ b/llvm/test/MC/RISCV/xsifive-valid.s @@ -0,0 +1,36 @@ +# RUN: llvm-mc %s -triple=riscv32 -mattr=+xsifivecdiscarddlone,+xsifivecflushdlone,+xsfcease -riscv-no-aliases -show-encoding \ +# RUN: | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST %s +# RUN: llvm-mc %s -triple=riscv64 -mattr=+xsifivecdiscarddlone,+xsifivecflushdlone,+xsfcease -riscv-no-aliases -show-encoding \ +# RUN: | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST %s +# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+xsifivecdiscarddlone,+xsifivecflushdlone,+xsfcease < %s \ +# RUN: | llvm-objdump --mattr=+xsifivecdiscarddlone,+xsifivecflushdlone,+xsfcease -M no-aliases -d - \ +# RUN: | FileCheck -check-prefix=CHECK-INST %s +# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+xsifivecdiscarddlone,+xsifivecflushdlone,+xsfcease < %s \ +# RUN: | llvm-objdump --mattr=+xsifivecdiscarddlone,+xsifivecflushdlone,+xsfcease -M no-aliases -d - \ +# RUN: | FileCheck -check-prefix=CHECK-INST %s + +# CHECK-INST: sf.cflush.d.l1 zero +# CHECK-ENC: encoding: [0x73,0x00,0x00,0xfc] +sf.cflush.d.l1 x0 +# CHECK-INST: sf.cflush.d.l1 zero +# CHECK-ENC: encoding: [0x73,0x00,0x00,0xfc] +sf.cflush.d.l1 + +# CHECK-INST: sf.cflush.d.l1 t2 +# CHECK-ENC: encoding: [0x73,0x80,0x03,0xfc] +sf.cflush.d.l1 x7 + +# CHECK-INST: sf.cdiscard.d.l1 zero +# CHECK-ENC: encoding: [0x73,0x00,0x20,0xfc] +sf.cdiscard.d.l1 x0 +# CHECK-INST: sf.cdiscard.d.l1 zero +# CHECK-ENC: encoding: [0x73,0x00,0x20,0xfc] +sf.cdiscard.d.l1 + +# CHECK-INST: sf.cdiscard.d.l1 t2 +# CHECK-ENC: encoding: [0x73,0x80,0x23,0xfc] +sf.cdiscard.d.l1 x7 + +# CHECK-INST: sf.cease +# CHECK-ENC: encoding: [0x73,0x00,0x50,0x30] +sf.cease diff --git a/llvm/unittests/Support/RISCVISAInfoTest.cpp b/llvm/unittests/Support/RISCVISAInfoTest.cpp index 82cf4c639b61..a331e6a74ceb 100644 --- a/llvm/unittests/Support/RISCVISAInfoTest.cpp +++ b/llvm/unittests/Support/RISCVISAInfoTest.cpp @@ -739,170 +739,173 @@ TEST(RiscvExtensionsHelp, CheckExtensions) { std::string ExpectedOutput = R"(All available -march extensions for RISC-V - Name Version Description - i 2.1 This is a long dummy description - e 2.0 - m 2.0 - a 2.1 - f 2.2 - d 2.2 - c 2.0 - v 1.0 - h 1.0 - zic64b 1.0 - zicbom 1.0 - zicbop 1.0 - zicboz 1.0 - ziccamoa 1.0 - ziccif 1.0 - zicclsm 1.0 - ziccrse 1.0 - zicntr 2.0 - zicond 1.0 - zicsr 2.0 - zifencei 2.0 - zihintntl 1.0 - zihintpause 2.0 - zihpm 2.0 - zmmul 1.0 - za128rs 1.0 - za64rs 1.0 - zacas 1.0 - zawrs 1.0 - zfa 1.0 - zfh 1.0 - zfhmin 1.0 - zfinx 1.0 - zdinx 1.0 - zca 1.0 - zcb 1.0 - zcd 1.0 - zce 1.0 - zcf 1.0 - zcmp 1.0 - zcmt 1.0 - zba 1.0 - zbb 1.0 - zbc 1.0 - zbkb 1.0 - zbkc 1.0 - zbkx 1.0 - zbs 1.0 - zk 1.0 - zkn 1.0 - zknd 1.0 - zkne 1.0 - zknh 1.0 - zkr 1.0 - zks 1.0 - zksed 1.0 - zksh 1.0 - zkt 1.0 - zvbb 1.0 - zvbc 1.0 - zve32f 1.0 - zve32x 1.0 - zve64d 1.0 - zve64f 1.0 - zve64x 1.0 - zvfh 1.0 - zvfhmin 1.0 - zvkb 1.0 - zvkg 1.0 - zvkn 1.0 - zvknc 1.0 - zvkned 1.0 - zvkng 1.0 - zvknha 1.0 - zvknhb 1.0 - zvks 1.0 - zvksc 1.0 - zvksed 1.0 - zvksg 1.0 - zvksh 1.0 - zvkt 1.0 - zvl1024b 1.0 - zvl128b 1.0 - zvl16384b 1.0 - zvl2048b 1.0 - zvl256b 1.0 - zvl32768b 1.0 - zvl32b 1.0 - zvl4096b 1.0 - zvl512b 1.0 - zvl64b 1.0 - zvl65536b 1.0 - zvl8192b 1.0 - zhinx 1.0 - zhinxmin 1.0 - shcounterenw 1.0 - shgatpa 1.0 - shtvala 1.0 - shvsatpa 1.0 - shvstvala 1.0 - shvstvecd 1.0 - smaia 1.0 - smepmp 1.0 - ssaia 1.0 - ssccptr 1.0 - sscofpmf 1.0 - sscounterenw 1.0 - ssstateen 1.0 - ssstrict 1.0 - sstc 1.0 - sstvala 1.0 - sstvecd 1.0 - ssu64xl 1.0 - svade 1.0 - svadu 1.0 - svbare 1.0 - svinval 1.0 - svnapot 1.0 - svpbmt 1.0 - xcvalu 1.0 - xcvbi 1.0 - xcvbitmanip 1.0 - xcvelw 1.0 - xcvmac 1.0 - xcvmem 1.0 - xcvsimd 1.0 - xsfvcp 1.0 - xsfvfnrclipxfqf 1.0 - xsfvfwmaccqqq 1.0 - xsfvqmaccdod 1.0 - xsfvqmaccqoq 1.0 - xtheadba 1.0 - xtheadbb 1.0 - xtheadbs 1.0 - xtheadcmo 1.0 - xtheadcondmov 1.0 - xtheadfmemidx 1.0 - xtheadmac 1.0 - xtheadmemidx 1.0 - xtheadmempair 1.0 - xtheadsync 1.0 - xtheadvdot 1.0 - xventanacondops 1.0 + Name Version Description + i 2.1 This is a long dummy description + e 2.0 + m 2.0 + a 2.1 + f 2.2 + d 2.2 + c 2.0 + v 1.0 + h 1.0 + zic64b 1.0 + zicbom 1.0 + zicbop 1.0 + zicboz 1.0 + ziccamoa 1.0 + ziccif 1.0 + zicclsm 1.0 + ziccrse 1.0 + zicntr 2.0 + zicond 1.0 + zicsr 2.0 + zifencei 2.0 + zihintntl 1.0 + zihintpause 2.0 + zihpm 2.0 + zmmul 1.0 + za128rs 1.0 + za64rs 1.0 + zacas 1.0 + zawrs 1.0 + zfa 1.0 + zfh 1.0 + zfhmin 1.0 + zfinx 1.0 + zdinx 1.0 + zca 1.0 + zcb 1.0 + zcd 1.0 + zce 1.0 + zcf 1.0 + zcmp 1.0 + zcmt 1.0 + zba 1.0 + zbb 1.0 + zbc 1.0 + zbkb 1.0 + zbkc 1.0 + zbkx 1.0 + zbs 1.0 + zk 1.0 + zkn 1.0 + zknd 1.0 + zkne 1.0 + zknh 1.0 + zkr 1.0 + zks 1.0 + zksed 1.0 + zksh 1.0 + zkt 1.0 + zvbb 1.0 + zvbc 1.0 + zve32f 1.0 + zve32x 1.0 + zve64d 1.0 + zve64f 1.0 + zve64x 1.0 + zvfh 1.0 + zvfhmin 1.0 + zvkb 1.0 + zvkg 1.0 + zvkn 1.0 + zvknc 1.0 + zvkned 1.0 + zvkng 1.0 + zvknha 1.0 + zvknhb 1.0 + zvks 1.0 + zvksc 1.0 + zvksed 1.0 + zvksg 1.0 + zvksh 1.0 + zvkt 1.0 + zvl1024b 1.0 + zvl128b 1.0 + zvl16384b 1.0 + zvl2048b 1.0 + zvl256b 1.0 + zvl32768b 1.0 + zvl32b 1.0 + zvl4096b 1.0 + zvl512b 1.0 + zvl64b 1.0 + zvl65536b 1.0 + zvl8192b 1.0 + zhinx 1.0 + zhinxmin 1.0 + shcounterenw 1.0 + shgatpa 1.0 + shtvala 1.0 + shvsatpa 1.0 + shvstvala 1.0 + shvstvecd 1.0 + smaia 1.0 + smepmp 1.0 + ssaia 1.0 + ssccptr 1.0 + sscofpmf 1.0 + sscounterenw 1.0 + ssstateen 1.0 + ssstrict 1.0 + sstc 1.0 + sstvala 1.0 + sstvecd 1.0 + ssu64xl 1.0 + svade 1.0 + svadu 1.0 + svbare 1.0 + svinval 1.0 + svnapot 1.0 + svpbmt 1.0 + xcvalu 1.0 + xcvbi 1.0 + xcvbitmanip 1.0 + xcvelw 1.0 + xcvmac 1.0 + xcvmem 1.0 + xcvsimd 1.0 + xsfcease 1.0 + xsfvcp 1.0 + xsfvfnrclipxfqf 1.0 + xsfvfwmaccqqq 1.0 + xsfvqmaccdod 1.0 + xsfvqmaccqoq 1.0 + xsifivecdiscarddlone 1.0 + xsifivecflushdlone 1.0 + xtheadba 1.0 + xtheadbb 1.0 + xtheadbs 1.0 + xtheadcmo 1.0 + xtheadcondmov 1.0 + xtheadfmemidx 1.0 + xtheadmac 1.0 + xtheadmemidx 1.0 + xtheadmempair 1.0 + xtheadsync 1.0 + xtheadvdot 1.0 + xventanacondops 1.0 Experimental extensions - zicfilp 0.4 This is a long dummy description - zicfiss 0.4 - zimop 0.1 - zaamo 0.2 - zabha 1.0 - zalasr 0.1 - zalrsc 0.2 - zfbfmin 1.0 - zcmop 0.2 - ztso 0.1 - zvfbfmin 1.0 - zvfbfwma 1.0 - smmpm 0.8 - smnpm 0.8 - ssnpm 0.8 - sspm 0.8 - ssqosid 1.0 - supm 0.8 + zicfilp 0.4 This is a long dummy description + zicfiss 0.4 + zimop 0.1 + zaamo 0.2 + zabha 1.0 + zalasr 0.1 + zalrsc 0.2 + zfbfmin 1.0 + zcmop 0.2 + ztso 0.1 + zvfbfmin 1.0 + zvfbfwma 1.0 + smmpm 0.8 + smnpm 0.8 + ssnpm 0.8 + sspm 0.8 + ssqosid 1.0 + supm 0.8 Use -march to specify the target's extension. For example, clang -march=rv32i_v1p0)"; -- GitLab From 7bdba956efae81f111f0cc6c7aaa92f9712444ba Mon Sep 17 00:00:00 2001 From: Fehr Mathieu Date: Wed, 13 Mar 2024 21:59:34 +0000 Subject: [PATCH 0199/1407] [mlir][arith] Fix `arith.select` canonicalization patterns (#84685) Because `arith.select` does not propagate poison of the second or third operand depending on the condition, some canonicalization patterns are currently incorrect. This patch removes these incorrect patterns, and adds a new pattern to fix the case of `i1` select with constants. Patterns that are removed: * select(predA, select(predB, x, y), y) => select(and(predA, predB), x, y) * select(predA, select(predB, y, x), y) => select(and(predA, not(predB)), x, y) * select(predA, x, select(predB, x, y)) => select(or(predA, predB), x, y) * select(predA, x, select(predB, y, x)) => select(or(predA, not(predB)), x, y) * arith.select %arg, %x, %y : i1 => and(%arg, %x) or and(!%arg, %y) Pattern that is added: * select(pred, false, true) => not(pred) for i1 The first two patterns are incorrect when `predB` is poison and `predA` is false, as a non-poison `y` gets compiled to `poison`. The next two patterns are incorrect when `predB` is poison and `predA` is true, as a non-poison `x` gets compiled to `poison`. The last pattern is incorrect as it propagates poison from all operands afer compilation. --- .../Dialect/Arith/IR/ArithCanonicalization.td | 34 ++------ mlir/lib/Dialect/Arith/IR/ArithOps.cpp | 35 +------- mlir/test/Dialect/Arith/canonicalize.mlir | 80 ------------------- 3 files changed, 8 insertions(+), 141 deletions(-) diff --git a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td index 11c4a29718e1..caca2ff81964 100644 --- a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td +++ b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td @@ -253,9 +253,6 @@ def CmpIExtUI : // SelectOp //===----------------------------------------------------------------------===// -def GetScalarOrVectorTrueAttribute : - NativeCodeCall<"cast(getBoolAttribute($0.getType(), true))">; - // select(not(pred), a, b) => select(pred, b, a) def SelectNotCond : Pat<(SelectOp (Arith_XOrIOp $pred, (ConstantLikeMatcher APIntAttr:$ones)), $a, $b), @@ -272,31 +269,12 @@ def RedundantSelectFalse : Pat<(SelectOp $pred, $a, (SelectOp $pred, $b, $c)), (SelectOp $pred, $a, $c)>; -// select(predA, select(predB, x, y), y) => select(and(predA, predB), x, y) -def SelectAndCond : - Pat<(SelectOp $predA, (SelectOp $predB, $x, $y), $y), - (SelectOp (Arith_AndIOp $predA, $predB), $x, $y)>; - -// select(predA, select(predB, y, x), y) => select(and(predA, not(predB)), x, y) -def SelectAndNotCond : - Pat<(SelectOp $predA, (SelectOp $predB, $y, $x), $y), - (SelectOp (Arith_AndIOp $predA, - (Arith_XOrIOp $predB, - (Arith_ConstantOp (GetScalarOrVectorTrueAttribute $predB)))), - $x, $y)>; - -// select(predA, x, select(predB, x, y)) => select(or(predA, predB), x, y) -def SelectOrCond : - Pat<(SelectOp $predA, $x, (SelectOp $predB, $x, $y)), - (SelectOp (Arith_OrIOp $predA, $predB), $x, $y)>; - -// select(predA, x, select(predB, y, x)) => select(or(predA, not(predB)), x, y) -def SelectOrNotCond : - Pat<(SelectOp $predA, $x, (SelectOp $predB, $y, $x)), - (SelectOp (Arith_OrIOp $predA, - (Arith_XOrIOp $predB, - (Arith_ConstantOp (GetScalarOrVectorTrueAttribute $predB)))), - $x, $y)>; +// select(pred, false, true) => not(pred) +def SelectI1ToNot : + Pat<(SelectOp $pred, + (ConstantLikeMatcher ConstantAttr), + (ConstantLikeMatcher ConstantAttr)), + (Arith_XOrIOp $pred, (Arith_ConstantOp ConstantAttr))>; //===----------------------------------------------------------------------===// // IndexCastOp diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp index 0f71c19c23b6..9f64a07f31e3 100644 --- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp +++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp @@ -969,7 +969,6 @@ OpFoldResult arith::MaxNumFOp::fold(FoldAdaptor adaptor) { [](const APFloat &a, const APFloat &b) { return llvm::maximum(a, b); }); } - //===----------------------------------------------------------------------===// // MaxSIOp //===----------------------------------------------------------------------===// @@ -2173,35 +2172,6 @@ void arith::CmpFOp::getCanonicalizationPatterns(RewritePatternSet &patterns, // SelectOp //===----------------------------------------------------------------------===// -// Transforms a select of a boolean to arithmetic operations -// -// arith.select %arg, %x, %y : i1 -// -// becomes -// -// and(%arg, %x) or and(!%arg, %y) -struct SelectI1Simplify : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(arith::SelectOp op, - PatternRewriter &rewriter) const override { - if (!op.getType().isInteger(1)) - return failure(); - - Value falseConstant = - rewriter.create(op.getLoc(), true, 1); - Value notCondition = rewriter.create( - op.getLoc(), op.getCondition(), falseConstant); - - Value trueVal = rewriter.create( - op.getLoc(), op.getCondition(), op.getTrueValue()); - Value falseVal = rewriter.create(op.getLoc(), notCondition, - op.getFalseValue()); - rewriter.replaceOpWithNewOp(op, trueVal, falseVal); - return success(); - } -}; - // select %arg, %c1, %c0 => extui %arg struct SelectToExtUI : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; @@ -2238,9 +2208,8 @@ struct SelectToExtUI : public OpRewritePattern { void arith::SelectOp::getCanonicalizationPatterns(RewritePatternSet &results, MLIRContext *context) { - results.add(context); + results.add(context); } OpFoldResult arith::SelectOp::fold(FoldAdaptor adaptor) { diff --git a/mlir/test/Dialect/Arith/canonicalize.mlir b/mlir/test/Dialect/Arith/canonicalize.mlir index cb98a10048a3..bdc6c91d9267 100644 --- a/mlir/test/Dialect/Arith/canonicalize.mlir +++ b/mlir/test/Dialect/Arith/canonicalize.mlir @@ -116,18 +116,6 @@ func.func @selToNot(%arg0: i1) -> i1 { return %res : i1 } -// CHECK-LABEL: @selToArith -// CHECK-NEXT: %[[trueval:.+]] = arith.constant true -// CHECK-NEXT: %[[notcmp:.+]] = arith.xori %arg0, %[[trueval]] : i1 -// CHECK-NEXT: %[[condtrue:.+]] = arith.andi %arg0, %arg1 : i1 -// CHECK-NEXT: %[[condfalse:.+]] = arith.andi %[[notcmp]], %arg2 : i1 -// CHECK-NEXT: %[[res:.+]] = arith.ori %[[condtrue]], %[[condfalse]] : i1 -// CHECK: return %[[res]] -func.func @selToArith(%arg0: i1, %arg1 : i1, %arg2 : i1) -> i1 { - %res = arith.select %arg0, %arg1, %arg2 : i1 - return %res : i1 -} - // CHECK-LABEL: @redundantSelectTrue // CHECK-NEXT: %[[res:.+]] = arith.select %arg0, %arg1, %arg3 // CHECK-NEXT: return %[[res]] @@ -160,74 +148,6 @@ func.func @selNotCond(%arg0: i1, %arg1 : i32, %arg2 : i32, %arg3 : i32, %arg4 : return %res1, %res2 : i32, i32 } -// CHECK-LABEL: @selAndCond -// CHECK-NEXT: %[[and:.+]] = arith.andi %arg1, %arg0 -// CHECK-NEXT: %[[res:.+]] = arith.select %[[and]], %arg2, %arg3 -// CHECK-NEXT: return %[[res]] -func.func @selAndCond(%arg0: i1, %arg1: i1, %arg2 : i32, %arg3 : i32) -> i32 { - %sel = arith.select %arg0, %arg2, %arg3 : i32 - %res = arith.select %arg1, %sel, %arg3 : i32 - return %res : i32 -} - -// CHECK-LABEL: @selAndNotCond -// CHECK-NEXT: %[[one:.+]] = arith.constant true -// CHECK-NEXT: %[[not:.+]] = arith.xori %arg0, %[[one]] -// CHECK-NEXT: %[[and:.+]] = arith.andi %arg1, %[[not]] -// CHECK-NEXT: %[[res:.+]] = arith.select %[[and]], %arg3, %arg2 -// CHECK-NEXT: return %[[res]] -func.func @selAndNotCond(%arg0: i1, %arg1: i1, %arg2 : i32, %arg3 : i32) -> i32 { - %sel = arith.select %arg0, %arg2, %arg3 : i32 - %res = arith.select %arg1, %sel, %arg2 : i32 - return %res : i32 -} - -// CHECK-LABEL: @selAndNotCondVec -// CHECK-NEXT: %[[one:.+]] = arith.constant dense : vector<4xi1> -// CHECK-NEXT: %[[not:.+]] = arith.xori %arg0, %[[one]] -// CHECK-NEXT: %[[and:.+]] = arith.andi %arg1, %[[not]] -// CHECK-NEXT: %[[res:.+]] = arith.select %[[and]], %arg3, %arg2 -// CHECK-NEXT: return %[[res]] -func.func @selAndNotCondVec(%arg0: vector<4xi1>, %arg1: vector<4xi1>, %arg2 : vector<4xi32>, %arg3 : vector<4xi32>) -> vector<4xi32> { - %sel = arith.select %arg0, %arg2, %arg3 : vector<4xi1>, vector<4xi32> - %res = arith.select %arg1, %sel, %arg2 : vector<4xi1>, vector<4xi32> - return %res : vector<4xi32> -} - -// CHECK-LABEL: @selOrCond -// CHECK-NEXT: %[[or:.+]] = arith.ori %arg1, %arg0 -// CHECK-NEXT: %[[res:.+]] = arith.select %[[or]], %arg2, %arg3 -// CHECK-NEXT: return %[[res]] -func.func @selOrCond(%arg0: i1, %arg1: i1, %arg2 : i32, %arg3 : i32) -> i32 { - %sel = arith.select %arg0, %arg2, %arg3 : i32 - %res = arith.select %arg1, %arg2, %sel : i32 - return %res : i32 -} - -// CHECK-LABEL: @selOrNotCond -// CHECK-NEXT: %[[one:.+]] = arith.constant true -// CHECK-NEXT: %[[not:.+]] = arith.xori %arg0, %[[one]] -// CHECK-NEXT: %[[or:.+]] = arith.ori %arg1, %[[not]] -// CHECK-NEXT: %[[res:.+]] = arith.select %[[or]], %arg3, %arg2 -// CHECK-NEXT: return %[[res]] -func.func @selOrNotCond(%arg0: i1, %arg1: i1, %arg2 : i32, %arg3 : i32) -> i32 { - %sel = arith.select %arg0, %arg2, %arg3 : i32 - %res = arith.select %arg1, %arg3, %sel : i32 - return %res : i32 -} - -// CHECK-LABEL: @selOrNotCondVec -// CHECK-NEXT: %[[one:.+]] = arith.constant dense : vector<4xi1> -// CHECK-NEXT: %[[not:.+]] = arith.xori %arg0, %[[one]] -// CHECK-NEXT: %[[or:.+]] = arith.ori %arg1, %[[not]] -// CHECK-NEXT: %[[res:.+]] = arith.select %[[or]], %arg3, %arg2 -// CHECK-NEXT: return %[[res]] -func.func @selOrNotCondVec(%arg0: vector<4xi1>, %arg1: vector<4xi1>, %arg2 : vector<4xi32>, %arg3 : vector<4xi32>) -> vector<4xi32> { - %sel = arith.select %arg0, %arg2, %arg3 : vector<4xi1>, vector<4xi32> - %res = arith.select %arg1, %arg3, %sel : vector<4xi1>, vector<4xi32> - return %res : vector<4xi32> -} - // Test case: Folding of comparisons with equal operands. // CHECK-LABEL: @cmpi_equal_operands // CHECK-DAG: %[[T:.*]] = arith.constant true -- GitLab From fc71a49eca630a0f201261b89c5c9b0252ddb48b Mon Sep 17 00:00:00 2001 From: Peter Klausler <35819229+klausler@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:02:00 -0700 Subject: [PATCH 0200/1407] [flang][runtime] Handle end of internal output correctly (#84994) At the end of an internal output statement, be sure to finish any following control edit descriptors in the format (if any), and (for output) advance to the next record. Return the right I/O error status code if output overruns the buffer. --- flang/runtime/format-implementation.h | 27 +++++++++++++++++---------- flang/runtime/internal-unit.cpp | 16 +++++----------- flang/runtime/internal-unit.h | 1 - flang/runtime/io-stmt.cpp | 19 +++++++++++++++---- flang/runtime/io-stmt.h | 1 + 5 files changed, 38 insertions(+), 26 deletions(-) diff --git a/flang/runtime/format-implementation.h b/flang/runtime/format-implementation.h index 9c342db2e19a..f554f740573f 100644 --- a/flang/runtime/format-implementation.h +++ b/flang/runtime/format-implementation.h @@ -66,15 +66,6 @@ template int FormatControl::GetIntField( IoErrorHandler &handler, CharType firstCh, bool *hadError) { CharType ch{firstCh ? firstCh : PeekNext()}; - if (ch != '-' && ch != '+' && (ch < '0' || ch > '9')) { - handler.SignalError(IostatErrorInFormat, - "Invalid FORMAT: integer expected at '%c'", static_cast(ch)); - if (hadError) { - *hadError = true; - } - return 0; - } - int result{0}; bool negate{ch == '-'}; if (negate || ch == '+') { if (firstCh) { @@ -84,6 +75,15 @@ int FormatControl::GetIntField( } ch = PeekNext(); } + if (ch < '0' || ch > '9') { + handler.SignalError(IostatErrorInFormat, + "Invalid FORMAT: integer expected at '%c'", static_cast(ch)); + if (hadError) { + *hadError = true; + } + return 0; + } + int result{0}; while (ch >= '0' && ch <= '9') { constexpr int tenth{std::numeric_limits::max() / 10}; if (result > tenth || @@ -246,8 +246,15 @@ int FormatControl::CueUpNextDataEdit(Context &context, bool stop) { ch = GetNextChar(context); } if (ch == '-' || ch == '+' || (ch >= '0' && ch <= '9')) { + bool hadSign{ch == '-' || ch == '+'}; repeat = GetIntField(context, ch); ch = GetNextChar(context); + if (hadSign && ch != 'p' && ch != 'P') { + ReportBadFormat(context, + "Invalid FORMAT: signed integer may appear only before 'P", + maybeReversionPoint); + return 0; + } } else if (ch == '*') { unlimited = true; ch = GetNextChar(context); @@ -297,11 +304,11 @@ int FormatControl::CueUpNextDataEdit(Context &context, bool stop) { return 0; } else if (ch == ')') { if (height_ == 1) { + hitEnd_ = true; if (stop) { return 0; // end of FORMAT and no data items remain } context.AdvanceRecord(); // implied / before rightmost ) - hitEnd_ = true; } auto restart{stack_[height_ - 1].start}; if (format_[restart] == '(') { diff --git a/flang/runtime/internal-unit.cpp b/flang/runtime/internal-unit.cpp index e3fffaa6f378..66140e005887 100644 --- a/flang/runtime/internal-unit.cpp +++ b/flang/runtime/internal-unit.cpp @@ -41,16 +41,6 @@ InternalDescriptorUnit::InternalDescriptorUnit( endfileRecordNumber = d.Elements() + 1; } -template void InternalDescriptorUnit::EndIoStatement() { - if constexpr (DIR == Direction::Output) { - // Clear the remainder of the current record. - auto end{endfileRecordNumber.value_or(0)}; - if (currentRecordNumber < end) { - BlankFillOutputRecord(); - } - } -} - template bool InternalDescriptorUnit::Emit( const char *data, std::size_t bytes, IoErrorHandler &handler) { @@ -109,7 +99,11 @@ std::size_t InternalDescriptorUnit::GetNextInputBytes( template bool InternalDescriptorUnit::AdvanceRecord(IoErrorHandler &handler) { if (currentRecordNumber >= endfileRecordNumber.value_or(0)) { - handler.SignalEnd(); + if constexpr (DIR == Direction::Input) { + handler.SignalEnd(); + } else { + handler.SignalError(IostatInternalWriteOverrun); + } return false; } if constexpr (DIR == Direction::Output) { diff --git a/flang/runtime/internal-unit.h b/flang/runtime/internal-unit.h index f0c50aac9887..b536ffb831d5 100644 --- a/flang/runtime/internal-unit.h +++ b/flang/runtime/internal-unit.h @@ -28,7 +28,6 @@ public: std::conditional_t; InternalDescriptorUnit(Scalar, std::size_t chars, int kind); InternalDescriptorUnit(const Descriptor &, const Terminator &); - void EndIoStatement(); bool Emit(const char *, std::size_t, IoErrorHandler &); std::size_t GetNextInputBytes(const char *&, IoErrorHandler &); diff --git a/flang/runtime/io-stmt.cpp b/flang/runtime/io-stmt.cpp index 3ec01ffba9bf..153195fd9656 100644 --- a/flang/runtime/io-stmt.cpp +++ b/flang/runtime/io-stmt.cpp @@ -119,9 +119,6 @@ template void InternalIoStatementState::BackspaceRecord() { } template int InternalIoStatementState::EndIoStatement() { - if constexpr (DIR == Direction::Output) { - unit_.EndIoStatement(); // fill - } auto result{IoStatementBase::EndIoStatement()}; if (free_) { FreeMemory(this); @@ -165,7 +162,8 @@ template void InternalFormattedIoStatementState::CompleteOperation() { if (!this->completedOperation()) { if constexpr (DIR == Direction::Output) { - format_.Finish(*this); // ignore any remaining input positioning actions + format_.Finish(*this); + unit_.AdvanceRecord(*this); } IoStatementBase::CompleteOperation(); } @@ -189,8 +187,21 @@ InternalListIoStatementState::InternalListIoStatementState( : InternalIoStatementState{d, sourceFile, sourceLine}, ioStatementState_{*this} {} +template +void InternalListIoStatementState::CompleteOperation() { + if (!this->completedOperation()) { + if constexpr (DIR == Direction::Output) { + if (unit_.furthestPositionInRecord > 0) { + unit_.AdvanceRecord(*this); + } + } + IoStatementBase::CompleteOperation(); + } +} + template int InternalListIoStatementState::EndIoStatement() { + CompleteOperation(); if constexpr (DIR == Direction::Input) { if (int status{ListDirectedStatementState::EndIoStatement()}; status != IostatOk) { diff --git a/flang/runtime/io-stmt.h b/flang/runtime/io-stmt.h index 0b6bcbd9af02..dcee7f936569 100644 --- a/flang/runtime/io-stmt.h +++ b/flang/runtime/io-stmt.h @@ -403,6 +403,7 @@ public: const Descriptor &, const char *sourceFile = nullptr, int sourceLine = 0); IoStatementState &ioStatementState() { return ioStatementState_; } using ListDirectedStatementState::GetNextDataEdit; + void CompleteOperation(); int EndIoStatement(); private: -- GitLab From 605abe0689dfd28aadc9413306f33a4494cf3fb8 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 13 Mar 2024 15:06:55 -0700 Subject: [PATCH 0201/1407] [clang] Initialize AllTocData after #67999 --- clang/include/clang/Basic/CodeGenOptions.def | 1 + clang/include/clang/Basic/CodeGenOptions.h | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def index 6ad050319625..340b08dd7e2a 100644 --- a/clang/include/clang/Basic/CodeGenOptions.def +++ b/clang/include/clang/Basic/CodeGenOptions.def @@ -59,6 +59,7 @@ CODEGENOPT(UniqueBasicBlockSectionNames, 1, 1) ///< Set for -funique-basic-block ///< basic block sections. CODEGENOPT(EnableAIXExtendedAltivecABI, 1, 0) ///< Set for -mabi=vec-extabi. Enables the extended Altivec ABI on AIX. CODEGENOPT(XCOFFReadOnlyPointers, 1, 0) ///< Set for -mxcoff-roptr. +CODEGENOPT(AllTocData, 1, 0) ///< AIX -mtocdata ENUM_CODEGENOPT(FramePointer, FramePointerKind, 2, FramePointerKind::None) /// frame-pointer: all,non-leaf,none CODEGENOPT(ClearASTBeforeBackend , 1, 0) ///< Free the AST before running backend code generation. Only works with -disable-free. diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h index cf29e576ef32..9469a424045b 100644 --- a/clang/include/clang/Basic/CodeGenOptions.h +++ b/clang/include/clang/Basic/CodeGenOptions.h @@ -410,9 +410,6 @@ public: /// List of global variables that over-ride the toc-data default. std::vector NoTocDataVars; - /// Flag for all global variables to be treated as toc-data. - bool AllTocData; - /// Path to allowlist file specifying which objects /// (files, functions) should exclusively be instrumented /// by sanitizer coverage pass. -- GitLab From 702a86a8f1e4d96c62574fc8d7dd9ccea243517a Mon Sep 17 00:00:00 2001 From: Peter Klausler <35819229+klausler@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:13:56 -0700 Subject: [PATCH 0202/1407] =?UTF-8?q?[flang]=20Correct=20accessibility=20o?= =?UTF-8?q?f=20name=20that=20is=20both=20generic=20and=20derive=E2=80=A6?= =?UTF-8?q?=20(#85098)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …d type When the same name is used for a derived type and generic interface in a module, and no explicit PUBLIC or PRIVATE statement appears for the name but the derived type definition does have an explicit accessibility, that accessibility must also apply to the generic interface. --- flang/lib/Semantics/resolve-names.cpp | 19 +++++++++++--- flang/test/Semantics/resolve11.f90 | 37 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp index 67392a02cf18..b13674573fe0 100644 --- a/flang/lib/Semantics/resolve-names.cpp +++ b/flang/lib/Semantics/resolve-names.cpp @@ -3391,12 +3391,25 @@ void ModuleVisitor::ApplyDefaultAccess() { const auto *moduleDetails{ DEREF(currScope().symbol()).detailsIf()}; CHECK(moduleDetails); + Attr defaultAttr{ + DEREF(moduleDetails).isDefaultPrivate() ? Attr::PRIVATE : Attr::PUBLIC}; for (auto &pair : currScope()) { Symbol &symbol{*pair.second}; if (!symbol.attrs().HasAny({Attr::PUBLIC, Attr::PRIVATE})) { - SetImplicitAttr(symbol, - DEREF(moduleDetails).isDefaultPrivate() ? Attr::PRIVATE - : Attr::PUBLIC); + Attr attr{defaultAttr}; + if (auto *generic{symbol.detailsIf()}) { + if (generic->derivedType()) { + // If a generic interface has a derived type of the same + // name that has an explicit accessibility attribute, then + // the generic must have the same accessibility. + if (generic->derivedType()->attrs().test(Attr::PUBLIC)) { + attr = Attr::PUBLIC; + } else if (generic->derivedType()->attrs().test(Attr::PRIVATE)) { + attr = Attr::PRIVATE; + } + } + } + SetImplicitAttr(symbol, attr); } } } diff --git a/flang/test/Semantics/resolve11.f90 b/flang/test/Semantics/resolve11.f90 index 33ce88342b49..db508f062d1d 100644 --- a/flang/test/Semantics/resolve11.f90 +++ b/flang/test/Semantics/resolve11.f90 @@ -49,3 +49,40 @@ module m3 !ERROR: The accessibility of 'OPERATOR(.GT.)' has already been specified as PUBLIC private :: operator(.gt.) end + +module m4 + private + type, public :: foo + end type + interface foo + procedure fun + end interface + contains + function fun + end +end + +subroutine s4 + !ERROR: 'fun' is PRIVATE in 'm4' + use m4, only: foo, fun + type(foo) x ! ok + print *, foo() ! ok +end + +module m5 + public + type, private :: foo + end type + interface foo + procedure fun + end interface + contains + function fun + end +end + +subroutine s5 + !ERROR: 'foo' is PRIVATE in 'm5' + use m5, only: foo, fun + print *, fun() ! ok +end -- GitLab From 35db929b50af51e18c75da74b23caa6c14beeaf6 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 13 Mar 2024 15:13:46 -0700 Subject: [PATCH 0204/1407] [RISCV] Add cost model coverage for fixed vector insert with known VLEN --- .../RISCV/shuffle-insert_subvector.ll | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll index 9a333dc8b8dd..a91d562b3f6f 100644 --- a/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll +++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll @@ -520,3 +520,69 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i ret void } + +define void @fixed_m1_in_m2_notail(<8 x i32> %src, <8 x i32> %passthru) vscale_range(2) { +; CHECK-LABEL: 'fixed_m1_in_m2_notail' +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; SIZE-LABEL: 'fixed_m1_in_m2_notail' +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %1 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> + shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> + shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> + shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> + shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> + ret void +} + +define void @fixed_m2_in_m4_notail(<8 x i64> %src, <8 x i64> %passthru) vscale_range(2) { +; CHECK-LABEL: 'fixed_m2_in_m4_notail' +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %1 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 41 for instruction: %2 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 41 for instruction: %3 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 41 for instruction: %4 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 41 for instruction: %5 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; SIZE-LABEL: 'fixed_m2_in_m4_notail' +; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %1 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> +; SIZE-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %2 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> +; SIZE-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %3 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> +; SIZE-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %4 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> +; SIZE-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %5 = shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> + shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> + shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> + shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> + shufflevector <8 x i64> %src, <8 x i64> %passthru, <8 x i32> + ret void +} + +define void @fixed_m1_in_m2_tail(<8 x i32> %src, <8 x i32> %passthru) vscale_range(2) { +; CHECK-LABEL: 'fixed_m1_in_m2_tail' +; CHECK-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %1 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %2 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; SIZE-LABEL: 'fixed_m1_in_m2_tail' +; SIZE-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %1 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; SIZE-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %2 = shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> + shufflevector <8 x i32> %src, <8 x i32> %passthru, <8 x i32> + ret void +} -- GitLab From 6885810e7de283ee8d3c8fc328a98544970b3db6 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Wed, 13 Mar 2024 15:33:11 -0700 Subject: [PATCH 0205/1407] [llvm] Include LLVM_REPOSITORY and LLVM_REVISION in tool version (#84990) Include the `LLVM_REPOSITORY` and `LLVM_REVISION` in the version output of tools using `cl::PrintVersionMessage()` such as dwarfdump and dsymutil. Before: ``` $ llvm-dwarfdump --version LLVM (http://llvm.org/): LLVM version 19.0.0git Optimized build with assertions. ``` After: ``` $ llvm-dwarfdump --version LLVM (http://llvm.org/): LLVM version 19.0.0git (git@github.com:llvm/llvm-project.git 8467457afc61d70e881c9817ace26356ef757733) Optimized build with assertions. ``` rdar://121526866 --- llvm/lib/Support/CMakeLists.txt | 3 +++ llvm/lib/Support/CommandLine.cpp | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt index 1f2d82427552..b9c13c43e9a7 100644 --- a/llvm/lib/Support/CMakeLists.txt +++ b/llvm/lib/Support/CMakeLists.txt @@ -286,6 +286,9 @@ add_llvm_component_library(LLVMSupport ${LLVM_MAIN_INCLUDE_DIR}/llvm/Support ${Backtrace_INCLUDE_DIRS} + DEPENDS + llvm_vcsrevision_h + LINK_LIBS ${system_libs} ${imported_libs} ${delayload_flags} diff --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp index c076ae8b8431..42dbc4de2003 100644 --- a/llvm/lib/Support/CommandLine.cpp +++ b/llvm/lib/Support/CommandLine.cpp @@ -39,6 +39,7 @@ #include "llvm/Support/Path.h" #include "llvm/Support/Process.h" #include "llvm/Support/StringSaver.h" +#include "llvm/Support/VCSRevision.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/raw_ostream.h" #include @@ -2538,7 +2539,15 @@ public: #else OS << "LLVM (http://llvm.org/):\n "; #endif - OS << PACKAGE_NAME << " version " << PACKAGE_VERSION << "\n "; + OS << PACKAGE_NAME << " version " << PACKAGE_VERSION; +#ifdef LLVM_REPOSITORY + OS << " (" << LLVM_REPOSITORY; +#ifdef LLVM_REVISION + OS << ' ' << LLVM_REVISION; +#endif + OS << ')'; +#endif + OS << "\n "; #if LLVM_IS_DEBUG_BUILD OS << "DEBUG build"; #else -- GitLab From db058b954a32dfb164926e407dfcf49bec054216 Mon Sep 17 00:00:00 2001 From: Charlie Barto Date: Wed, 13 Mar 2024 15:34:12 -0700 Subject: [PATCH 0206/1407] [asan][windows] fix issue64990 test (#85137) This was broken by https://github.com/llvm/llvm-project/pull/84971 --- compiler-rt/test/asan/TestCases/Windows/issue64990.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp b/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp index aab66502bd16..b1b6e42148cb 100644 --- a/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp +++ b/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp @@ -16,5 +16,5 @@ int main(int argc, char **argv) { } return 0; } - -// CHECK: SUMMARY: AddressSanitizer: global-buffer-overflow {{.*}} in __asan_memcpy +// CHECK: #0 {{.*}} in __asan_memcpy +// CHECK: SUMMARY: AddressSanitizer: global-buffer-overflow {{.*}} in main -- GitLab From b87db5b6c28f1b5b2358213351d837bc72777cd5 Mon Sep 17 00:00:00 2001 From: Slava Zakharin Date: Wed, 13 Mar 2024 16:04:16 -0700 Subject: [PATCH 0207/1407] [flang][runtime] Fixed flang-runtime-cuda-gcc builder after af964c7. (#85144) --- flang/include/flang/Runtime/api-attrs.h | 12 ++++++++++++ flang/runtime/environment.cpp | 4 +++- flang/runtime/environment.h | 12 ++++++++++-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/flang/include/flang/Runtime/api-attrs.h b/flang/include/flang/Runtime/api-attrs.h index 9c8a67ffc34a..fc3eb42e1b73 100644 --- a/flang/include/flang/Runtime/api-attrs.h +++ b/flang/include/flang/Runtime/api-attrs.h @@ -109,6 +109,18 @@ #endif #endif /* !defined(RT_CONST_VAR_ATTRS) */ +/* + * RT_VAR_ATTRS is marking non-const/constexpr module scope variables + * referenced by Flang runtime. + */ +#ifndef RT_VAR_ATTRS +#if (defined(__CUDACC__) || defined(__CUDA__)) && defined(__CUDA_ARCH__) +#define RT_VAR_ATTRS __device__ +#else +#define RT_VAR_ATTRS +#endif +#endif /* !defined(RT_VAR_ATTRS) */ + /* * RT_DEVICE_COMPILATION is defined for any device compilation. * Note that it can only be used reliably with compilers that perform diff --git a/flang/runtime/environment.cpp b/flang/runtime/environment.cpp index 29196ae8f310..fe6701d72c9f 100644 --- a/flang/runtime/environment.cpp +++ b/flang/runtime/environment.cpp @@ -23,7 +23,9 @@ extern char **environ; namespace Fortran::runtime { -ExecutionEnvironment executionEnvironment; +RT_OFFLOAD_VAR_GROUP_BEGIN +RT_VAR_ATTRS ExecutionEnvironment executionEnvironment; +RT_OFFLOAD_VAR_GROUP_END static void SetEnvironmentDefaults(const EnvironmentDefaultList *envDefaults) { if (!envDefaults) { diff --git a/flang/runtime/environment.h b/flang/runtime/environment.h index 6da2c7bb3cf7..49c7dbd2940f 100644 --- a/flang/runtime/environment.h +++ b/flang/runtime/environment.h @@ -10,6 +10,7 @@ #define FORTRAN_RUNTIME_ENVIRONMENT_H_ #include "flang/Decimal/decimal.h" +#include "flang/Runtime/api-attrs.h" #include struct EnvironmentDefaultList; @@ -32,7 +33,11 @@ enum class Convert { Unknown, Native, LittleEndian, BigEndian, Swap }; std::optional GetConvertFromString(const char *, std::size_t); struct ExecutionEnvironment { - constexpr ExecutionEnvironment(){}; +#if !defined(_OPENMP) + // FIXME: https://github.com/llvm/llvm-project/issues/84942 + constexpr +#endif + ExecutionEnvironment(){}; void Configure(int argc, const char *argv[], const char *envp[], const EnvironmentDefaultList *envDefaults); const char *GetEnv( @@ -51,7 +56,10 @@ struct ExecutionEnvironment { bool checkPointerDeallocation{true}; // FORT_CHECK_POINTER_DEALLOCATION }; -extern ExecutionEnvironment executionEnvironment; +RT_OFFLOAD_VAR_GROUP_BEGIN +extern RT_VAR_ATTRS ExecutionEnvironment executionEnvironment; +RT_OFFLOAD_VAR_GROUP_END + } // namespace Fortran::runtime #endif // FORTRAN_RUNTIME_ENVIRONMENT_H_ -- GitLab From 2dc9ec47fb16a01c8f7cbb76fba4ad00ac4cb81b Mon Sep 17 00:00:00 2001 From: ChiaHungDuan Date: Wed, 13 Mar 2024 16:05:24 -0700 Subject: [PATCH 0208/1407] [scudo] Refactor allocator config to support optional flags (#81805) Instead of explicitly disabling a feature by declaring the variable and set it to false, this change supports the optional flags. I.e., you can skip certain flags if you are not using it. This optional feature supports both forms, 1. Value: A parameter for a feature. E.g., EnableRandomOffset 2. Type: A C++ type implementing a feature. E.g., ConditionVariableT On the other hand, to access the flags will be through one of the wrappers, BaseConfig/PrimaryConfig/SecondaryConfig/CacheConfig (CacheConfig is embedded in SecondaryConfig). These wrappers have the getters to access the value and the type. When adding a new feature, we need to add it to `allocator_config.def` and mark the new variable with either *_REQUIRED_* or *_OPTIONAL_* macro so that the accessor will be generated properly. In addition, also remove the need of `UseConditionVariable` to flip on/off of condition variable. Now we only need to define the type of condition variable. --- .../lib/scudo/standalone/CMakeLists.txt | 1 + .../lib/scudo/standalone/allocator_config.def | 124 ++++++++++++++++ .../lib/scudo/standalone/allocator_config.h | 78 +--------- .../standalone/allocator_config_wrapper.h | 135 ++++++++++++++++++ compiler-rt/lib/scudo/standalone/combined.h | 54 +++---- .../lib/scudo/standalone/condition_variable.h | 16 --- compiler-rt/lib/scudo/standalone/memtag.h | 2 +- compiler-rt/lib/scudo/standalone/primary32.h | 25 ++-- compiler-rt/lib/scudo/standalone/primary64.h | 25 ++-- compiler-rt/lib/scudo/standalone/secondary.h | 36 +++-- .../lib/scudo/standalone/tests/CMakeLists.txt | 1 + .../tests/allocator_config_test.cpp | 119 +++++++++++++++ .../scudo/standalone/tests/combined_test.cpp | 1 - .../scudo/standalone/tests/primary_test.cpp | 29 +++- .../scudo/standalone/tests/secondary_test.cpp | 22 ++- 15 files changed, 498 insertions(+), 170 deletions(-) create mode 100644 compiler-rt/lib/scudo/standalone/allocator_config.def create mode 100644 compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h create mode 100644 compiler-rt/lib/scudo/standalone/tests/allocator_config_test.cpp diff --git a/compiler-rt/lib/scudo/standalone/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/CMakeLists.txt index 60092005cc33..6fb4e88de315 100644 --- a/compiler-rt/lib/scudo/standalone/CMakeLists.txt +++ b/compiler-rt/lib/scudo/standalone/CMakeLists.txt @@ -58,6 +58,7 @@ endif() set(SCUDO_HEADERS allocator_common.h allocator_config.h + allocator_config_wrapper.h atomic_helpers.h bytemap.h checksum.h diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.def b/compiler-rt/lib/scudo/standalone/allocator_config.def new file mode 100644 index 000000000000..92f4e39872d4 --- /dev/null +++ b/compiler-rt/lib/scudo/standalone/allocator_config.def @@ -0,0 +1,124 @@ +//===-- allocator_config.def ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines all the flags and types supported in Scudo. For optional +// flags and types, only explicitly define them when interested (i.e., unused +// optional flags or types can be skipped). + +#ifndef BASE_REQUIRED_TEMPLATE_TYPE +#define BASE_REQUIRED_TEMPLATE_TYPE(...) +#endif +#ifndef BASE_OPTIONAL +#define BASE_OPTIONAL(...) +#endif +#ifndef PRIMARY_REQUIRED_TYPE +#define PRIMARY_REQUIRED_TYPE(...) +#endif +#ifndef PRIMARY_REQUIRED +#define PRIMARY_REQUIRED(...) +#endif +#ifndef PRIMARY_OPTIONAL +#define PRIMARY_OPTIONAL(...) +#endif +#ifndef PRIMARY_OPTIONAL_TYPE +#define PRIMARY_OPTIONAL_TYPE(...) +#endif +#ifndef SECONDARY_REQUIRED_TEMPLATE_TYPE +#define SECONDARY_REQUIRED_TEMPLATE_TYPE(...) +#endif +#ifndef SECONDARY_CACHE_OPTIONAL +#define SECONDARY_CACHE_OPTIONAL(...) +#endif + +// BASE_REQUIRED_TEMPLATE_TYPE(NAME) +// +// Thread-Specific Data Registry used, shared or exclusive. +BASE_REQUIRED_TEMPLATE_TYPE(TSDRegistryT) + +// Defines the type of Primary allocator to use. +BASE_REQUIRED_TEMPLATE_TYPE(PrimaryT) + +// Defines the type of Secondary allocator to use. +BASE_REQUIRED_TEMPLATE_TYPE(SecondaryT) + +// BASE_OPTIONAL(TYPE, NAME, DEFAULT) +// +// Indicates possible support for Memory Tagging. +BASE_OPTIONAL(const bool, MaySupportMemoryTagging, false) + +// PRIMARY_REQUIRED_TYPE(NAME) +// +// SizeClassMap to use with the Primary. +PRIMARY_REQUIRED_TYPE(SizeClassMap) + +// Defines the type and scale of a compact pointer. A compact pointer can +// be understood as the offset of a pointer within the region it belongs +// to, in increments of a power-of-2 scale. See `CompactPtrScale` also. +PRIMARY_REQUIRED_TYPE(CompactPtrT) + +// PRIMARY_REQUIRED(TYPE, NAME) +// +// The scale of a compact pointer. E.g., Ptr = Base + (CompactPtr << Scale). +PRIMARY_REQUIRED(const uptr, CompactPtrScale) + +// Log2 of the size of a size class region, as used by the Primary. +PRIMARY_REQUIRED(const uptr, RegionSizeLog) + +// Conceptually, a region will be divided into groups based on the address +// range. Each allocation consumes blocks in the same group until exhaustion +// then it pops out blocks in a new group. Therefore, `GroupSizeLog` is always +// smaller or equal to `RegionSizeLog`. Note that `GroupSizeLog` needs to be +// equal to `RegionSizeLog` for SizeClassAllocator32 because of certain +// constraints. +PRIMARY_REQUIRED(const uptr, GroupSizeLog) + +// Call map for user memory with at least this size. Only used with primary64. +PRIMARY_REQUIRED(const uptr, MapSizeIncrement) + +// Defines the minimal & maximal release interval that can be set. +PRIMARY_REQUIRED(const s32, MinReleaseToOsIntervalMs) +PRIMARY_REQUIRED(const s32, MaxReleaseToOsIntervalMs) + +// PRIMARY_OPTIONAL(TYPE, NAME, DEFAULT) +// +// Indicates support for offsetting the start of a region by a random number of +// pages. Only used with primary64. +PRIMARY_OPTIONAL(const bool, EnableRandomOffset, false) + +// PRIMARY_OPTIONAL_TYPE(NAME, DEFAULT) +// +// Use condition variable to shorten the waiting time of refillment of +// freelist. Note that this depends on the implementation of condition +// variable on each platform and the performance may vary so that it does not +// guarantee a performance benefit. +PRIMARY_OPTIONAL_TYPE(ConditionVariableT, ConditionVariableDummy) + +// SECONDARY_REQUIRED_TEMPLATE_TYPE(NAME) +// +// Defines the type of Secondary Cache to use. +SECONDARY_REQUIRED_TEMPLATE_TYPE(CacheT) + +// SECONDARY_CACHE_OPTIONAL(TYPE, NAME, DEFAULT) +// +// Defines the type of cache used by the Secondary. Some additional +// configuration entries can be necessary depending on the Cache. +SECONDARY_CACHE_OPTIONAL(const u32, EntriesArraySize, 0) +SECONDARY_CACHE_OPTIONAL(const u32, QuarantineSize, 0) +SECONDARY_CACHE_OPTIONAL(const u32, DefaultMaxEntriesCount, 0) +SECONDARY_CACHE_OPTIONAL(const u32, DefaultMaxEntrySize, 0) +SECONDARY_CACHE_OPTIONAL(const s32, MinReleaseToOsIntervalMs, INT32_MIN) +SECONDARY_CACHE_OPTIONAL(const s32, MaxReleaseToOsIntervalMs, INT32_MAX) + +#undef SECONDARY_CACHE_OPTIONAL +#undef SECONDARY_REQUIRED_TEMPLATE_TYPE +#undef PRIMARY_OPTIONAL_TYPE +#undef PRIMARY_OPTIONAL +#undef PRIMARY_REQUIRED +#undef PRIMARY_REQUIRED_TYPE +#undef BASE_OPTIONAL +#undef BASE_REQUIRED_TEMPLATE_TYPE diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.h b/compiler-rt/lib/scudo/standalone/allocator_config.h index 3c6aa3acb0e4..1e0cf1015ba6 100644 --- a/compiler-rt/lib/scudo/standalone/allocator_config.h +++ b/compiler-rt/lib/scudo/standalone/allocator_config.h @@ -38,80 +38,10 @@ namespace scudo { -// The combined allocator uses a structure as a template argument that -// specifies the configuration options for the various subcomponents of the -// allocator. -// -// struct ExampleConfig { -// // Indicates possible support for Memory Tagging. -// static const bool MaySupportMemoryTagging = false; -// -// // Thread-Specific Data Registry used, shared or exclusive. -// template using TSDRegistryT = TSDRegistrySharedT; -// -// struct Primary { -// // SizeClassMap to use with the Primary. -// using SizeClassMap = DefaultSizeClassMap; -// -// // Log2 of the size of a size class region, as used by the Primary. -// static const uptr RegionSizeLog = 30U; -// -// // Log2 of the size of block group, as used by the Primary. Each group -// // contains a range of memory addresses, blocks in the range will belong -// // to the same group. In general, single region may have 1 or 2MB group -// // size. Multiple regions will have the group size equal to the region -// // size because the region size is usually smaller than 1 MB. -// // Smaller value gives fine-grained control of memory usage but the -// // trade-off is that it may take longer time of deallocation. -// static const uptr GroupSizeLog = 20U; -// -// // Defines the type and scale of a compact pointer. A compact pointer can -// // be understood as the offset of a pointer within the region it belongs -// // to, in increments of a power-of-2 scale. -// // eg: Ptr = Base + (CompactPtr << Scale). -// typedef u32 CompactPtrT; -// static const uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; -// -// // Indicates support for offsetting the start of a region by -// // a random number of pages. Only used with primary64. -// static const bool EnableRandomOffset = true; -// -// // Call map for user memory with at least this size. Only used with -// // primary64. -// static const uptr MapSizeIncrement = 1UL << 18; -// -// // Defines the minimal & maximal release interval that can be set. -// static const s32 MinReleaseToOsIntervalMs = INT32_MIN; -// static const s32 MaxReleaseToOsIntervalMs = INT32_MAX; -// -// // Use condition variable to shorten the waiting time of refillment of -// // freelist. Note that this depends on the implementation of condition -// // variable on each platform and the performance may vary so that it -// // doesn't guarantee a performance benefit. -// // Note that both variables have to be defined to enable it. -// static const bool UseConditionVariable = true; -// using ConditionVariableT = ConditionVariableLinux; -// }; -// // Defines the type of Primary allocator to use. -// template using PrimaryT = SizeClassAllocator64; -// -// // Defines the type of cache used by the Secondary. Some additional -// // configuration entries can be necessary depending on the Cache. -// struct Secondary { -// struct Cache { -// static const u32 EntriesArraySize = 32U; -// static const u32 QuarantineSize = 0U; -// static const u32 DefaultMaxEntriesCount = 32U; -// static const uptr DefaultMaxEntrySize = 1UL << 19; -// static const s32 MinReleaseToOsIntervalMs = INT32_MIN; -// static const s32 MaxReleaseToOsIntervalMs = INT32_MAX; -// }; -// // Defines the type of Secondary Cache to use. -// template using CacheT = MapAllocatorCache; -// }; -// // Defines the type of Secondary allocator to use. -// template using SecondaryT = MapAllocator; -// }; +// Scudo uses a structure as a template argument that specifies the +// configuration options for the various subcomponents of the allocator. See the +// following configs as examples and check `allocator_config.def` for all the +// available options. #ifndef SCUDO_USE_CUSTOM_CONFIG diff --git a/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h b/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h new file mode 100644 index 000000000000..a51d770b4664 --- /dev/null +++ b/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h @@ -0,0 +1,135 @@ +//===-- allocator_config_wrapper.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_ALLOCATOR_CONFIG_WRAPPER_H_ +#define SCUDO_ALLOCATOR_CONFIG_WRAPPER_H_ + +#include "condition_variable.h" +#include "internal_defs.h" +#include "secondary.h" + +namespace { + +template struct removeConst { + using type = T; +}; +template struct removeConst { + using type = T; +}; + +// This is only used for SFINAE when detecting if a type is defined. +template struct voidAdaptor { + using type = void; +}; + +} // namespace + +namespace scudo { + +#define OPTIONAL_TEMPLATE(TYPE, NAME, DEFAULT, MEMBER) \ + template struct NAME##State { \ + static constexpr removeConst::type getValue() { return DEFAULT; } \ + }; \ + template \ + struct NAME##State { \ + static constexpr removeConst::type getValue() { \ + return Config::MEMBER; \ + } \ + }; + +#define OPTIONAL_TYPE_TEMPLATE(NAME, DEFAULT, MEMBER) \ + template struct NAME##Type { \ + static constexpr bool enabled() { return false; } \ + using NAME = DEFAULT; \ + }; \ + template \ + struct NAME##Type::type> { \ + static constexpr bool enabled() { return true; } \ + using NAME = typename Config::MEMBER; \ + }; + +template struct BaseConfig { +#define BASE_REQUIRED_TEMPLATE_TYPE(NAME) \ + template using NAME = typename AllocatorConfig::template NAME; + +#define BASE_OPTIONAL(TYPE, NAME, DEFAULT) \ + OPTIONAL_TEMPLATE(TYPE, NAME, DEFAULT, NAME) \ + static constexpr removeConst::type get##NAME() { \ + return NAME##State::getValue(); \ + } + +#include "allocator_config.def" +}; // BaseConfig + +template struct PrimaryConfig { + // TODO: Pass this flag through template argument to remove this hard-coded + // function. + static constexpr bool getMaySupportMemoryTagging() { + return BaseConfig::getMaySupportMemoryTagging(); + } + +#define PRIMARY_REQUIRED_TYPE(NAME) \ + using NAME = typename AllocatorConfig::Primary::NAME; + +#define PRIMARY_REQUIRED(TYPE, NAME) \ + static constexpr removeConst::type get##NAME() { \ + return AllocatorConfig::Primary::NAME; \ + } + +#define PRIMARY_OPTIONAL(TYPE, NAME, DEFAULT) \ + OPTIONAL_TEMPLATE(TYPE, NAME, DEFAULT, NAME) \ + static constexpr removeConst::type get##NAME() { \ + return NAME##State::getValue(); \ + } + +#define PRIMARY_OPTIONAL_TYPE(NAME, DEFAULT) \ + OPTIONAL_TYPE_TEMPLATE(NAME, DEFAULT, NAME) \ + static constexpr bool has##NAME() { \ + return NAME##Type::enabled(); \ + } \ + using NAME = typename NAME##Type::NAME; + +#include "allocator_config.def" + +}; // PrimaryConfig + +template struct SecondaryConfig { + // TODO: Pass this flag through template argument to remove this hard-coded + // function. + static constexpr bool getMaySupportMemoryTagging() { + return BaseConfig::getMaySupportMemoryTagging(); + } + +#define SECONDARY_REQUIRED_TEMPLATE_TYPE(NAME) \ + template \ + using NAME = typename AllocatorConfig::Secondary::template NAME; +#include "allocator_config.def" + + struct CacheConfig { + // TODO: Pass this flag through template argument to remove this hard-coded + // function. + static constexpr bool getMaySupportMemoryTagging() { + return BaseConfig::getMaySupportMemoryTagging(); + } + +#define SECONDARY_CACHE_OPTIONAL(TYPE, NAME, DEFAULT) \ + OPTIONAL_TEMPLATE(TYPE, NAME, DEFAULT, Cache::NAME) \ + static constexpr removeConst::type get##NAME() { \ + return NAME##State::getValue(); \ + } +#include "allocator_config.def" + }; // CacheConfig +}; // SecondaryConfig + +#undef OPTIONAL_TEMPLATE +#undef OPTIONAL_TEMPLATE_TYPE + +} // namespace scudo + +#endif // SCUDO_ALLOCATOR_CONFIG_WRAPPER_H_ diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h index 9e1fd6d6dca3..f4dd90aac665 100644 --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -9,6 +9,7 @@ #ifndef SCUDO_COMBINED_H_ #define SCUDO_COMBINED_H_ +#include "allocator_config_wrapper.h" #include "atomic_helpers.h" #include "chunk.h" #include "common.h" @@ -47,11 +48,14 @@ namespace scudo { template class Allocator { public: - using PrimaryT = typename Config::template PrimaryT; - using SecondaryT = typename Config::template SecondaryT; + using AllocatorConfig = BaseConfig; + using PrimaryT = + typename AllocatorConfig::template PrimaryT>; + using SecondaryT = + typename AllocatorConfig::template SecondaryT>; using CacheT = typename PrimaryT::CacheT; typedef Allocator ThisT; - typedef typename Config::template TSDRegistryT TSDRegistryT; + typedef typename AllocatorConfig::template TSDRegistryT TSDRegistryT; void callPostInitCallback() { pthread_once(&PostInitNonce, PostInitCallback); @@ -72,7 +76,7 @@ public: Header.State = Chunk::State::Available; Chunk::storeHeader(Allocator.Cookie, Ptr, &Header); - if (allocatorSupportsMemoryTagging()) + if (allocatorSupportsMemoryTagging()) Ptr = untagPointer(Ptr); void *BlockBegin = Allocator::getBlockBegin(Ptr, &Header); Cache.deallocate(Header.ClassId, BlockBegin); @@ -99,7 +103,8 @@ public: // Reset tag to 0 as this chunk may have been previously used for a tagged // user allocation. - if (UNLIKELY(useMemoryTagging(Allocator.Primary.Options.load()))) + if (UNLIKELY(useMemoryTagging( + Allocator.Primary.Options.load()))) storeTags(reinterpret_cast(Ptr), reinterpret_cast(Ptr) + sizeof(QuarantineBatch)); @@ -159,7 +164,7 @@ public: Primary.Options.set(OptionBit::DeallocTypeMismatch); if (getFlags()->delete_size_mismatch) Primary.Options.set(OptionBit::DeleteSizeMismatch); - if (allocatorSupportsMemoryTagging() && + if (allocatorSupportsMemoryTagging() && systemSupportsMemoryTagging()) Primary.Options.set(OptionBit::UseMemoryTagging); @@ -274,7 +279,7 @@ public: void drainCaches() { TSDRegistry.drainCaches(this); } ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) { - if (!allocatorSupportsMemoryTagging()) + if (!allocatorSupportsMemoryTagging()) return Ptr; auto UntaggedPtr = untagPointer(Ptr); if (UntaggedPtr != Ptr) @@ -286,7 +291,7 @@ public: } ALWAYS_INLINE uptr addHeaderTag(uptr Ptr) { - if (!allocatorSupportsMemoryTagging()) + if (!allocatorSupportsMemoryTagging()) return Ptr; return addFixedTag(Ptr, 2); } @@ -419,7 +424,7 @@ public: // // When memory tagging is enabled, zeroing the contents is done as part of // setting the tag. - if (UNLIKELY(useMemoryTagging(Options))) { + if (UNLIKELY(useMemoryTagging(Options))) { uptr PrevUserPtr; Chunk::UnpackedHeader Header; const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId); @@ -501,7 +506,7 @@ public: } else { Block = addHeaderTag(Block); Ptr = addHeaderTag(Ptr); - if (UNLIKELY(useMemoryTagging(Options))) { + if (UNLIKELY(useMemoryTagging(Options))) { storeTags(reinterpret_cast(Block), reinterpret_cast(Ptr)); storeSecondaryAllocationStackMaybe(Options, Ptr, Size); } @@ -661,7 +666,7 @@ public: (reinterpret_cast(OldTaggedPtr) + NewSize)) & Chunk::SizeOrUnusedBytesMask; Chunk::storeHeader(Cookie, OldPtr, &Header); - if (UNLIKELY(useMemoryTagging(Options))) { + if (UNLIKELY(useMemoryTagging(Options))) { if (ClassId) { resizeTaggedChunk(reinterpret_cast(OldTaggedPtr) + OldSize, reinterpret_cast(OldTaggedPtr) + NewSize, @@ -764,8 +769,9 @@ public: Base = untagPointer(Base); const uptr From = Base; const uptr To = Base + Size; - bool MayHaveTaggedPrimary = allocatorSupportsMemoryTagging() && - systemSupportsMemoryTagging(); + bool MayHaveTaggedPrimary = + allocatorSupportsMemoryTagging() && + systemSupportsMemoryTagging(); auto Lambda = [this, From, To, MayHaveTaggedPrimary, Callback, Arg](uptr Block) { if (Block < From || Block >= To) @@ -786,9 +792,9 @@ public: } if (Header.State == Chunk::State::Allocated) { uptr TaggedChunk = Chunk; - if (allocatorSupportsMemoryTagging()) + if (allocatorSupportsMemoryTagging()) TaggedChunk = untagPointer(TaggedChunk); - if (useMemoryTagging(Primary.Options.load())) + if (useMemoryTagging(Primary.Options.load())) TaggedChunk = loadTag(Chunk); Callback(TaggedChunk, getSize(reinterpret_cast(Chunk), &Header), Arg); @@ -887,7 +893,7 @@ public: } bool useMemoryTaggingTestOnly() const { - return useMemoryTagging(Primary.Options.load()); + return useMemoryTagging(Primary.Options.load()); } void disableMemoryTagging() { // If we haven't been initialized yet, we need to initialize now in order to @@ -897,7 +903,7 @@ public: // callback), which may cause mappings to be created with memory tagging // enabled. TSDRegistry.initOnceMaybe(this); - if (allocatorSupportsMemoryTagging()) { + if (allocatorSupportsMemoryTagging()) { Secondary.disableMemoryTagging(); Primary.Options.clear(OptionBit::UseMemoryTagging); } @@ -983,7 +989,7 @@ public: // should not be able to crash the crash dumper (crash_dump on Android). // See also the get_error_info_fuzzer. *ErrorInfo = {}; - if (!allocatorSupportsMemoryTagging() || + if (!allocatorSupportsMemoryTagging() || MemoryAddr + MemorySize < MemoryAddr) return; @@ -1032,7 +1038,7 @@ private: static_assert(MinAlignment >= sizeof(Chunk::PackedHeader), "Minimal alignment must at least cover a chunk header."); - static_assert(!allocatorSupportsMemoryTagging() || + static_assert(!allocatorSupportsMemoryTagging() || MinAlignment >= archMemoryTagGranuleSize(), ""); @@ -1142,7 +1148,7 @@ private: const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes; if (LIKELY(Header->ClassId)) return SizeOrUnusedBytes; - if (allocatorSupportsMemoryTagging()) + if (allocatorSupportsMemoryTagging()) Ptr = untagPointer(const_cast(Ptr)); return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) - reinterpret_cast(Ptr) - SizeOrUnusedBytes; @@ -1162,12 +1168,12 @@ private: Header->State = Chunk::State::Available; else Header->State = Chunk::State::Quarantined; - Header->OriginOrWasZeroed = useMemoryTagging(Options) && + Header->OriginOrWasZeroed = useMemoryTagging(Options) && Header->ClassId && !TSDRegistry.getDisableMemInit(); Chunk::storeHeader(Cookie, Ptr, Header); - if (UNLIKELY(useMemoryTagging(Options))) { + if (UNLIKELY(useMemoryTagging(Options))) { u8 PrevTag = extractTag(reinterpret_cast(TaggedPtr)); storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size); if (Header->ClassId) { @@ -1184,7 +1190,7 @@ private: } } if (BypassQuarantine) { - if (allocatorSupportsMemoryTagging()) + if (allocatorSupportsMemoryTagging()) Ptr = untagPointer(Ptr); void *BlockBegin = getBlockBegin(Ptr, Header); const uptr ClassId = Header->ClassId; @@ -1201,7 +1207,7 @@ private: if (CacheDrained) Primary.tryReleaseToOS(ClassId, ReleaseToOS::Normal); } else { - if (UNLIKELY(useMemoryTagging(Options))) + if (UNLIKELY(useMemoryTagging(Options))) storeTags(reinterpret_cast(BlockBegin), reinterpret_cast(Ptr)); Secondary.deallocate(Options, BlockBegin); diff --git a/compiler-rt/lib/scudo/standalone/condition_variable.h b/compiler-rt/lib/scudo/standalone/condition_variable.h index 4afebdc9d04c..3f16c86651e7 100644 --- a/compiler-rt/lib/scudo/standalone/condition_variable.h +++ b/compiler-rt/lib/scudo/standalone/condition_variable.h @@ -39,22 +39,6 @@ public: } }; -template -struct ConditionVariableState { - static constexpr bool enabled() { return false; } - // This is only used for compilation purpose so that we won't end up having - // many conditional compilations. If you want to use `ConditionVariableDummy`, - // define `ConditionVariableT` in your allocator configuration. See - // allocator_config.h for more details. - using ConditionVariableT = ConditionVariableDummy; -}; - -template -struct ConditionVariableState { - static constexpr bool enabled() { return Config::UseConditionVariable; } - using ConditionVariableT = typename Config::ConditionVariableT; -}; - } // namespace scudo #endif // SCUDO_CONDITION_VARIABLE_H_ diff --git a/compiler-rt/lib/scudo/standalone/memtag.h b/compiler-rt/lib/scudo/standalone/memtag.h index aaed2192ad75..1f6983e99404 100644 --- a/compiler-rt/lib/scudo/standalone/memtag.h +++ b/compiler-rt/lib/scudo/standalone/memtag.h @@ -326,7 +326,7 @@ inline void *addFixedTag(void *Ptr, uptr Tag) { template inline constexpr bool allocatorSupportsMemoryTagging() { - return archSupportsMemoryTagging() && Config::MaySupportMemoryTagging && + return archSupportsMemoryTagging() && Config::getMaySupportMemoryTagging() && (1 << SCUDO_MIN_ALIGNMENT_LOG) >= archMemoryTagGranuleSize(); } diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h index c86e75b8fd66..1d8a77b73e5c 100644 --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -43,14 +43,13 @@ namespace scudo { template class SizeClassAllocator32 { public: - typedef typename Config::Primary::CompactPtrT CompactPtrT; - typedef typename Config::Primary::SizeClassMap SizeClassMap; - static const uptr GroupSizeLog = Config::Primary::GroupSizeLog; + typedef typename Config::CompactPtrT CompactPtrT; + typedef typename Config::SizeClassMap SizeClassMap; + static const uptr GroupSizeLog = Config::getGroupSizeLog(); // The bytemap can only track UINT8_MAX - 1 classes. static_assert(SizeClassMap::LargestClassId <= (UINT8_MAX - 1), ""); // Regions should be large enough to hold the largest Block. - static_assert((1UL << Config::Primary::RegionSizeLog) >= - SizeClassMap::MaxSize, + static_assert((1UL << Config::getRegionSizeLog()) >= SizeClassMap::MaxSize, ""); typedef SizeClassAllocator32 ThisT; typedef SizeClassAllocatorLocalCache CacheT; @@ -331,9 +330,9 @@ public: bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { - const s32 Interval = Max(Min(static_cast(Value), - Config::Primary::MaxReleaseToOsIntervalMs), - Config::Primary::MinReleaseToOsIntervalMs); + const s32 Interval = Max( + Min(static_cast(Value), Config::getMaxReleaseToOsIntervalMs()), + Config::getMinReleaseToOsIntervalMs()); atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval); return true; } @@ -373,9 +372,9 @@ public: private: static const uptr NumClasses = SizeClassMap::NumClasses; - static const uptr RegionSize = 1UL << Config::Primary::RegionSizeLog; - static const uptr NumRegions = - SCUDO_MMAP_RANGE_SIZE >> Config::Primary::RegionSizeLog; + static const uptr RegionSize = 1UL << Config::getRegionSizeLog(); + static const uptr NumRegions = SCUDO_MMAP_RANGE_SIZE >> + Config::getRegionSizeLog(); static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U; typedef FlatByteMap ByteMap; @@ -408,7 +407,7 @@ private: static_assert(sizeof(SizeClassInfo) % SCUDO_CACHE_LINE_SIZE == 0, ""); uptr computeRegionId(uptr Mem) { - const uptr Id = Mem >> Config::Primary::RegionSizeLog; + const uptr Id = Mem >> Config::getRegionSizeLog(); CHECK_LT(Id, NumRegions); return Id; } @@ -437,7 +436,7 @@ private: unmap(reinterpret_cast(End), MapEnd - End); DCHECK_EQ(Region % RegionSize, 0U); - static_assert(Config::Primary::RegionSizeLog == GroupSizeLog, + static_assert(Config::getRegionSizeLog() == GroupSizeLog, "Memory group should be the same size as Region"); return Region; diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h index d89a2e6a4e5c..f5e4ab57b4df 100644 --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -47,13 +47,12 @@ namespace scudo { template class SizeClassAllocator64 { public: - typedef typename Config::Primary::CompactPtrT CompactPtrT; - typedef typename Config::Primary::SizeClassMap SizeClassMap; - typedef typename ConditionVariableState< - typename Config::Primary>::ConditionVariableT ConditionVariableT; - static const uptr CompactPtrScale = Config::Primary::CompactPtrScale; - static const uptr RegionSizeLog = Config::Primary::RegionSizeLog; - static const uptr GroupSizeLog = Config::Primary::GroupSizeLog; + typedef typename Config::CompactPtrT CompactPtrT; + typedef typename Config::SizeClassMap SizeClassMap; + typedef typename Config::ConditionVariableT ConditionVariableT; + static const uptr CompactPtrScale = Config::getCompactPtrScale(); + static const uptr RegionSizeLog = Config::getRegionSizeLog(); + static const uptr GroupSizeLog = Config::getGroupSizeLog(); static_assert(RegionSizeLog >= GroupSizeLog, "Group size shouldn't be greater than the region size"); static const uptr GroupScale = GroupSizeLog - CompactPtrScale; @@ -74,7 +73,7 @@ public: static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; } static bool conditionVariableEnabled() { - return ConditionVariableState::enabled(); + return Config::hasConditionVariableT(); } void init(s32 ReleaseToOsInterval) NO_THREAD_SAFETY_ANALYSIS { @@ -135,7 +134,7 @@ public: // The actual start of a region is offset by a random number of pages // when PrimaryEnableRandomOffset is set. Region->RegionBeg = (PrimaryBase + (I << RegionSizeLog)) + - (Config::Primary::EnableRandomOffset + (Config::getEnableRandomOffset() ? ((getRandomModN(&Seed, 16) + 1) * PageSize) : 0); Region->RandState = getRandomU32(&Seed); @@ -400,9 +399,9 @@ public: bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { - const s32 Interval = Max(Min(static_cast(Value), - Config::Primary::MaxReleaseToOsIntervalMs), - Config::Primary::MinReleaseToOsIntervalMs); + const s32 Interval = Max( + Min(static_cast(Value), Config::getMaxReleaseToOsIntervalMs()), + Config::getMinReleaseToOsIntervalMs()); atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval); return true; } @@ -516,7 +515,7 @@ private: static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr PrimarySize = RegionSize * NumClasses; - static const uptr MapSizeIncrement = Config::Primary::MapSizeIncrement; + static const uptr MapSizeIncrement = Config::getMapSizeIncrement(); // Fill at most this number of batches from the newly map'd memory. static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U; diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h index 732fd307ed2f..202c55cc1a92 100644 --- a/compiler-rt/lib/scudo/standalone/secondary.h +++ b/compiler-rt/lib/scudo/standalone/secondary.h @@ -173,8 +173,6 @@ public: template class MapAllocatorCache { public: - using CacheConfig = typename Config::Secondary::Cache; - void getStats(ScopedString *Str) { ScopedLock L(Mutex); uptr Integral; @@ -199,16 +197,16 @@ public: } // Ensure the default maximum specified fits the array. - static_assert(CacheConfig::DefaultMaxEntriesCount <= - CacheConfig::EntriesArraySize, + static_assert(Config::getDefaultMaxEntriesCount() <= + Config::getEntriesArraySize(), ""); void init(s32 ReleaseToOsInterval) NO_THREAD_SAFETY_ANALYSIS { DCHECK_EQ(EntriesCount, 0U); setOption(Option::MaxCacheEntriesCount, - static_cast(CacheConfig::DefaultMaxEntriesCount)); + static_cast(Config::getDefaultMaxEntriesCount())); setOption(Option::MaxCacheEntrySize, - static_cast(CacheConfig::DefaultMaxEntrySize)); + static_cast(Config::getDefaultMaxEntrySize())); setOption(Option::ReleaseInterval, static_cast(ReleaseToOsInterval)); } @@ -253,9 +251,9 @@ public: // just unmap it. break; } - if (CacheConfig::QuarantineSize && useMemoryTagging(Options)) { + if (Config::getQuarantineSize() && useMemoryTagging(Options)) { QuarantinePos = - (QuarantinePos + 1) % Max(CacheConfig::QuarantineSize, 1u); + (QuarantinePos + 1) % Max(Config::getQuarantineSize(), 1u); if (!Quarantine[QuarantinePos].isValid()) { Quarantine[QuarantinePos] = Entry; return; @@ -382,14 +380,14 @@ public: bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { const s32 Interval = Max( - Min(static_cast(Value), CacheConfig::MaxReleaseToOsIntervalMs), - CacheConfig::MinReleaseToOsIntervalMs); + Min(static_cast(Value), Config::getMaxReleaseToOsIntervalMs()), + Config::getMinReleaseToOsIntervalMs()); atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval); return true; } if (O == Option::MaxCacheEntriesCount) { const u32 MaxCount = static_cast(Value); - if (MaxCount > CacheConfig::EntriesArraySize) + if (MaxCount > Config::getEntriesArraySize()) return false; atomic_store_relaxed(&MaxEntriesCount, MaxCount); return true; @@ -406,7 +404,7 @@ public: void disableMemoryTagging() EXCLUDES(Mutex) { ScopedLock L(Mutex); - for (u32 I = 0; I != CacheConfig::QuarantineSize; ++I) { + for (u32 I = 0; I != Config::getQuarantineSize(); ++I) { if (Quarantine[I].isValid()) { MemMapT &MemMap = Quarantine[I].MemMap; MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); @@ -431,11 +429,11 @@ public: private: void empty() { - MemMapT MapInfo[CacheConfig::EntriesArraySize]; + MemMapT MapInfo[Config::getEntriesArraySize()]; uptr N = 0; { ScopedLock L(Mutex); - for (uptr I = 0; I < CacheConfig::EntriesArraySize; I++) { + for (uptr I = 0; I < Config::getEntriesArraySize(); I++) { if (!Entries[I].isValid()) continue; MapInfo[N] = Entries[I].MemMap; @@ -468,9 +466,9 @@ private: if (!EntriesCount || OldestTime == 0 || OldestTime > Time) return; OldestTime = 0; - for (uptr I = 0; I < CacheConfig::QuarantineSize; I++) + for (uptr I = 0; I < Config::getQuarantineSize(); I++) releaseIfOlderThan(Quarantine[I], Time); - for (uptr I = 0; I < CacheConfig::EntriesArraySize; I++) + for (uptr I = 0; I < Config::getEntriesArraySize(); I++) releaseIfOlderThan(Entries[I], Time); } @@ -485,8 +483,8 @@ private: u32 CallsToRetrieve GUARDED_BY(Mutex) = 0; u32 SuccessfulRetrieves GUARDED_BY(Mutex) = 0; - CachedBlock Entries[CacheConfig::EntriesArraySize] GUARDED_BY(Mutex) = {}; - NonZeroLengthArray + CachedBlock Entries[Config::getEntriesArraySize()] GUARDED_BY(Mutex) = {}; + NonZeroLengthArray Quarantine GUARDED_BY(Mutex) = {}; }; @@ -555,7 +553,7 @@ public: void getStats(ScopedString *Str); private: - typename Config::Secondary::template CacheT Cache; + typename Config::template CacheT Cache; mutable HybridMutex Mutex; DoublyLinkedList InUseBlocks GUARDED_BY(Mutex); diff --git a/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt index ac92805872f9..1786756fa5ea 100644 --- a/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt +++ b/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt @@ -90,6 +90,7 @@ macro(add_scudo_unittest testname) endmacro() set(SCUDO_UNIT_TEST_SOURCES + allocator_config_test.cpp atomic_test.cpp bytemap_test.cpp checksum_test.cpp diff --git a/compiler-rt/lib/scudo/standalone/tests/allocator_config_test.cpp b/compiler-rt/lib/scudo/standalone/tests/allocator_config_test.cpp new file mode 100644 index 000000000000..4c4ceb832e27 --- /dev/null +++ b/compiler-rt/lib/scudo/standalone/tests/allocator_config_test.cpp @@ -0,0 +1,119 @@ +//===-- allocator_config_test.cpp -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "tests/scudo_unit_test.h" + +#include "allocator_config.h" +#include "allocator_config_wrapper.h" +#include "common.h" +#include "secondary.h" + +#include + +struct TestBaseConfig { + template using TSDRegistryT = void; + template using PrimaryT = void; + template using SecondaryT = void; +}; + +struct TestBaseConfigEnableOptionalFlag : public TestBaseConfig { + static const bool MaySupportMemoryTagging = true; + // Use the getter to avoid the test to `use` the address of static const + // variable (which requires additional explicit definition). + static bool getMaySupportMemoryTagging() { return MaySupportMemoryTagging; } +}; + +struct TestBasePrimaryConfig { + using SizeClassMap = void; + static const scudo::uptr RegionSizeLog = 18U; + static const scudo::uptr GroupSizeLog = 18U; + static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX; + typedef scudo::uptr CompactPtrT; + static const scudo::uptr CompactPtrScale = 0; + static const scudo::uptr MapSizeIncrement = 1UL << 18; +}; + +struct TestPrimaryConfig : public TestBaseConfig { + struct Primary : TestBasePrimaryConfig {}; +}; + +struct TestPrimaryConfigEnableOptionalFlag : public TestBaseConfig { + struct Primary : TestBasePrimaryConfig { + static const bool EnableRandomOffset = true; + static bool getEnableRandomOffset() { return EnableRandomOffset; } + }; +}; + +struct TestPrimaryConfigEnableOptionalType : public TestBaseConfig { + struct DummyConditionVariable {}; + + struct Primary : TestBasePrimaryConfig { + using ConditionVariableT = DummyConditionVariable; + }; +}; + +struct TestSecondaryConfig : public TestPrimaryConfig { + struct Secondary { + template + using CacheT = scudo::MapAllocatorNoCache; + }; +}; + +struct TestSecondaryCacheConfigEnableOptionalFlag : public TestPrimaryConfig { + struct Secondary { + struct Cache { + static const scudo::u32 EntriesArraySize = 256U; + static scudo::u32 getEntriesArraySize() { return EntriesArraySize; } + }; + template using CacheT = scudo::MapAllocatorCache; + }; +}; + +TEST(ScudoAllocatorConfigTest, VerifyOptionalFlags) { + // Test the top level allocator optional config. + // + // `MaySupportMemoryTagging` is default off. + EXPECT_FALSE(scudo::BaseConfig::getMaySupportMemoryTagging()); + EXPECT_EQ(scudo::BaseConfig< + TestBaseConfigEnableOptionalFlag>::getMaySupportMemoryTagging(), + TestBaseConfigEnableOptionalFlag::getMaySupportMemoryTagging()); + + // Test primary optional config. + // + // `EnableRandomeOffset` is default off. + EXPECT_FALSE( + scudo::PrimaryConfig::getEnableRandomOffset()); + EXPECT_EQ( + scudo::PrimaryConfig< + TestPrimaryConfigEnableOptionalFlag>::getEnableRandomOffset(), + TestPrimaryConfigEnableOptionalFlag::Primary::getEnableRandomOffset()); + + // `ConditionVariableT` is default off. + EXPECT_FALSE( + scudo::PrimaryConfig::hasConditionVariableT()); + EXPECT_TRUE(scudo::PrimaryConfig< + TestPrimaryConfigEnableOptionalType>::hasConditionVariableT()); + EXPECT_TRUE((std::is_same_v< + typename scudo::PrimaryConfig< + TestPrimaryConfigEnableOptionalType>::ConditionVariableT, + typename TestPrimaryConfigEnableOptionalType::Primary:: + ConditionVariableT>)); + + // Test secondary cache optional config. + using NoCacheConfig = + scudo::SecondaryConfig::CacheConfig; + // `EntriesArraySize` is default 0. + EXPECT_EQ(NoCacheConfig::getEntriesArraySize(), 0U); + + using CacheConfig = scudo::SecondaryConfig< + TestSecondaryCacheConfigEnableOptionalFlag>::CacheConfig; + EXPECT_EQ(CacheConfig::getEntriesArraySize(), + TestSecondaryCacheConfigEnableOptionalFlag::Secondary::Cache:: + getEntriesArraySize()); +} diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp index 13d627b11680..6a311adc55e4 100644 --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -190,7 +190,6 @@ struct TestConditionVariableConfig { #endif static const scudo::s32 MinReleaseToOsIntervalMs = 1000; static const scudo::s32 MaxReleaseToOsIntervalMs = 1000; - static const bool UseConditionVariable = true; #if SCUDO_LINUX using ConditionVariableT = scudo::ConditionVariableLinux; #else diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp index f64a5143b30d..683ce3e59659 100644 --- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp @@ -9,6 +9,7 @@ #include "tests/scudo_unit_test.h" #include "allocator_config.h" +#include "allocator_config_wrapper.h" #include "condition_variable.h" #include "primary32.h" #include "primary64.h" @@ -29,6 +30,9 @@ template struct TestConfig1 { static const bool MaySupportMemoryTagging = false; + template using TSDRegistryT = void; + template using PrimaryT = void; + template using SecondaryT = void; struct Primary { using SizeClassMap = SizeClassMapT; @@ -45,6 +49,9 @@ template struct TestConfig1 { template struct TestConfig2 { static const bool MaySupportMemoryTagging = false; + template using TSDRegistryT = void; + template using PrimaryT = void; + template using SecondaryT = void; struct Primary { using SizeClassMap = SizeClassMapT; @@ -66,6 +73,9 @@ template struct TestConfig2 { template struct TestConfig3 { static const bool MaySupportMemoryTagging = true; + template using TSDRegistryT = void; + template using PrimaryT = void; + template using SecondaryT = void; struct Primary { using SizeClassMap = SizeClassMapT; @@ -87,6 +97,9 @@ template struct TestConfig3 { template struct TestConfig4 { static const bool MaySupportMemoryTagging = true; + template using TSDRegistryT = void; + template using PrimaryT = void; + template using SecondaryT = void; struct Primary { using SizeClassMap = SizeClassMapT; @@ -109,6 +122,9 @@ template struct TestConfig4 { // This is the only test config that enables the condition variable. template struct TestConfig5 { static const bool MaySupportMemoryTagging = true; + template using TSDRegistryT = void; + template using PrimaryT = void; + template using SecondaryT = void; struct Primary { using SizeClassMap = SizeClassMapT; @@ -125,7 +141,6 @@ template struct TestConfig5 { typedef scudo::u32 CompactPtrT; static const bool EnableRandomOffset = true; static const scudo::uptr MapSizeIncrement = 1UL << 18; - static const bool UseConditionVariable = true; #if SCUDO_LINUX using ConditionVariableT = scudo::ConditionVariableLinux; #else @@ -139,10 +154,12 @@ struct Config : public BaseConfig {}; template